]> git.proxmox.com Git - ceph.git/blame - ceph/src/seastar/dpdk/drivers/net/mlx5/mlx5_fdir.c
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update download target update for octopus release
[ceph.git] / ceph / src / seastar / dpdk / drivers / net / mlx5 / mlx5_fdir.c
CommitLineData
7c673cae
FG		 1/*-
2 * BSD LICENSE
3 *
4 * Copyright 2015 6WIND S.A.
5 * Copyright 2015 Mellanox.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <stddef.h>
35#include <assert.h>
36#include <stdint.h>
37#include <string.h>
38#include <errno.h>
39
40/* Verbs header. */
41/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
42#ifdef PEDANTIC
43#pragma GCC diagnostic ignored "-Wpedantic"
44#endif
45#include <infiniband/verbs.h>
46#ifdef PEDANTIC
47#pragma GCC diagnostic error "-Wpedantic"
48#endif
49
50/* DPDK headers don't like -pedantic. */
51#ifdef PEDANTIC
52#pragma GCC diagnostic ignored "-Wpedantic"
53#endif
54#include <rte_ether.h>
55#include <rte_malloc.h>
56#include <rte_ethdev.h>
57#include <rte_common.h>
11fdf7f2
TL		 58#include <rte_flow.h>
59#include <rte_flow_driver.h>
7c673cae
FG		 60#ifdef PEDANTIC
61#pragma GCC diagnostic error "-Wpedantic"
62#endif
63
64#include "mlx5.h"
65#include "mlx5_rxtx.h"
66
67struct fdir_flow_desc {
68 uint16_t dst_port;
69 uint16_t src_port;
70 uint32_t src_ip[4];
71 uint32_t dst_ip[4];
72 uint8_t mac[6];
73 uint16_t vlan_tag;
74 enum hash_rxq_type type;
75};
76
77struct mlx5_fdir_filter {
78 LIST_ENTRY(mlx5_fdir_filter) next;
79 uint16_t queue; /* Queue assigned to if FDIR match. */
80 enum rte_eth_fdir_behavior behavior;
81 struct fdir_flow_desc desc;
82 struct ibv_exp_flow *flow;
83};
84
85LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
86
87/**
88 * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
89 *
90 * @param[in] fdir_filter
91 * DPDK filter structure to convert.
92 * @param[out] desc
93 * Resulting mlx5 filter descriptor.
94 * @param mode
95 * Flow director mode.
96 */
97static void
98fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
99 struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
100{
101 /* Initialize descriptor. */
102 memset(desc, 0, sizeof(*desc));
103
104 /* Set VLAN ID. */
105 desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
106
107 /* Set MAC address. */
108 if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
109 rte_memcpy(desc->mac,
110 fdir_filter->input.flow.mac_vlan_flow.mac_addr.
111 addr_bytes,
112 sizeof(desc->mac));
113 desc->type = HASH_RXQ_ETH;
114 return;
115 }
116
117 /* Set mode */
118 switch (fdir_filter->input.flow_type) {
119 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
120 desc->type = HASH_RXQ_UDPV4;
121 break;
122 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
123 desc->type = HASH_RXQ_TCPV4;
124 break;
125 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
126 desc->type = HASH_RXQ_IPV4;
127 break;
128 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
129 desc->type = HASH_RXQ_UDPV6;
130 break;
131 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
132 desc->type = HASH_RXQ_TCPV6;
133 break;
134 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
135 desc->type = HASH_RXQ_IPV6;
136 break;
137 default:
138 break;
139 }
140
141 /* Set flow values */
142 switch (fdir_filter->input.flow_type) {
143 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
144 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
145 desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
146 desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
147 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
148 desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
149 desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
150 break;
151 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
152 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
153 desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
154 desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
155 /* Fall through. */
156 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
157 rte_memcpy(desc->src_ip,
158 fdir_filter->input.flow.ipv6_flow.src_ip,
159 sizeof(desc->src_ip));
160 rte_memcpy(desc->dst_ip,
161 fdir_filter->input.flow.ipv6_flow.dst_ip,
162 sizeof(desc->dst_ip));
163 break;
164 default:
165 break;
166 }
167}
168
169/**
170 * Check if two flow descriptors overlap according to configured mask.
171 *
172 * @param priv
173 * Private structure that provides flow director mask.
174 * @param desc1
175 * First flow descriptor to compare.
176 * @param desc2
177 * Second flow descriptor to compare.
178 *
179 * @return
180 * Nonzero if descriptors overlap.
181 */
182static int
183priv_fdir_overlap(const struct priv *priv,
184 const struct fdir_flow_desc *desc1,
185 const struct fdir_flow_desc *desc2)
186{
187 const struct rte_eth_fdir_masks *mask =
188 &priv->dev->data->dev_conf.fdir_conf.mask;
189 unsigned int i;
190
191 if (desc1->type != desc2->type)
192 return 0;
193 /* Ignore non masked bits. */
194 for (i = 0; i != RTE_DIM(desc1->mac); ++i)
195 if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
196 (desc2->mac[i] & mask->mac_addr_byte_mask))
197 return 0;
198 if (((desc1->src_port & mask->src_port_mask) !=
199 (desc2->src_port & mask->src_port_mask)) ||
200 ((desc1->dst_port & mask->dst_port_mask) !=
201 (desc2->dst_port & mask->dst_port_mask)))
202 return 0;
203 switch (desc1->type) {
204 case HASH_RXQ_IPV4:
205 case HASH_RXQ_UDPV4:
206 case HASH_RXQ_TCPV4:
207 if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
208 (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
209 ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
210 (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
211 return 0;
212 break;
213 case HASH_RXQ_IPV6:
214 case HASH_RXQ_UDPV6:
215 case HASH_RXQ_TCPV6:
216 for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
217 if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
218 (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
219 ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
220 (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
221 return 0;
222 break;
223 default:
224 break;
225 }
226 return 1;
227}
228
229/**
230 * Create flow director steering rule for a specific filter.
231 *
232 * @param priv
233 * Private structure.
234 * @param mlx5_fdir_filter
235 * Filter to create a steering rule for.
236 * @param fdir_queue
237 * Flow director queue for matching packets.
238 *
239 * @return
240 * 0 on success, errno value on failure.
241 */
242static int
243priv_fdir_flow_add(struct priv *priv,
244 struct mlx5_fdir_filter *mlx5_fdir_filter,
245 struct fdir_queue *fdir_queue)
246{
247 struct ibv_exp_flow *flow;
248 struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
249 enum rte_fdir_mode fdir_mode =
250 priv->dev->data->dev_conf.fdir_conf.mode;
251 struct rte_eth_fdir_masks *mask =
252 &priv->dev->data->dev_conf.fdir_conf.mask;
253 FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
254 struct ibv_exp_flow_attr *attr = &data->attr;
255 uintptr_t spec_offset = (uintptr_t)&data->spec;
256 struct ibv_exp_flow_spec_eth *spec_eth;
257 struct ibv_exp_flow_spec_ipv4 *spec_ipv4;
258 struct ibv_exp_flow_spec_ipv6 *spec_ipv6;
259 struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
260 struct mlx5_fdir_filter *iter_fdir_filter;
261 unsigned int i;
262
263 /* Abort if an existing flow overlaps this one to avoid packet
264 * duplication, even if it targets another queue. */
265 LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
266 if ((iter_fdir_filter != mlx5_fdir_filter) &&
267 (iter_fdir_filter->flow != NULL) &&
268 (priv_fdir_overlap(priv,
269 &mlx5_fdir_filter->desc,
270 &iter_fdir_filter->desc)))
271 return EEXIST;
272
273 /*
274 * No padding must be inserted by the compiler between attr and spec.
275 * This layout is expected by libibverbs.
276 */
277 assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
278 priv_flow_attr(priv, attr, sizeof(data), desc->type);
279
280 /* Set Ethernet spec */
281 spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
282
283 /* The first specification must be Ethernet. */
284 assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
285 assert(spec_eth->size == sizeof(*spec_eth));
286
287 /* VLAN ID */
288 spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
289 spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
290
291 /* Update priority */
292 attr->priority = 2;
293
294 if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
295 /* MAC Address */
296 for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
297 spec_eth->val.dst_mac[i] =
298 desc->mac[i] & mask->mac_addr_byte_mask;
299 spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
300 }
301 goto create_flow;
302 }
303
304 switch (desc->type) {
305 case HASH_RXQ_IPV4:
306 case HASH_RXQ_UDPV4:
307 case HASH_RXQ_TCPV4:
308 spec_offset += spec_eth->size;
309
310 /* Set IP spec */
311 spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset;
312
313 /* The second specification must be IP. */
314 assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4);
315 assert(spec_ipv4->size == sizeof(*spec_ipv4));
316
317 spec_ipv4->val.src_ip =
318 desc->src_ip[0] & mask->ipv4_mask.src_ip;
319 spec_ipv4->val.dst_ip =
320 desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
321 spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
322 spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
323
324 /* Update priority */
325 attr->priority = 1;
326
327 if (desc->type == HASH_RXQ_IPV4)
328 goto create_flow;
329
330 spec_offset += spec_ipv4->size;
331 break;
332 case HASH_RXQ_IPV6:
333 case HASH_RXQ_UDPV6:
334 case HASH_RXQ_TCPV6:
335 spec_offset += spec_eth->size;
336
337 /* Set IP spec */
338 spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset;
339
340 /* The second specification must be IP. */
341 assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
342 assert(spec_ipv6->size == sizeof(*spec_ipv6));
343
344 for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
345 ((uint32_t *)spec_ipv6->val.src_ip)[i] =
346 desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
347 ((uint32_t *)spec_ipv6->val.dst_ip)[i] =
348 desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
349 }
350 rte_memcpy(spec_ipv6->mask.src_ip,
351 mask->ipv6_mask.src_ip,
352 sizeof(spec_ipv6->mask.src_ip));
353 rte_memcpy(spec_ipv6->mask.dst_ip,
354 mask->ipv6_mask.dst_ip,
355 sizeof(spec_ipv6->mask.dst_ip));
356
357 /* Update priority */
358 attr->priority = 1;
359
360 if (desc->type == HASH_RXQ_IPV6)
361 goto create_flow;
362
363 spec_offset += spec_ipv6->size;
364 break;
365 default:
366 ERROR("invalid flow attribute type");
367 return EINVAL;
368 }
369
370 /* Set TCP/UDP flow specification. */
371 spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
372
373 /* The third specification must be TCP/UDP. */
374 assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
375 spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
376 assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
377
378 spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
379 spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
380 spec_tcp_udp->mask.src_port = mask->src_port_mask;
381 spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
382
383 /* Update priority */
384 attr->priority = 0;
385
386create_flow:
387
388 errno = 0;
389 flow = ibv_exp_create_flow(fdir_queue->qp, attr);
390 if (flow == NULL) {
391 /* It's not clear whether errno is always set in this case. */
392 ERROR("%p: flow director configuration failed, errno=%d: %s",
393 (void *)priv, errno,
394 (errno ? strerror(errno) : "Unknown error"));
395 if (errno)
396 return errno;
397 return EINVAL;
398 }
399
400 DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
401 mlx5_fdir_filter->flow = flow;
402 return 0;
403}
404
405/**
406 * Destroy a flow director queue.
407 *
408 * @param fdir_queue
409 * Flow director queue to be destroyed.
410 */
411void
412priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue)
413{
414 struct mlx5_fdir_filter *fdir_filter;
415
416 /* Disable filter flows still applying to this queue. */
417 LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) {
418 unsigned int idx = fdir_filter->queue;
419 struct rxq_ctrl *rxq_ctrl =
420 container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
421
422 assert(idx < priv->rxqs_n);
423 if (fdir_queue == rxq_ctrl->fdir_queue &&
424 fdir_filter->flow != NULL) {
425 claim_zero(ibv_exp_destroy_flow(fdir_filter->flow));
426 fdir_filter->flow = NULL;
427 }
428 }
429 assert(fdir_queue->qp);
430 claim_zero(ibv_destroy_qp(fdir_queue->qp));
431 assert(fdir_queue->ind_table);
432 claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table));
433 if (fdir_queue->wq)
434 claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
435 if (fdir_queue->cq)
436 claim_zero(ibv_destroy_cq(fdir_queue->cq));
437#ifndef NDEBUG
438 memset(fdir_queue, 0x2a, sizeof(*fdir_queue));
439#endif
440 rte_free(fdir_queue);
441}
442
443/**
444 * Create a flow director queue.
445 *
446 * @param priv
447 * Private structure.
448 * @param wq
449 * Work queue to route matched packets to, NULL if one needs to
450 * be created.
451 *
452 * @return
453 * Related flow director queue on success, NULL otherwise.
454 */
455static struct fdir_queue *
456priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
457 unsigned int socket)
458{
459 struct fdir_queue *fdir_queue;
460
461 fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue),
462 0, socket);
463 if (!fdir_queue) {
464 ERROR("cannot allocate flow director queue");
465 return NULL;
466 }
467 assert(priv->pd);
468 assert(priv->ctx);
469 if (!wq) {
470 fdir_queue->cq = ibv_exp_create_cq(
471 priv->ctx, 1, NULL, NULL, 0,
472 &(struct ibv_exp_cq_init_attr){
473 .comp_mask = 0,
474 });
475 if (!fdir_queue->cq) {
476 ERROR("cannot create flow director CQ");
477 goto error;
478 }
479 fdir_queue->wq = ibv_exp_create_wq(
480 priv->ctx,
481 &(struct ibv_exp_wq_init_attr){
482 .wq_type = IBV_EXP_WQT_RQ,
483 .max_recv_wr = 1,
484 .max_recv_sge = 1,
485 .pd = priv->pd,
486 .cq = fdir_queue->cq,
487 });
488 if (!fdir_queue->wq) {
489 ERROR("cannot create flow director WQ");
490 goto error;
491 }
492 wq = fdir_queue->wq;
493 }
494 fdir_queue->ind_table = ibv_exp_create_rwq_ind_table(
495 priv->ctx,
496 &(struct ibv_exp_rwq_ind_table_init_attr){
497 .pd = priv->pd,
498 .log_ind_tbl_size = 0,
499 .ind_tbl = &wq,
500 .comp_mask = 0,
501 });
502 if (!fdir_queue->ind_table) {
503 ERROR("cannot create flow director indirection table");
504 goto error;
505 }
506 fdir_queue->qp = ibv_exp_create_qp(
507 priv->ctx,
508 &(struct ibv_exp_qp_init_attr){
509 .qp_type = IBV_QPT_RAW_PACKET,
510 .comp_mask =
511 IBV_EXP_QP_INIT_ATTR_PD |
512 IBV_EXP_QP_INIT_ATTR_PORT |
513 IBV_EXP_QP_INIT_ATTR_RX_HASH,
514 .pd = priv->pd,
515 .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
516 .rx_hash_function =
517 IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
518 .rx_hash_key_len = rss_hash_default_key_len,
519 .rx_hash_key = rss_hash_default_key,
520 .rx_hash_fields_mask = 0,
521 .rwq_ind_tbl = fdir_queue->ind_table,
522 },
523 .port_num = priv->port,
524 });
525 if (!fdir_queue->qp) {
526 ERROR("cannot create flow director hash RX QP");
527 goto error;
528 }
529 return fdir_queue;
530error:
531 assert(fdir_queue);
532 assert(!fdir_queue->qp);
533 if (fdir_queue->ind_table)
534 claim_zero(ibv_exp_destroy_rwq_ind_table
535 (fdir_queue->ind_table));
536 if (fdir_queue->wq)
537 claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
538 if (fdir_queue->cq)
539 claim_zero(ibv_destroy_cq(fdir_queue->cq));
540 rte_free(fdir_queue);
541 return NULL;
542}
543
544/**
545 * Get flow director queue for a specific RX queue, create it in case
546 * it does not exist.
547 *
548 * @param priv
549 * Private structure.
550 * @param idx
551 * RX queue index.
552 *
553 * @return
554 * Related flow director queue on success, NULL otherwise.
555 */
556static struct fdir_queue *
557priv_get_fdir_queue(struct priv *priv, uint16_t idx)
558{
559 struct rxq_ctrl *rxq_ctrl =
560 container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
561 struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue;
562
563 assert(rxq_ctrl->wq);
564 if (fdir_queue == NULL) {
565 fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq,
566 rxq_ctrl->socket);
567 rxq_ctrl->fdir_queue = fdir_queue;
568 }
569 return fdir_queue;
570}
571
572/**
573 * Get or flow director drop queue. Create it if it does not exist.
574 *
575 * @param priv
576 * Private structure.
577 *
578 * @return
579 * Flow director drop queue on success, NULL otherwise.
580 */
581static struct fdir_queue *
582priv_get_fdir_drop_queue(struct priv *priv)
583{
584 struct fdir_queue *fdir_queue = priv->fdir_drop_queue;
585
586 if (fdir_queue == NULL) {
587 unsigned int socket = SOCKET_ID_ANY;
588
589 /* Select a known NUMA socket if possible. */
590 if (priv->rxqs_n && (*priv->rxqs)[0])
591 socket = container_of((*priv->rxqs)[0],
592 struct rxq_ctrl, rxq)->socket;
593 fdir_queue = priv_fdir_queue_create(priv, NULL, socket);
594 priv->fdir_drop_queue = fdir_queue;
595 }
596 return fdir_queue;
597}
598
599/**
600 * Enable flow director filter and create steering rules.
601 *
602 * @param priv
603 * Private structure.
604 * @param mlx5_fdir_filter
605 * Filter to create steering rule for.
606 *
607 * @return
608 * 0 on success, errno value on failure.
609 */
610static int
611priv_fdir_filter_enable(struct priv *priv,
612 struct mlx5_fdir_filter *mlx5_fdir_filter)
613{
614 struct fdir_queue *fdir_queue;
615
616 /* Check if flow already exists. */
617 if (mlx5_fdir_filter->flow != NULL)
618 return 0;
619
620 /* Get fdir_queue for specific queue. */
621 if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT)
622 fdir_queue = priv_get_fdir_drop_queue(priv);
623 else
624 fdir_queue = priv_get_fdir_queue(priv,
625 mlx5_fdir_filter->queue);
626
627 if (fdir_queue == NULL) {
628 ERROR("failed to create flow director rxq for queue %d",
629 mlx5_fdir_filter->queue);
630 return EINVAL;
631 }
632
633 /* Create flow */
634 return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
635}
636
637/**
638 * Initialize flow director filters list.
639 *
640 * @param priv
641 * Private structure.
642 *
643 * @return
644 * 0 on success, errno value on failure.
645 */
646int
647fdir_init_filters_list(struct priv *priv)
648{
649 /* Filter list initialization should be done only once. */
650 if (priv->fdir_filter_list)
651 return 0;
652
653 /* Create filters list. */
654 priv->fdir_filter_list =
655 rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
656
657 if (priv->fdir_filter_list == NULL) {
658 int err = ENOMEM;
659
660 ERROR("cannot allocate flow director filter list: %s",
661 strerror(err));
662 return err;
663 }
664
665 LIST_INIT(priv->fdir_filter_list);
666
667 return 0;
668}
669
670/**
671 * Flush all filters.
672 *
673 * @param priv
674 * Private structure.
675 */
676static void
677priv_fdir_filter_flush(struct priv *priv)
678{
679 struct mlx5_fdir_filter *mlx5_fdir_filter;
680
681 while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
682 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
683
684 DEBUG("%p: flushing flow director filter %p",
685 (void *)priv, (void *)mlx5_fdir_filter);
686 LIST_REMOVE(mlx5_fdir_filter, next);
687 if (flow != NULL)
688 claim_zero(ibv_exp_destroy_flow(flow));
689 rte_free(mlx5_fdir_filter);
690 }
691}
692
693/**
694 * Remove all flow director filters and delete list.
695 *
696 * @param priv
697 * Private structure.
698 */
699void
700priv_fdir_delete_filters_list(struct priv *priv)
701{
702 priv_fdir_filter_flush(priv);
703 rte_free(priv->fdir_filter_list);
704 priv->fdir_filter_list = NULL;
705}
706
707/**
708 * Disable flow director, remove all steering rules.
709 *
710 * @param priv
711 * Private structure.
712 */
713void
714priv_fdir_disable(struct priv *priv)
715{
716 unsigned int i;
717 struct mlx5_fdir_filter *mlx5_fdir_filter;
718
719 /* Run on every flow director filter and destroy flow handle. */
720 LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
721 struct ibv_exp_flow *flow;
722
723 /* Only valid elements should be in the list */
724 assert(mlx5_fdir_filter != NULL);
725 flow = mlx5_fdir_filter->flow;
726
727 /* Destroy flow handle */
728 if (flow != NULL) {
729 claim_zero(ibv_exp_destroy_flow(flow));
730 mlx5_fdir_filter->flow = NULL;
731 }
732 }
733
734 /* Destroy flow director context in each RX queue. */
735 for (i = 0; (i != priv->rxqs_n); i++) {
736 struct rxq_ctrl *rxq_ctrl =
737 container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
738
739 if (!rxq_ctrl->fdir_queue)
740 continue;
741 priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue);
742 rxq_ctrl->fdir_queue = NULL;
743 }
744 if (priv->fdir_drop_queue) {
745 priv_fdir_queue_destroy(priv, priv->fdir_drop_queue);
746 priv->fdir_drop_queue = NULL;
747 }
748}
749
750/**
751 * Enable flow director, create steering rules.
752 *
753 * @param priv
754 * Private structure.
755 */
756void
757priv_fdir_enable(struct priv *priv)
758{
759 struct mlx5_fdir_filter *mlx5_fdir_filter;
760
761 /* Run on every fdir filter and create flow handle */
762 LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
763 /* Only valid elements should be in the list */
764 assert(mlx5_fdir_filter != NULL);
765
766 priv_fdir_filter_enable(priv, mlx5_fdir_filter);
767 }
768}
769
770/**
771 * Find specific filter in list.
772 *
773 * @param priv
774 * Private structure.
775 * @param fdir_filter
776 * Flow director filter to find.
777 *
778 * @return
779 * Filter element if found, otherwise NULL.
780 */
781static struct mlx5_fdir_filter *
782priv_find_filter_in_list(struct priv *priv,
783 const struct rte_eth_fdir_filter *fdir_filter)
784{
785 struct fdir_flow_desc desc;
786 struct mlx5_fdir_filter *mlx5_fdir_filter;
787 enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
788
789 /* Get flow director filter to look for. */
790 fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
791
792 /* Look for the requested element. */
793 LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
794 /* Only valid elements should be in the list. */
795 assert(mlx5_fdir_filter != NULL);
796
797 /* Return matching filter. */
798 if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
799 return mlx5_fdir_filter;
800 }
801
802 /* Filter not found */
803 return NULL;
804}
805
806/**
807 * Add new flow director filter and store it in list.
808 *
809 * @param priv
810 * Private structure.
811 * @param fdir_filter
812 * Flow director filter to add.
813 *
814 * @return
815 * 0 on success, errno value on failure.
816 */
817static int
818priv_fdir_filter_add(struct priv *priv,
819 const struct rte_eth_fdir_filter *fdir_filter)
820{
821 struct mlx5_fdir_filter *mlx5_fdir_filter;
822 enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
823 int err = 0;
824
825 /* Validate queue number. */
826 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
827 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
828 return EINVAL;
829 }
830
831 /* Duplicate filters are currently unsupported. */
832 mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
833 if (mlx5_fdir_filter != NULL) {
834 ERROR("filter already exists");
835 return EINVAL;
836 }
837
838 /* Create new flow director filter. */
839 mlx5_fdir_filter =
840 rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
841 if (mlx5_fdir_filter == NULL) {
842 err = ENOMEM;
843 ERROR("cannot allocate flow director filter: %s",
844 strerror(err));
845 return err;
846 }
847
848 /* Set action parameters. */
849 mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
850 mlx5_fdir_filter->behavior = fdir_filter->action.behavior;
851
852 /* Convert to mlx5 filter descriptor. */
853 fdir_filter_to_flow_desc(fdir_filter,
854 &mlx5_fdir_filter->desc, fdir_mode);
855
856 /* Insert new filter into list. */
857 LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
858
859 DEBUG("%p: flow director filter %p added",
860 (void *)priv, (void *)mlx5_fdir_filter);
861
862 /* Enable filter immediately if device is started. */
863 if (priv->started)
864 err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
865
866 return err;
867}
868
869/**
870 * Update queue for specific filter.
871 *
872 * @param priv
873 * Private structure.
874 * @param fdir_filter
875 * Filter to be updated.
876 *
877 * @return
878 * 0 on success, errno value on failure.
879 */
880static int
881priv_fdir_filter_update(struct priv *priv,
882 const struct rte_eth_fdir_filter *fdir_filter)
883{
884 struct mlx5_fdir_filter *mlx5_fdir_filter;
885
886 /* Validate queue number. */
887 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
888 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
889 return EINVAL;
890 }
891
892 mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
893 if (mlx5_fdir_filter != NULL) {
894 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
895 int err = 0;
896
897 /* Update queue number. */
898 mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
899
900 /* Destroy flow handle. */
901 if (flow != NULL) {
902 claim_zero(ibv_exp_destroy_flow(flow));
903 mlx5_fdir_filter->flow = NULL;
904 }
905 DEBUG("%p: flow director filter %p updated",
906 (void *)priv, (void *)mlx5_fdir_filter);
907
908 /* Enable filter if device is started. */
909 if (priv->started)
910 err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
911
912 return err;
913 }
914
915 /* Filter not found, create it. */
916 DEBUG("%p: filter not found for update, creating new filter",
917 (void *)priv);
918 return priv_fdir_filter_add(priv, fdir_filter);
919}
920
921/**
922 * Delete specific filter.
923 *
924 * @param priv
925 * Private structure.
926 * @param fdir_filter
927 * Filter to be deleted.
928 *
929 * @return
930 * 0 on success, errno value on failure.
931 */
932static int
933priv_fdir_filter_delete(struct priv *priv,
934 const struct rte_eth_fdir_filter *fdir_filter)
935{
936 struct mlx5_fdir_filter *mlx5_fdir_filter;
937
938 mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
939 if (mlx5_fdir_filter != NULL) {
940 struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
941
942 /* Remove element from list. */
943 LIST_REMOVE(mlx5_fdir_filter, next);
944
945 /* Destroy flow handle. */
946 if (flow != NULL) {
947 claim_zero(ibv_exp_destroy_flow(flow));
948 mlx5_fdir_filter->flow = NULL;
949 }
950
951 DEBUG("%p: flow director filter %p deleted",
952 (void *)priv, (void *)mlx5_fdir_filter);
953
954 /* Delete filter. */
955 rte_free(mlx5_fdir_filter);
956
957 return 0;
958 }
959
960 ERROR("%p: flow director delete failed, cannot find filter",
961 (void *)priv);
962 return EINVAL;
963}
964
965/**
966 * Get flow director information.
967 *
968 * @param priv
969 * Private structure.
970 * @param[out] fdir_info
971 * Resulting flow director information.
972 */
973static void
974priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
975{
976 struct rte_eth_fdir_masks *mask =
977 &priv->dev->data->dev_conf.fdir_conf.mask;
978
979 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
980 fdir_info->guarant_spc = 0;
981
982 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
983
984 fdir_info->max_flexpayload = 0;
985 fdir_info->flow_types_mask[0] = 0;
986
987 fdir_info->flex_payload_unit = 0;
988 fdir_info->max_flex_payload_segment_num = 0;
989 fdir_info->flex_payload_limit = 0;
990 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
991}
992
993/**
994 * Deal with flow director operations.
995 *
996 * @param priv
997 * Pointer to private structure.
998 * @param filter_op
999 * Operation to perform.
1000 * @param arg
1001 * Pointer to operation-specific structure.
1002 *
1003 * @return
1004 * 0 on success, errno value on failure.
1005 */
1006static int
1007priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
1008{
1009 enum rte_fdir_mode fdir_mode =
1010 priv->dev->data->dev_conf.fdir_conf.mode;
1011 int ret = 0;
1012
1013 if (filter_op == RTE_ETH_FILTER_NOP)
1014 return 0;
1015
1016 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
1017 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
1018 ERROR("%p: flow director mode %d not supported",
1019 (void *)priv, fdir_mode);
1020 return EINVAL;
1021 }
1022
1023 switch (filter_op) {
1024 case RTE_ETH_FILTER_ADD:
1025 ret = priv_fdir_filter_add(priv, arg);
1026 break;
1027 case RTE_ETH_FILTER_UPDATE:
1028 ret = priv_fdir_filter_update(priv, arg);
1029 break;
1030 case RTE_ETH_FILTER_DELETE:
1031 ret = priv_fdir_filter_delete(priv, arg);
1032 break;
1033 case RTE_ETH_FILTER_FLUSH:
1034 priv_fdir_filter_flush(priv);
1035 break;
1036 case RTE_ETH_FILTER_INFO:
1037 priv_fdir_info_get(priv, arg);
1038 break;
1039 default:
1040 DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
1041 ret = EINVAL;
1042 break;
1043 }
1044 return ret;
1045}
1046
11fdf7f2
TL		 1047static const struct rte_flow_ops mlx5_flow_ops = {
1048 .validate = mlx5_flow_validate,
1049 .create = mlx5_flow_create,
1050 .destroy = mlx5_flow_destroy,
1051 .flush = mlx5_flow_flush,
1052 .query = NULL,
1053};
1054
7c673cae
FG		 1055/**
1056 * Manage filter operations.
1057 *
1058 * @param dev
1059 * Pointer to Ethernet device structure.
1060 * @param filter_type
1061 * Filter type.
1062 * @param filter_op
1063 * Operation to perform.
1064 * @param arg
1065 * Pointer to operation-specific structure.
1066 *
1067 * @return
1068 * 0 on success, negative errno value on failure.
1069 */
1070int
1071mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
1072 enum rte_filter_type filter_type,
1073 enum rte_filter_op filter_op,
1074 void *arg)
1075{
1076 int ret = EINVAL;
1077 struct priv *priv = dev->data->dev_private;
1078
1079 switch (filter_type) {
11fdf7f2
TL		 1080 case RTE_ETH_FILTER_GENERIC:
1081 if (filter_op != RTE_ETH_FILTER_GET)
1082 return -EINVAL;
1083 *(const void **)arg = &mlx5_flow_ops;
1084 return 0;
7c673cae
FG		 1085 case RTE_ETH_FILTER_FDIR:
1086 priv_lock(priv);
1087 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
1088 priv_unlock(priv);
1089 break;
1090 default:
1091 ERROR("%p: filter type (%d) not supported",
1092 (void *)dev, filter_type);
1093 break;
1094 }
1095
1096 return -ret;
1097}
Ceph