]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright 2015 6WIND S.A. | |
5 | * Copyright 2015 Mellanox. | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * * Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * * Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * * Neither the name of 6WIND S.A. nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | ||
34 | #include <stddef.h> | |
35 | #include <assert.h> | |
36 | #include <stdint.h> | |
37 | #include <string.h> | |
38 | #include <errno.h> | |
39 | ||
40 | /* Verbs header. */ | |
41 | /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ | |
42 | #ifdef PEDANTIC | |
43 | #pragma GCC diagnostic ignored "-Wpedantic" | |
44 | #endif | |
45 | #include <infiniband/verbs.h> | |
46 | #ifdef PEDANTIC | |
47 | #pragma GCC diagnostic error "-Wpedantic" | |
48 | #endif | |
49 | ||
50 | /* DPDK headers don't like -pedantic. */ | |
51 | #ifdef PEDANTIC | |
52 | #pragma GCC diagnostic ignored "-Wpedantic" | |
53 | #endif | |
54 | #include <rte_ether.h> | |
55 | #include <rte_malloc.h> | |
56 | #include <rte_ethdev.h> | |
57 | #include <rte_common.h> | |
58 | #ifdef PEDANTIC | |
59 | #pragma GCC diagnostic error "-Wpedantic" | |
60 | #endif | |
61 | ||
62 | #include "mlx5.h" | |
63 | #include "mlx5_rxtx.h" | |
64 | ||
65 | struct fdir_flow_desc { | |
66 | uint16_t dst_port; | |
67 | uint16_t src_port; | |
68 | uint32_t src_ip[4]; | |
69 | uint32_t dst_ip[4]; | |
70 | uint8_t mac[6]; | |
71 | uint16_t vlan_tag; | |
72 | enum hash_rxq_type type; | |
73 | }; | |
74 | ||
75 | struct mlx5_fdir_filter { | |
76 | LIST_ENTRY(mlx5_fdir_filter) next; | |
77 | uint16_t queue; /* Queue assigned to if FDIR match. */ | |
78 | enum rte_eth_fdir_behavior behavior; | |
79 | struct fdir_flow_desc desc; | |
80 | struct ibv_exp_flow *flow; | |
81 | }; | |
82 | ||
83 | LIST_HEAD(fdir_filter_list, mlx5_fdir_filter); | |
84 | ||
85 | /** | |
86 | * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor. | |
87 | * | |
88 | * @param[in] fdir_filter | |
89 | * DPDK filter structure to convert. | |
90 | * @param[out] desc | |
91 | * Resulting mlx5 filter descriptor. | |
92 | * @param mode | |
93 | * Flow director mode. | |
94 | */ | |
95 | static void | |
96 | fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter, | |
97 | struct fdir_flow_desc *desc, enum rte_fdir_mode mode) | |
98 | { | |
99 | /* Initialize descriptor. */ | |
100 | memset(desc, 0, sizeof(*desc)); | |
101 | ||
102 | /* Set VLAN ID. */ | |
103 | desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci; | |
104 | ||
105 | /* Set MAC address. */ | |
106 | if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) { | |
107 | rte_memcpy(desc->mac, | |
108 | fdir_filter->input.flow.mac_vlan_flow.mac_addr. | |
109 | addr_bytes, | |
110 | sizeof(desc->mac)); | |
111 | desc->type = HASH_RXQ_ETH; | |
112 | return; | |
113 | } | |
114 | ||
115 | /* Set mode */ | |
116 | switch (fdir_filter->input.flow_type) { | |
117 | case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: | |
118 | desc->type = HASH_RXQ_UDPV4; | |
119 | break; | |
120 | case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: | |
121 | desc->type = HASH_RXQ_TCPV4; | |
122 | break; | |
123 | case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: | |
124 | desc->type = HASH_RXQ_IPV4; | |
125 | break; | |
126 | case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: | |
127 | desc->type = HASH_RXQ_UDPV6; | |
128 | break; | |
129 | case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: | |
130 | desc->type = HASH_RXQ_TCPV6; | |
131 | break; | |
132 | case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: | |
133 | desc->type = HASH_RXQ_IPV6; | |
134 | break; | |
135 | default: | |
136 | break; | |
137 | } | |
138 | ||
139 | /* Set flow values */ | |
140 | switch (fdir_filter->input.flow_type) { | |
141 | case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: | |
142 | case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: | |
143 | desc->src_port = fdir_filter->input.flow.udp4_flow.src_port; | |
144 | desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port; | |
145 | case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: | |
146 | desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip; | |
147 | desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip; | |
148 | break; | |
149 | case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: | |
150 | case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: | |
151 | desc->src_port = fdir_filter->input.flow.udp6_flow.src_port; | |
152 | desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port; | |
153 | /* Fall through. */ | |
154 | case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: | |
155 | rte_memcpy(desc->src_ip, | |
156 | fdir_filter->input.flow.ipv6_flow.src_ip, | |
157 | sizeof(desc->src_ip)); | |
158 | rte_memcpy(desc->dst_ip, | |
159 | fdir_filter->input.flow.ipv6_flow.dst_ip, | |
160 | sizeof(desc->dst_ip)); | |
161 | break; | |
162 | default: | |
163 | break; | |
164 | } | |
165 | } | |
166 | ||
167 | /** | |
168 | * Check if two flow descriptors overlap according to configured mask. | |
169 | * | |
170 | * @param priv | |
171 | * Private structure that provides flow director mask. | |
172 | * @param desc1 | |
173 | * First flow descriptor to compare. | |
174 | * @param desc2 | |
175 | * Second flow descriptor to compare. | |
176 | * | |
177 | * @return | |
178 | * Nonzero if descriptors overlap. | |
179 | */ | |
180 | static int | |
181 | priv_fdir_overlap(const struct priv *priv, | |
182 | const struct fdir_flow_desc *desc1, | |
183 | const struct fdir_flow_desc *desc2) | |
184 | { | |
185 | const struct rte_eth_fdir_masks *mask = | |
186 | &priv->dev->data->dev_conf.fdir_conf.mask; | |
187 | unsigned int i; | |
188 | ||
189 | if (desc1->type != desc2->type) | |
190 | return 0; | |
191 | /* Ignore non masked bits. */ | |
192 | for (i = 0; i != RTE_DIM(desc1->mac); ++i) | |
193 | if ((desc1->mac[i] & mask->mac_addr_byte_mask) != | |
194 | (desc2->mac[i] & mask->mac_addr_byte_mask)) | |
195 | return 0; | |
196 | if (((desc1->src_port & mask->src_port_mask) != | |
197 | (desc2->src_port & mask->src_port_mask)) || | |
198 | ((desc1->dst_port & mask->dst_port_mask) != | |
199 | (desc2->dst_port & mask->dst_port_mask))) | |
200 | return 0; | |
201 | switch (desc1->type) { | |
202 | case HASH_RXQ_IPV4: | |
203 | case HASH_RXQ_UDPV4: | |
204 | case HASH_RXQ_TCPV4: | |
205 | if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) != | |
206 | (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) || | |
207 | ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) != | |
208 | (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip))) | |
209 | return 0; | |
210 | break; | |
211 | case HASH_RXQ_IPV6: | |
212 | case HASH_RXQ_UDPV6: | |
213 | case HASH_RXQ_TCPV6: | |
214 | for (i = 0; i != RTE_DIM(desc1->src_ip); ++i) | |
215 | if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) != | |
216 | (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) || | |
217 | ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) != | |
218 | (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i]))) | |
219 | return 0; | |
220 | break; | |
221 | default: | |
222 | break; | |
223 | } | |
224 | return 1; | |
225 | } | |
226 | ||
227 | /** | |
228 | * Create flow director steering rule for a specific filter. | |
229 | * | |
230 | * @param priv | |
231 | * Private structure. | |
232 | * @param mlx5_fdir_filter | |
233 | * Filter to create a steering rule for. | |
234 | * @param fdir_queue | |
235 | * Flow director queue for matching packets. | |
236 | * | |
237 | * @return | |
238 | * 0 on success, errno value on failure. | |
239 | */ | |
240 | static int | |
241 | priv_fdir_flow_add(struct priv *priv, | |
242 | struct mlx5_fdir_filter *mlx5_fdir_filter, | |
243 | struct fdir_queue *fdir_queue) | |
244 | { | |
245 | struct ibv_exp_flow *flow; | |
246 | struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc; | |
247 | enum rte_fdir_mode fdir_mode = | |
248 | priv->dev->data->dev_conf.fdir_conf.mode; | |
249 | struct rte_eth_fdir_masks *mask = | |
250 | &priv->dev->data->dev_conf.fdir_conf.mask; | |
251 | FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type)); | |
252 | struct ibv_exp_flow_attr *attr = &data->attr; | |
253 | uintptr_t spec_offset = (uintptr_t)&data->spec; | |
254 | struct ibv_exp_flow_spec_eth *spec_eth; | |
255 | struct ibv_exp_flow_spec_ipv4 *spec_ipv4; | |
256 | struct ibv_exp_flow_spec_ipv6 *spec_ipv6; | |
257 | struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp; | |
258 | struct mlx5_fdir_filter *iter_fdir_filter; | |
259 | unsigned int i; | |
260 | ||
261 | /* Abort if an existing flow overlaps this one to avoid packet | |
262 | * duplication, even if it targets another queue. */ | |
263 | LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next) | |
264 | if ((iter_fdir_filter != mlx5_fdir_filter) && | |
265 | (iter_fdir_filter->flow != NULL) && | |
266 | (priv_fdir_overlap(priv, | |
267 | &mlx5_fdir_filter->desc, | |
268 | &iter_fdir_filter->desc))) | |
269 | return EEXIST; | |
270 | ||
271 | /* | |
272 | * No padding must be inserted by the compiler between attr and spec. | |
273 | * This layout is expected by libibverbs. | |
274 | */ | |
275 | assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset); | |
276 | priv_flow_attr(priv, attr, sizeof(data), desc->type); | |
277 | ||
278 | /* Set Ethernet spec */ | |
279 | spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset; | |
280 | ||
281 | /* The first specification must be Ethernet. */ | |
282 | assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH); | |
283 | assert(spec_eth->size == sizeof(*spec_eth)); | |
284 | ||
285 | /* VLAN ID */ | |
286 | spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask; | |
287 | spec_eth->mask.vlan_tag = mask->vlan_tci_mask; | |
288 | ||
289 | /* Update priority */ | |
290 | attr->priority = 2; | |
291 | ||
292 | if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) { | |
293 | /* MAC Address */ | |
294 | for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) { | |
295 | spec_eth->val.dst_mac[i] = | |
296 | desc->mac[i] & mask->mac_addr_byte_mask; | |
297 | spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask; | |
298 | } | |
299 | goto create_flow; | |
300 | } | |
301 | ||
302 | switch (desc->type) { | |
303 | case HASH_RXQ_IPV4: | |
304 | case HASH_RXQ_UDPV4: | |
305 | case HASH_RXQ_TCPV4: | |
306 | spec_offset += spec_eth->size; | |
307 | ||
308 | /* Set IP spec */ | |
309 | spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset; | |
310 | ||
311 | /* The second specification must be IP. */ | |
312 | assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4); | |
313 | assert(spec_ipv4->size == sizeof(*spec_ipv4)); | |
314 | ||
315 | spec_ipv4->val.src_ip = | |
316 | desc->src_ip[0] & mask->ipv4_mask.src_ip; | |
317 | spec_ipv4->val.dst_ip = | |
318 | desc->dst_ip[0] & mask->ipv4_mask.dst_ip; | |
319 | spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip; | |
320 | spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip; | |
321 | ||
322 | /* Update priority */ | |
323 | attr->priority = 1; | |
324 | ||
325 | if (desc->type == HASH_RXQ_IPV4) | |
326 | goto create_flow; | |
327 | ||
328 | spec_offset += spec_ipv4->size; | |
329 | break; | |
330 | case HASH_RXQ_IPV6: | |
331 | case HASH_RXQ_UDPV6: | |
332 | case HASH_RXQ_TCPV6: | |
333 | spec_offset += spec_eth->size; | |
334 | ||
335 | /* Set IP spec */ | |
336 | spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset; | |
337 | ||
338 | /* The second specification must be IP. */ | |
339 | assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6); | |
340 | assert(spec_ipv6->size == sizeof(*spec_ipv6)); | |
341 | ||
342 | for (i = 0; i != RTE_DIM(desc->src_ip); ++i) { | |
343 | ((uint32_t *)spec_ipv6->val.src_ip)[i] = | |
344 | desc->src_ip[i] & mask->ipv6_mask.src_ip[i]; | |
345 | ((uint32_t *)spec_ipv6->val.dst_ip)[i] = | |
346 | desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i]; | |
347 | } | |
348 | rte_memcpy(spec_ipv6->mask.src_ip, | |
349 | mask->ipv6_mask.src_ip, | |
350 | sizeof(spec_ipv6->mask.src_ip)); | |
351 | rte_memcpy(spec_ipv6->mask.dst_ip, | |
352 | mask->ipv6_mask.dst_ip, | |
353 | sizeof(spec_ipv6->mask.dst_ip)); | |
354 | ||
355 | /* Update priority */ | |
356 | attr->priority = 1; | |
357 | ||
358 | if (desc->type == HASH_RXQ_IPV6) | |
359 | goto create_flow; | |
360 | ||
361 | spec_offset += spec_ipv6->size; | |
362 | break; | |
363 | default: | |
364 | ERROR("invalid flow attribute type"); | |
365 | return EINVAL; | |
366 | } | |
367 | ||
368 | /* Set TCP/UDP flow specification. */ | |
369 | spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset; | |
370 | ||
371 | /* The third specification must be TCP/UDP. */ | |
372 | assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP || | |
373 | spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP); | |
374 | assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp)); | |
375 | ||
376 | spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask; | |
377 | spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask; | |
378 | spec_tcp_udp->mask.src_port = mask->src_port_mask; | |
379 | spec_tcp_udp->mask.dst_port = mask->dst_port_mask; | |
380 | ||
381 | /* Update priority */ | |
382 | attr->priority = 0; | |
383 | ||
384 | create_flow: | |
385 | ||
386 | errno = 0; | |
387 | flow = ibv_exp_create_flow(fdir_queue->qp, attr); | |
388 | if (flow == NULL) { | |
389 | /* It's not clear whether errno is always set in this case. */ | |
390 | ERROR("%p: flow director configuration failed, errno=%d: %s", | |
391 | (void *)priv, errno, | |
392 | (errno ? strerror(errno) : "Unknown error")); | |
393 | if (errno) | |
394 | return errno; | |
395 | return EINVAL; | |
396 | } | |
397 | ||
398 | DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow); | |
399 | mlx5_fdir_filter->flow = flow; | |
400 | return 0; | |
401 | } | |
402 | ||
403 | /** | |
404 | * Destroy a flow director queue. | |
405 | * | |
406 | * @param fdir_queue | |
407 | * Flow director queue to be destroyed. | |
408 | */ | |
409 | void | |
410 | priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue) | |
411 | { | |
412 | struct mlx5_fdir_filter *fdir_filter; | |
413 | ||
414 | /* Disable filter flows still applying to this queue. */ | |
415 | LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) { | |
416 | unsigned int idx = fdir_filter->queue; | |
417 | struct rxq_ctrl *rxq_ctrl = | |
418 | container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq); | |
419 | ||
420 | assert(idx < priv->rxqs_n); | |
421 | if (fdir_queue == rxq_ctrl->fdir_queue && | |
422 | fdir_filter->flow != NULL) { | |
423 | claim_zero(ibv_exp_destroy_flow(fdir_filter->flow)); | |
424 | fdir_filter->flow = NULL; | |
425 | } | |
426 | } | |
427 | assert(fdir_queue->qp); | |
428 | claim_zero(ibv_destroy_qp(fdir_queue->qp)); | |
429 | assert(fdir_queue->ind_table); | |
430 | claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table)); | |
431 | if (fdir_queue->wq) | |
432 | claim_zero(ibv_exp_destroy_wq(fdir_queue->wq)); | |
433 | if (fdir_queue->cq) | |
434 | claim_zero(ibv_destroy_cq(fdir_queue->cq)); | |
435 | #ifndef NDEBUG | |
436 | memset(fdir_queue, 0x2a, sizeof(*fdir_queue)); | |
437 | #endif | |
438 | rte_free(fdir_queue); | |
439 | } | |
440 | ||
441 | /** | |
442 | * Create a flow director queue. | |
443 | * | |
444 | * @param priv | |
445 | * Private structure. | |
446 | * @param wq | |
447 | * Work queue to route matched packets to, NULL if one needs to | |
448 | * be created. | |
449 | * | |
450 | * @return | |
451 | * Related flow director queue on success, NULL otherwise. | |
452 | */ | |
453 | static struct fdir_queue * | |
454 | priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq, | |
455 | unsigned int socket) | |
456 | { | |
457 | struct fdir_queue *fdir_queue; | |
458 | ||
459 | fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue), | |
460 | 0, socket); | |
461 | if (!fdir_queue) { | |
462 | ERROR("cannot allocate flow director queue"); | |
463 | return NULL; | |
464 | } | |
465 | assert(priv->pd); | |
466 | assert(priv->ctx); | |
467 | if (!wq) { | |
468 | fdir_queue->cq = ibv_exp_create_cq( | |
469 | priv->ctx, 1, NULL, NULL, 0, | |
470 | &(struct ibv_exp_cq_init_attr){ | |
471 | .comp_mask = 0, | |
472 | }); | |
473 | if (!fdir_queue->cq) { | |
474 | ERROR("cannot create flow director CQ"); | |
475 | goto error; | |
476 | } | |
477 | fdir_queue->wq = ibv_exp_create_wq( | |
478 | priv->ctx, | |
479 | &(struct ibv_exp_wq_init_attr){ | |
480 | .wq_type = IBV_EXP_WQT_RQ, | |
481 | .max_recv_wr = 1, | |
482 | .max_recv_sge = 1, | |
483 | .pd = priv->pd, | |
484 | .cq = fdir_queue->cq, | |
485 | }); | |
486 | if (!fdir_queue->wq) { | |
487 | ERROR("cannot create flow director WQ"); | |
488 | goto error; | |
489 | } | |
490 | wq = fdir_queue->wq; | |
491 | } | |
492 | fdir_queue->ind_table = ibv_exp_create_rwq_ind_table( | |
493 | priv->ctx, | |
494 | &(struct ibv_exp_rwq_ind_table_init_attr){ | |
495 | .pd = priv->pd, | |
496 | .log_ind_tbl_size = 0, | |
497 | .ind_tbl = &wq, | |
498 | .comp_mask = 0, | |
499 | }); | |
500 | if (!fdir_queue->ind_table) { | |
501 | ERROR("cannot create flow director indirection table"); | |
502 | goto error; | |
503 | } | |
504 | fdir_queue->qp = ibv_exp_create_qp( | |
505 | priv->ctx, | |
506 | &(struct ibv_exp_qp_init_attr){ | |
507 | .qp_type = IBV_QPT_RAW_PACKET, | |
508 | .comp_mask = | |
509 | IBV_EXP_QP_INIT_ATTR_PD | | |
510 | IBV_EXP_QP_INIT_ATTR_PORT | | |
511 | IBV_EXP_QP_INIT_ATTR_RX_HASH, | |
512 | .pd = priv->pd, | |
513 | .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){ | |
514 | .rx_hash_function = | |
515 | IBV_EXP_RX_HASH_FUNC_TOEPLITZ, | |
516 | .rx_hash_key_len = rss_hash_default_key_len, | |
517 | .rx_hash_key = rss_hash_default_key, | |
518 | .rx_hash_fields_mask = 0, | |
519 | .rwq_ind_tbl = fdir_queue->ind_table, | |
520 | }, | |
521 | .port_num = priv->port, | |
522 | }); | |
523 | if (!fdir_queue->qp) { | |
524 | ERROR("cannot create flow director hash RX QP"); | |
525 | goto error; | |
526 | } | |
527 | return fdir_queue; | |
528 | error: | |
529 | assert(fdir_queue); | |
530 | assert(!fdir_queue->qp); | |
531 | if (fdir_queue->ind_table) | |
532 | claim_zero(ibv_exp_destroy_rwq_ind_table | |
533 | (fdir_queue->ind_table)); | |
534 | if (fdir_queue->wq) | |
535 | claim_zero(ibv_exp_destroy_wq(fdir_queue->wq)); | |
536 | if (fdir_queue->cq) | |
537 | claim_zero(ibv_destroy_cq(fdir_queue->cq)); | |
538 | rte_free(fdir_queue); | |
539 | return NULL; | |
540 | } | |
541 | ||
542 | /** | |
543 | * Get flow director queue for a specific RX queue, create it in case | |
544 | * it does not exist. | |
545 | * | |
546 | * @param priv | |
547 | * Private structure. | |
548 | * @param idx | |
549 | * RX queue index. | |
550 | * | |
551 | * @return | |
552 | * Related flow director queue on success, NULL otherwise. | |
553 | */ | |
554 | static struct fdir_queue * | |
555 | priv_get_fdir_queue(struct priv *priv, uint16_t idx) | |
556 | { | |
557 | struct rxq_ctrl *rxq_ctrl = | |
558 | container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq); | |
559 | struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue; | |
560 | ||
561 | assert(rxq_ctrl->wq); | |
562 | if (fdir_queue == NULL) { | |
563 | fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq, | |
564 | rxq_ctrl->socket); | |
565 | rxq_ctrl->fdir_queue = fdir_queue; | |
566 | } | |
567 | return fdir_queue; | |
568 | } | |
569 | ||
570 | /** | |
571 | * Get or flow director drop queue. Create it if it does not exist. | |
572 | * | |
573 | * @param priv | |
574 | * Private structure. | |
575 | * | |
576 | * @return | |
577 | * Flow director drop queue on success, NULL otherwise. | |
578 | */ | |
579 | static struct fdir_queue * | |
580 | priv_get_fdir_drop_queue(struct priv *priv) | |
581 | { | |
582 | struct fdir_queue *fdir_queue = priv->fdir_drop_queue; | |
583 | ||
584 | if (fdir_queue == NULL) { | |
585 | unsigned int socket = SOCKET_ID_ANY; | |
586 | ||
587 | /* Select a known NUMA socket if possible. */ | |
588 | if (priv->rxqs_n && (*priv->rxqs)[0]) | |
589 | socket = container_of((*priv->rxqs)[0], | |
590 | struct rxq_ctrl, rxq)->socket; | |
591 | fdir_queue = priv_fdir_queue_create(priv, NULL, socket); | |
592 | priv->fdir_drop_queue = fdir_queue; | |
593 | } | |
594 | return fdir_queue; | |
595 | } | |
596 | ||
597 | /** | |
598 | * Enable flow director filter and create steering rules. | |
599 | * | |
600 | * @param priv | |
601 | * Private structure. | |
602 | * @param mlx5_fdir_filter | |
603 | * Filter to create steering rule for. | |
604 | * | |
605 | * @return | |
606 | * 0 on success, errno value on failure. | |
607 | */ | |
608 | static int | |
609 | priv_fdir_filter_enable(struct priv *priv, | |
610 | struct mlx5_fdir_filter *mlx5_fdir_filter) | |
611 | { | |
612 | struct fdir_queue *fdir_queue; | |
613 | ||
614 | /* Check if flow already exists. */ | |
615 | if (mlx5_fdir_filter->flow != NULL) | |
616 | return 0; | |
617 | ||
618 | /* Get fdir_queue for specific queue. */ | |
619 | if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT) | |
620 | fdir_queue = priv_get_fdir_drop_queue(priv); | |
621 | else | |
622 | fdir_queue = priv_get_fdir_queue(priv, | |
623 | mlx5_fdir_filter->queue); | |
624 | ||
625 | if (fdir_queue == NULL) { | |
626 | ERROR("failed to create flow director rxq for queue %d", | |
627 | mlx5_fdir_filter->queue); | |
628 | return EINVAL; | |
629 | } | |
630 | ||
631 | /* Create flow */ | |
632 | return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue); | |
633 | } | |
634 | ||
635 | /** | |
636 | * Initialize flow director filters list. | |
637 | * | |
638 | * @param priv | |
639 | * Private structure. | |
640 | * | |
641 | * @return | |
642 | * 0 on success, errno value on failure. | |
643 | */ | |
644 | int | |
645 | fdir_init_filters_list(struct priv *priv) | |
646 | { | |
647 | /* Filter list initialization should be done only once. */ | |
648 | if (priv->fdir_filter_list) | |
649 | return 0; | |
650 | ||
651 | /* Create filters list. */ | |
652 | priv->fdir_filter_list = | |
653 | rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0); | |
654 | ||
655 | if (priv->fdir_filter_list == NULL) { | |
656 | int err = ENOMEM; | |
657 | ||
658 | ERROR("cannot allocate flow director filter list: %s", | |
659 | strerror(err)); | |
660 | return err; | |
661 | } | |
662 | ||
663 | LIST_INIT(priv->fdir_filter_list); | |
664 | ||
665 | return 0; | |
666 | } | |
667 | ||
668 | /** | |
669 | * Flush all filters. | |
670 | * | |
671 | * @param priv | |
672 | * Private structure. | |
673 | */ | |
674 | static void | |
675 | priv_fdir_filter_flush(struct priv *priv) | |
676 | { | |
677 | struct mlx5_fdir_filter *mlx5_fdir_filter; | |
678 | ||
679 | while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) { | |
680 | struct ibv_exp_flow *flow = mlx5_fdir_filter->flow; | |
681 | ||
682 | DEBUG("%p: flushing flow director filter %p", | |
683 | (void *)priv, (void *)mlx5_fdir_filter); | |
684 | LIST_REMOVE(mlx5_fdir_filter, next); | |
685 | if (flow != NULL) | |
686 | claim_zero(ibv_exp_destroy_flow(flow)); | |
687 | rte_free(mlx5_fdir_filter); | |
688 | } | |
689 | } | |
690 | ||
691 | /** | |
692 | * Remove all flow director filters and delete list. | |
693 | * | |
694 | * @param priv | |
695 | * Private structure. | |
696 | */ | |
697 | void | |
698 | priv_fdir_delete_filters_list(struct priv *priv) | |
699 | { | |
700 | priv_fdir_filter_flush(priv); | |
701 | rte_free(priv->fdir_filter_list); | |
702 | priv->fdir_filter_list = NULL; | |
703 | } | |
704 | ||
705 | /** | |
706 | * Disable flow director, remove all steering rules. | |
707 | * | |
708 | * @param priv | |
709 | * Private structure. | |
710 | */ | |
711 | void | |
712 | priv_fdir_disable(struct priv *priv) | |
713 | { | |
714 | unsigned int i; | |
715 | struct mlx5_fdir_filter *mlx5_fdir_filter; | |
716 | ||
717 | /* Run on every flow director filter and destroy flow handle. */ | |
718 | LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) { | |
719 | struct ibv_exp_flow *flow; | |
720 | ||
721 | /* Only valid elements should be in the list */ | |
722 | assert(mlx5_fdir_filter != NULL); | |
723 | flow = mlx5_fdir_filter->flow; | |
724 | ||
725 | /* Destroy flow handle */ | |
726 | if (flow != NULL) { | |
727 | claim_zero(ibv_exp_destroy_flow(flow)); | |
728 | mlx5_fdir_filter->flow = NULL; | |
729 | } | |
730 | } | |
731 | ||
732 | /* Destroy flow director context in each RX queue. */ | |
733 | for (i = 0; (i != priv->rxqs_n); i++) { | |
734 | struct rxq_ctrl *rxq_ctrl = | |
735 | container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq); | |
736 | ||
737 | if (!rxq_ctrl->fdir_queue) | |
738 | continue; | |
739 | priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue); | |
740 | rxq_ctrl->fdir_queue = NULL; | |
741 | } | |
742 | if (priv->fdir_drop_queue) { | |
743 | priv_fdir_queue_destroy(priv, priv->fdir_drop_queue); | |
744 | priv->fdir_drop_queue = NULL; | |
745 | } | |
746 | } | |
747 | ||
748 | /** | |
749 | * Enable flow director, create steering rules. | |
750 | * | |
751 | * @param priv | |
752 | * Private structure. | |
753 | */ | |
754 | void | |
755 | priv_fdir_enable(struct priv *priv) | |
756 | { | |
757 | struct mlx5_fdir_filter *mlx5_fdir_filter; | |
758 | ||
759 | /* Run on every fdir filter and create flow handle */ | |
760 | LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) { | |
761 | /* Only valid elements should be in the list */ | |
762 | assert(mlx5_fdir_filter != NULL); | |
763 | ||
764 | priv_fdir_filter_enable(priv, mlx5_fdir_filter); | |
765 | } | |
766 | } | |
767 | ||
768 | /** | |
769 | * Find specific filter in list. | |
770 | * | |
771 | * @param priv | |
772 | * Private structure. | |
773 | * @param fdir_filter | |
774 | * Flow director filter to find. | |
775 | * | |
776 | * @return | |
777 | * Filter element if found, otherwise NULL. | |
778 | */ | |
779 | static struct mlx5_fdir_filter * | |
780 | priv_find_filter_in_list(struct priv *priv, | |
781 | const struct rte_eth_fdir_filter *fdir_filter) | |
782 | { | |
783 | struct fdir_flow_desc desc; | |
784 | struct mlx5_fdir_filter *mlx5_fdir_filter; | |
785 | enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode; | |
786 | ||
787 | /* Get flow director filter to look for. */ | |
788 | fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode); | |
789 | ||
790 | /* Look for the requested element. */ | |
791 | LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) { | |
792 | /* Only valid elements should be in the list. */ | |
793 | assert(mlx5_fdir_filter != NULL); | |
794 | ||
795 | /* Return matching filter. */ | |
796 | if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc))) | |
797 | return mlx5_fdir_filter; | |
798 | } | |
799 | ||
800 | /* Filter not found */ | |
801 | return NULL; | |
802 | } | |
803 | ||
804 | /** | |
805 | * Add new flow director filter and store it in list. | |
806 | * | |
807 | * @param priv | |
808 | * Private structure. | |
809 | * @param fdir_filter | |
810 | * Flow director filter to add. | |
811 | * | |
812 | * @return | |
813 | * 0 on success, errno value on failure. | |
814 | */ | |
815 | static int | |
816 | priv_fdir_filter_add(struct priv *priv, | |
817 | const struct rte_eth_fdir_filter *fdir_filter) | |
818 | { | |
819 | struct mlx5_fdir_filter *mlx5_fdir_filter; | |
820 | enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode; | |
821 | int err = 0; | |
822 | ||
823 | /* Validate queue number. */ | |
824 | if (fdir_filter->action.rx_queue >= priv->rxqs_n) { | |
825 | ERROR("invalid queue number %d", fdir_filter->action.rx_queue); | |
826 | return EINVAL; | |
827 | } | |
828 | ||
829 | /* Duplicate filters are currently unsupported. */ | |
830 | mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter); | |
831 | if (mlx5_fdir_filter != NULL) { | |
832 | ERROR("filter already exists"); | |
833 | return EINVAL; | |
834 | } | |
835 | ||
836 | /* Create new flow director filter. */ | |
837 | mlx5_fdir_filter = | |
838 | rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0); | |
839 | if (mlx5_fdir_filter == NULL) { | |
840 | err = ENOMEM; | |
841 | ERROR("cannot allocate flow director filter: %s", | |
842 | strerror(err)); | |
843 | return err; | |
844 | } | |
845 | ||
846 | /* Set action parameters. */ | |
847 | mlx5_fdir_filter->queue = fdir_filter->action.rx_queue; | |
848 | mlx5_fdir_filter->behavior = fdir_filter->action.behavior; | |
849 | ||
850 | /* Convert to mlx5 filter descriptor. */ | |
851 | fdir_filter_to_flow_desc(fdir_filter, | |
852 | &mlx5_fdir_filter->desc, fdir_mode); | |
853 | ||
854 | /* Insert new filter into list. */ | |
855 | LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next); | |
856 | ||
857 | DEBUG("%p: flow director filter %p added", | |
858 | (void *)priv, (void *)mlx5_fdir_filter); | |
859 | ||
860 | /* Enable filter immediately if device is started. */ | |
861 | if (priv->started) | |
862 | err = priv_fdir_filter_enable(priv, mlx5_fdir_filter); | |
863 | ||
864 | return err; | |
865 | } | |
866 | ||
867 | /** | |
868 | * Update queue for specific filter. | |
869 | * | |
870 | * @param priv | |
871 | * Private structure. | |
872 | * @param fdir_filter | |
873 | * Filter to be updated. | |
874 | * | |
875 | * @return | |
876 | * 0 on success, errno value on failure. | |
877 | */ | |
878 | static int | |
879 | priv_fdir_filter_update(struct priv *priv, | |
880 | const struct rte_eth_fdir_filter *fdir_filter) | |
881 | { | |
882 | struct mlx5_fdir_filter *mlx5_fdir_filter; | |
883 | ||
884 | /* Validate queue number. */ | |
885 | if (fdir_filter->action.rx_queue >= priv->rxqs_n) { | |
886 | ERROR("invalid queue number %d", fdir_filter->action.rx_queue); | |
887 | return EINVAL; | |
888 | } | |
889 | ||
890 | mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter); | |
891 | if (mlx5_fdir_filter != NULL) { | |
892 | struct ibv_exp_flow *flow = mlx5_fdir_filter->flow; | |
893 | int err = 0; | |
894 | ||
895 | /* Update queue number. */ | |
896 | mlx5_fdir_filter->queue = fdir_filter->action.rx_queue; | |
897 | ||
898 | /* Destroy flow handle. */ | |
899 | if (flow != NULL) { | |
900 | claim_zero(ibv_exp_destroy_flow(flow)); | |
901 | mlx5_fdir_filter->flow = NULL; | |
902 | } | |
903 | DEBUG("%p: flow director filter %p updated", | |
904 | (void *)priv, (void *)mlx5_fdir_filter); | |
905 | ||
906 | /* Enable filter if device is started. */ | |
907 | if (priv->started) | |
908 | err = priv_fdir_filter_enable(priv, mlx5_fdir_filter); | |
909 | ||
910 | return err; | |
911 | } | |
912 | ||
913 | /* Filter not found, create it. */ | |
914 | DEBUG("%p: filter not found for update, creating new filter", | |
915 | (void *)priv); | |
916 | return priv_fdir_filter_add(priv, fdir_filter); | |
917 | } | |
918 | ||
919 | /** | |
920 | * Delete specific filter. | |
921 | * | |
922 | * @param priv | |
923 | * Private structure. | |
924 | * @param fdir_filter | |
925 | * Filter to be deleted. | |
926 | * | |
927 | * @return | |
928 | * 0 on success, errno value on failure. | |
929 | */ | |
930 | static int | |
931 | priv_fdir_filter_delete(struct priv *priv, | |
932 | const struct rte_eth_fdir_filter *fdir_filter) | |
933 | { | |
934 | struct mlx5_fdir_filter *mlx5_fdir_filter; | |
935 | ||
936 | mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter); | |
937 | if (mlx5_fdir_filter != NULL) { | |
938 | struct ibv_exp_flow *flow = mlx5_fdir_filter->flow; | |
939 | ||
940 | /* Remove element from list. */ | |
941 | LIST_REMOVE(mlx5_fdir_filter, next); | |
942 | ||
943 | /* Destroy flow handle. */ | |
944 | if (flow != NULL) { | |
945 | claim_zero(ibv_exp_destroy_flow(flow)); | |
946 | mlx5_fdir_filter->flow = NULL; | |
947 | } | |
948 | ||
949 | DEBUG("%p: flow director filter %p deleted", | |
950 | (void *)priv, (void *)mlx5_fdir_filter); | |
951 | ||
952 | /* Delete filter. */ | |
953 | rte_free(mlx5_fdir_filter); | |
954 | ||
955 | return 0; | |
956 | } | |
957 | ||
958 | ERROR("%p: flow director delete failed, cannot find filter", | |
959 | (void *)priv); | |
960 | return EINVAL; | |
961 | } | |
962 | ||
963 | /** | |
964 | * Get flow director information. | |
965 | * | |
966 | * @param priv | |
967 | * Private structure. | |
968 | * @param[out] fdir_info | |
969 | * Resulting flow director information. | |
970 | */ | |
971 | static void | |
972 | priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) | |
973 | { | |
974 | struct rte_eth_fdir_masks *mask = | |
975 | &priv->dev->data->dev_conf.fdir_conf.mask; | |
976 | ||
977 | fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; | |
978 | fdir_info->guarant_spc = 0; | |
979 | ||
980 | rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); | |
981 | ||
982 | fdir_info->max_flexpayload = 0; | |
983 | fdir_info->flow_types_mask[0] = 0; | |
984 | ||
985 | fdir_info->flex_payload_unit = 0; | |
986 | fdir_info->max_flex_payload_segment_num = 0; | |
987 | fdir_info->flex_payload_limit = 0; | |
988 | memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); | |
989 | } | |
990 | ||
991 | /** | |
992 | * Deal with flow director operations. | |
993 | * | |
994 | * @param priv | |
995 | * Pointer to private structure. | |
996 | * @param filter_op | |
997 | * Operation to perform. | |
998 | * @param arg | |
999 | * Pointer to operation-specific structure. | |
1000 | * | |
1001 | * @return | |
1002 | * 0 on success, errno value on failure. | |
1003 | */ | |
1004 | static int | |
1005 | priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) | |
1006 | { | |
1007 | enum rte_fdir_mode fdir_mode = | |
1008 | priv->dev->data->dev_conf.fdir_conf.mode; | |
1009 | int ret = 0; | |
1010 | ||
1011 | if (filter_op == RTE_ETH_FILTER_NOP) | |
1012 | return 0; | |
1013 | ||
1014 | if (fdir_mode != RTE_FDIR_MODE_PERFECT && | |
1015 | fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { | |
1016 | ERROR("%p: flow director mode %d not supported", | |
1017 | (void *)priv, fdir_mode); | |
1018 | return EINVAL; | |
1019 | } | |
1020 | ||
1021 | switch (filter_op) { | |
1022 | case RTE_ETH_FILTER_ADD: | |
1023 | ret = priv_fdir_filter_add(priv, arg); | |
1024 | break; | |
1025 | case RTE_ETH_FILTER_UPDATE: | |
1026 | ret = priv_fdir_filter_update(priv, arg); | |
1027 | break; | |
1028 | case RTE_ETH_FILTER_DELETE: | |
1029 | ret = priv_fdir_filter_delete(priv, arg); | |
1030 | break; | |
1031 | case RTE_ETH_FILTER_FLUSH: | |
1032 | priv_fdir_filter_flush(priv); | |
1033 | break; | |
1034 | case RTE_ETH_FILTER_INFO: | |
1035 | priv_fdir_info_get(priv, arg); | |
1036 | break; | |
1037 | default: | |
1038 | DEBUG("%p: unknown operation %u", (void *)priv, filter_op); | |
1039 | ret = EINVAL; | |
1040 | break; | |
1041 | } | |
1042 | return ret; | |
1043 | } | |
1044 | ||
1045 | /** | |
1046 | * Manage filter operations. | |
1047 | * | |
1048 | * @param dev | |
1049 | * Pointer to Ethernet device structure. | |
1050 | * @param filter_type | |
1051 | * Filter type. | |
1052 | * @param filter_op | |
1053 | * Operation to perform. | |
1054 | * @param arg | |
1055 | * Pointer to operation-specific structure. | |
1056 | * | |
1057 | * @return | |
1058 | * 0 on success, negative errno value on failure. | |
1059 | */ | |
1060 | int | |
1061 | mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, | |
1062 | enum rte_filter_type filter_type, | |
1063 | enum rte_filter_op filter_op, | |
1064 | void *arg) | |
1065 | { | |
1066 | int ret = EINVAL; | |
1067 | struct priv *priv = dev->data->dev_private; | |
1068 | ||
1069 | switch (filter_type) { | |
1070 | case RTE_ETH_FILTER_FDIR: | |
1071 | priv_lock(priv); | |
1072 | ret = priv_fdir_ctrl_func(priv, filter_op, arg); | |
1073 | priv_unlock(priv); | |
1074 | break; | |
1075 | default: | |
1076 | ERROR("%p: filter type (%d) not supported", | |
1077 | (void *)dev, filter_type); | |
1078 | break; | |
1079 | } | |
1080 | ||
1081 | return -ret; | |
1082 | } |