]> git.proxmox.com Git - ceph.git/blob - ceph/src/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
bump version to 12.2.12-pve1
[ceph.git] / ceph / src / dpdk / examples / ip_pipeline / pipeline / pipeline_passthrough_be.c
1 /*-
2 * BSD LICENSE
3 *
4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include <stdio.h>
35 #include <string.h>
36
37 #include <rte_common.h>
38 #include <rte_malloc.h>
39 #include <rte_byteorder.h>
40 #include <rte_table_stub.h>
41 #include <rte_table_hash.h>
42 #include <rte_pipeline.h>
43
44 #include "pipeline_passthrough_be.h"
45 #include "pipeline_actions_common.h"
46 #include "parser.h"
47 #include "hash_func.h"
48
49 #define SWAP_DIM (PIPELINE_PASSTHROUGH_SWAP_N_FIELDS_MAX * \
50 (PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX / sizeof(uint64_t)))
51
52 struct pipeline_passthrough {
53 struct pipeline p;
54 struct pipeline_passthrough_params params;
55 rte_table_hash_op_hash f_hash;
56 uint32_t swap_field0_offset[SWAP_DIM];
57 uint32_t swap_field1_offset[SWAP_DIM];
58 uint64_t swap_field_mask[SWAP_DIM];
59 uint32_t swap_n_fields;
60 } __rte_cache_aligned;
61
62 static pipeline_msg_req_handler handlers[] = {
63 [PIPELINE_MSG_REQ_PING] =
64 pipeline_msg_req_ping_handler,
65 [PIPELINE_MSG_REQ_STATS_PORT_IN] =
66 pipeline_msg_req_stats_port_in_handler,
67 [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
68 pipeline_msg_req_stats_port_out_handler,
69 [PIPELINE_MSG_REQ_STATS_TABLE] =
70 pipeline_msg_req_stats_table_handler,
71 [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
72 pipeline_msg_req_port_in_enable_handler,
73 [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
74 pipeline_msg_req_port_in_disable_handler,
75 [PIPELINE_MSG_REQ_CUSTOM] =
76 pipeline_msg_req_invalid_handler,
77 };
78
79 static inline __attribute__((always_inline)) void
80 pkt_work_dma(
81 struct rte_mbuf *pkt,
82 void *arg,
83 uint32_t dma_size,
84 uint32_t hash_enabled,
85 uint32_t lb_hash,
86 uint32_t port_out_pow2)
87 {
88 struct pipeline_passthrough *p = arg;
89
90 uint64_t *dma_dst = RTE_MBUF_METADATA_UINT64_PTR(pkt,
91 p->params.dma_dst_offset);
92 uint64_t *dma_src = RTE_MBUF_METADATA_UINT64_PTR(pkt,
93 p->params.dma_src_offset);
94 uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
95 uint32_t *dma_hash = RTE_MBUF_METADATA_UINT32_PTR(pkt,
96 p->params.dma_hash_offset);
97 uint32_t i;
98
99 /* Read (dma_src), compute (dma_dst), write (dma_dst) */
100 for (i = 0; i < (dma_size / 8); i++)
101 dma_dst[i] = dma_src[i] & dma_mask[i];
102
103 /* Read (dma_dst), compute (hash), write (hash) */
104 if (hash_enabled) {
105 uint32_t hash = p->f_hash(dma_dst, dma_size, 0);
106 *dma_hash = hash;
107
108 if (lb_hash) {
109 uint32_t port_out;
110
111 if (port_out_pow2)
112 port_out
113 = hash & (p->p.n_ports_out - 1);
114 else
115 port_out
116 = hash % p->p.n_ports_out;
117
118 rte_pipeline_port_out_packet_insert(p->p.p,
119 port_out, pkt);
120 }
121 }
122 }
123
124 static inline __attribute__((always_inline)) void
125 pkt4_work_dma(
126 struct rte_mbuf **pkts,
127 void *arg,
128 uint32_t dma_size,
129 uint32_t hash_enabled,
130 uint32_t lb_hash,
131 uint32_t port_out_pow2)
132 {
133 struct pipeline_passthrough *p = arg;
134
135 uint64_t *dma_dst0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
136 p->params.dma_dst_offset);
137 uint64_t *dma_dst1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
138 p->params.dma_dst_offset);
139 uint64_t *dma_dst2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
140 p->params.dma_dst_offset);
141 uint64_t *dma_dst3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
142 p->params.dma_dst_offset);
143
144 uint64_t *dma_src0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
145 p->params.dma_src_offset);
146 uint64_t *dma_src1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
147 p->params.dma_src_offset);
148 uint64_t *dma_src2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
149 p->params.dma_src_offset);
150 uint64_t *dma_src3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
151 p->params.dma_src_offset);
152
153 uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
154
155 uint32_t *dma_hash0 = RTE_MBUF_METADATA_UINT32_PTR(pkts[0],
156 p->params.dma_hash_offset);
157 uint32_t *dma_hash1 = RTE_MBUF_METADATA_UINT32_PTR(pkts[1],
158 p->params.dma_hash_offset);
159 uint32_t *dma_hash2 = RTE_MBUF_METADATA_UINT32_PTR(pkts[2],
160 p->params.dma_hash_offset);
161 uint32_t *dma_hash3 = RTE_MBUF_METADATA_UINT32_PTR(pkts[3],
162 p->params.dma_hash_offset);
163
164 uint32_t i;
165
166 /* Read (dma_src), compute (dma_dst), write (dma_dst) */
167 for (i = 0; i < (dma_size / 8); i++) {
168 dma_dst0[i] = dma_src0[i] & dma_mask[i];
169 dma_dst1[i] = dma_src1[i] & dma_mask[i];
170 dma_dst2[i] = dma_src2[i] & dma_mask[i];
171 dma_dst3[i] = dma_src3[i] & dma_mask[i];
172 }
173
174 /* Read (dma_dst), compute (hash), write (hash) */
175 if (hash_enabled) {
176 uint32_t hash0 = p->f_hash(dma_dst0, dma_size, 0);
177 uint32_t hash1 = p->f_hash(dma_dst1, dma_size, 0);
178 uint32_t hash2 = p->f_hash(dma_dst2, dma_size, 0);
179 uint32_t hash3 = p->f_hash(dma_dst3, dma_size, 0);
180
181 *dma_hash0 = hash0;
182 *dma_hash1 = hash1;
183 *dma_hash2 = hash2;
184 *dma_hash3 = hash3;
185
186 if (lb_hash) {
187 uint32_t port_out0, port_out1, port_out2, port_out3;
188
189 if (port_out_pow2) {
190 port_out0
191 = hash0 & (p->p.n_ports_out - 1);
192 port_out1
193 = hash1 & (p->p.n_ports_out - 1);
194 port_out2
195 = hash2 & (p->p.n_ports_out - 1);
196 port_out3
197 = hash3 & (p->p.n_ports_out - 1);
198 } else {
199 port_out0
200 = hash0 % p->p.n_ports_out;
201 port_out1
202 = hash1 % p->p.n_ports_out;
203 port_out2
204 = hash2 % p->p.n_ports_out;
205 port_out3
206 = hash3 % p->p.n_ports_out;
207 }
208 rte_pipeline_port_out_packet_insert(p->p.p,
209 port_out0, pkts[0]);
210 rte_pipeline_port_out_packet_insert(p->p.p,
211 port_out1, pkts[1]);
212 rte_pipeline_port_out_packet_insert(p->p.p,
213 port_out2, pkts[2]);
214 rte_pipeline_port_out_packet_insert(p->p.p,
215 port_out3, pkts[3]);
216 }
217 }
218 }
219
220 static inline __attribute__((always_inline)) void
221 pkt_work_swap(
222 struct rte_mbuf *pkt,
223 void *arg)
224 {
225 struct pipeline_passthrough *p = arg;
226 uint32_t i;
227
228 /* Read(field0, field1), compute(field0, field1), write(field0, field1) */
229 for (i = 0; i < p->swap_n_fields; i++) {
230 uint64_t *field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt,
231 p->swap_field0_offset[i]);
232 uint64_t *field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt,
233 p->swap_field1_offset[i]);
234 uint64_t mask = p->swap_field_mask[i];
235
236 uint64_t field0 = *field0_ptr;
237 uint64_t field1 = *field1_ptr;
238
239 *field0_ptr = (field0 & (~mask)) + (field1 & mask);
240 *field1_ptr = (field0 & mask) + (field1 & (~mask));
241 }
242 }
243
244 static inline __attribute__((always_inline)) void
245 pkt4_work_swap(
246 struct rte_mbuf **pkts,
247 void *arg)
248 {
249 struct pipeline_passthrough *p = arg;
250 uint32_t i;
251
252 /* Read(field0, field1), compute(field0, field1), write(field0, field1) */
253 for (i = 0; i < p->swap_n_fields; i++) {
254 uint64_t *pkt0_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
255 p->swap_field0_offset[i]);
256 uint64_t *pkt1_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
257 p->swap_field0_offset[i]);
258 uint64_t *pkt2_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
259 p->swap_field0_offset[i]);
260 uint64_t *pkt3_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
261 p->swap_field0_offset[i]);
262
263 uint64_t *pkt0_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
264 p->swap_field1_offset[i]);
265 uint64_t *pkt1_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
266 p->swap_field1_offset[i]);
267 uint64_t *pkt2_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
268 p->swap_field1_offset[i]);
269 uint64_t *pkt3_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
270 p->swap_field1_offset[i]);
271
272 uint64_t mask = p->swap_field_mask[i];
273
274 uint64_t pkt0_field0 = *pkt0_field0_ptr;
275 uint64_t pkt1_field0 = *pkt1_field0_ptr;
276 uint64_t pkt2_field0 = *pkt2_field0_ptr;
277 uint64_t pkt3_field0 = *pkt3_field0_ptr;
278
279 uint64_t pkt0_field1 = *pkt0_field1_ptr;
280 uint64_t pkt1_field1 = *pkt1_field1_ptr;
281 uint64_t pkt2_field1 = *pkt2_field1_ptr;
282 uint64_t pkt3_field1 = *pkt3_field1_ptr;
283
284 *pkt0_field0_ptr = (pkt0_field0 & (~mask)) + (pkt0_field1 & mask);
285 *pkt1_field0_ptr = (pkt1_field0 & (~mask)) + (pkt1_field1 & mask);
286 *pkt2_field0_ptr = (pkt2_field0 & (~mask)) + (pkt2_field1 & mask);
287 *pkt3_field0_ptr = (pkt3_field0 & (~mask)) + (pkt3_field1 & mask);
288
289 *pkt0_field1_ptr = (pkt0_field0 & mask) + (pkt0_field1 & (~mask));
290 *pkt1_field1_ptr = (pkt1_field0 & mask) + (pkt1_field1 & (~mask));
291 *pkt2_field1_ptr = (pkt2_field0 & mask) + (pkt2_field1 & (~mask));
292 *pkt3_field1_ptr = (pkt3_field0 & mask) + (pkt3_field1 & (~mask));
293 }
294 }
295
296 #define PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
297 static inline void \
298 pkt_work_dma_size##dma_size##_hash##hash_enabled \
299 ##_lb##lb_hash##_pw##port_pow2( \
300 struct rte_mbuf *pkt, \
301 void *arg) \
302 { \
303 pkt_work_dma(pkt, arg, dma_size, hash_enabled, lb_hash, port_pow2); \
304 }
305
306 #define PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
307 static inline void \
308 pkt4_work_dma_size##dma_size##_hash##hash_enabled \
309 ##_lb##lb_hash##_pw##port_pow2( \
310 struct rte_mbuf **pkts, \
311 void *arg) \
312 { \
313 pkt4_work_dma(pkts, arg, dma_size, hash_enabled, lb_hash, port_pow2); \
314 }
315
316 #define port_in_ah_dma(dma_size, hash_enabled, lb_hash, port_pow2) \
317 PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
318 PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
319 PIPELINE_PORT_IN_AH(port_in_ah_dma_size##dma_size##_hash \
320 ##hash_enabled##_lb##lb_hash##_pw##port_pow2, \
321 pkt_work_dma_size##dma_size##_hash##hash_enabled \
322 ##_lb##lb_hash##_pw##port_pow2, \
323 pkt4_work_dma_size##dma_size##_hash##hash_enabled \
324 ##_lb##lb_hash##_pw##port_pow2)
325
326
327 #define port_in_ah_lb(dma_size, hash_enabled, lb_hash, port_pow2) \
328 PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
329 PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
330 PIPELINE_PORT_IN_AH_HIJACK_ALL( \
331 port_in_ah_lb_size##dma_size##_hash##hash_enabled \
332 ##_lb##lb_hash##_pw##port_pow2, \
333 pkt_work_dma_size##dma_size##_hash##hash_enabled \
334 ##_lb##lb_hash##_pw##port_pow2, \
335 pkt4_work_dma_size##dma_size##_hash##hash_enabled \
336 ##_lb##lb_hash##_pw##port_pow2)
337
338 PIPELINE_PORT_IN_AH(port_in_ah_swap, pkt_work_swap, pkt4_work_swap)
339
340
341 /* Port in AH DMA(dma_size, hash_enabled, lb_hash, port_pow2) */
342
343 port_in_ah_dma(8, 0, 0, 0)
344 port_in_ah_dma(8, 1, 0, 0)
345 port_in_ah_lb(8, 1, 1, 0)
346 port_in_ah_lb(8, 1, 1, 1)
347
348 port_in_ah_dma(16, 0, 0, 0)
349 port_in_ah_dma(16, 1, 0, 0)
350 port_in_ah_lb(16, 1, 1, 0)
351 port_in_ah_lb(16, 1, 1, 1)
352
353 port_in_ah_dma(24, 0, 0, 0)
354 port_in_ah_dma(24, 1, 0, 0)
355 port_in_ah_lb(24, 1, 1, 0)
356 port_in_ah_lb(24, 1, 1, 1)
357
358 port_in_ah_dma(32, 0, 0, 0)
359 port_in_ah_dma(32, 1, 0, 0)
360 port_in_ah_lb(32, 1, 1, 0)
361 port_in_ah_lb(32, 1, 1, 1)
362
363 port_in_ah_dma(40, 0, 0, 0)
364 port_in_ah_dma(40, 1, 0, 0)
365 port_in_ah_lb(40, 1, 1, 0)
366 port_in_ah_lb(40, 1, 1, 1)
367
368 port_in_ah_dma(48, 0, 0, 0)
369 port_in_ah_dma(48, 1, 0, 0)
370 port_in_ah_lb(48, 1, 1, 0)
371 port_in_ah_lb(48, 1, 1, 1)
372
373 port_in_ah_dma(56, 0, 0, 0)
374 port_in_ah_dma(56, 1, 0, 0)
375 port_in_ah_lb(56, 1, 1, 0)
376 port_in_ah_lb(56, 1, 1, 1)
377
378 port_in_ah_dma(64, 0, 0, 0)
379 port_in_ah_dma(64, 1, 0, 0)
380 port_in_ah_lb(64, 1, 1, 0)
381 port_in_ah_lb(64, 1, 1, 1)
382
383 static rte_pipeline_port_in_action_handler
384 get_port_in_ah(struct pipeline_passthrough *p)
385 {
386 if ((p->params.dma_enabled == 0) &&
387 (p->params.swap_enabled == 0))
388 return NULL;
389
390 if (p->params.swap_enabled)
391 return port_in_ah_swap;
392
393 if (p->params.dma_hash_enabled) {
394 if (p->params.dma_hash_lb_enabled) {
395 if (rte_is_power_of_2(p->p.n_ports_out))
396 switch (p->params.dma_size) {
397
398 case 8: return port_in_ah_lb_size8_hash1_lb1_pw1;
399 case 16: return port_in_ah_lb_size16_hash1_lb1_pw1;
400 case 24: return port_in_ah_lb_size24_hash1_lb1_pw1;
401 case 32: return port_in_ah_lb_size32_hash1_lb1_pw1;
402 case 40: return port_in_ah_lb_size40_hash1_lb1_pw1;
403 case 48: return port_in_ah_lb_size48_hash1_lb1_pw1;
404 case 56: return port_in_ah_lb_size56_hash1_lb1_pw1;
405 case 64: return port_in_ah_lb_size64_hash1_lb1_pw1;
406 default: return NULL;
407 }
408 else
409 switch (p->params.dma_size) {
410
411 case 8: return port_in_ah_lb_size8_hash1_lb1_pw0;
412 case 16: return port_in_ah_lb_size16_hash1_lb1_pw0;
413 case 24: return port_in_ah_lb_size24_hash1_lb1_pw0;
414 case 32: return port_in_ah_lb_size32_hash1_lb1_pw0;
415 case 40: return port_in_ah_lb_size40_hash1_lb1_pw0;
416 case 48: return port_in_ah_lb_size48_hash1_lb1_pw0;
417 case 56: return port_in_ah_lb_size56_hash1_lb1_pw0;
418 case 64: return port_in_ah_lb_size64_hash1_lb1_pw0;
419 default: return NULL;
420 }
421 } else
422 switch (p->params.dma_size) {
423
424 case 8: return port_in_ah_dma_size8_hash1_lb0_pw0;
425 case 16: return port_in_ah_dma_size16_hash1_lb0_pw0;
426 case 24: return port_in_ah_dma_size24_hash1_lb0_pw0;
427 case 32: return port_in_ah_dma_size32_hash1_lb0_pw0;
428 case 40: return port_in_ah_dma_size40_hash1_lb0_pw0;
429 case 48: return port_in_ah_dma_size48_hash1_lb0_pw0;
430 case 56: return port_in_ah_dma_size56_hash1_lb0_pw0;
431 case 64: return port_in_ah_dma_size64_hash1_lb0_pw0;
432 default: return NULL;
433 }
434 } else
435 switch (p->params.dma_size) {
436
437 case 8: return port_in_ah_dma_size8_hash0_lb0_pw0;
438 case 16: return port_in_ah_dma_size16_hash0_lb0_pw0;
439 case 24: return port_in_ah_dma_size24_hash0_lb0_pw0;
440 case 32: return port_in_ah_dma_size32_hash0_lb0_pw0;
441 case 40: return port_in_ah_dma_size40_hash0_lb0_pw0;
442 case 48: return port_in_ah_dma_size48_hash0_lb0_pw0;
443 case 56: return port_in_ah_dma_size56_hash0_lb0_pw0;
444 case 64: return port_in_ah_dma_size64_hash0_lb0_pw0;
445 default: return NULL;
446 }
447 }
448
449 int
450 pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p,
451 struct pipeline_params *params)
452 {
453 uint32_t dma_dst_offset_present = 0;
454 uint32_t dma_src_offset_present = 0;
455 uint32_t dma_src_mask_present = 0;
456 char dma_mask_str[PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2 + 1];
457 uint32_t dma_size_present = 0;
458 uint32_t dma_hash_offset_present = 0;
459 uint32_t dma_hash_lb_present = 0;
460 uint32_t i;
461
462 /* default values */
463 p->dma_enabled = 0;
464 p->dma_hash_enabled = 0;
465 p->dma_hash_lb_enabled = 0;
466 memset(p->dma_src_mask, 0xFF, sizeof(p->dma_src_mask));
467 p->swap_enabled = 0;
468 p->swap_n_fields = 0;
469
470 for (i = 0; i < params->n_args; i++) {
471 char *arg_name = params->args_name[i];
472 char *arg_value = params->args_value[i];
473
474 /* dma_dst_offset */
475 if (strcmp(arg_name, "dma_dst_offset") == 0) {
476 int status;
477
478 PIPELINE_PARSE_ERR_DUPLICATE(
479 dma_dst_offset_present == 0, params->name,
480 arg_name);
481 dma_dst_offset_present = 1;
482
483 status = parser_read_uint32(&p->dma_dst_offset,
484 arg_value);
485 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
486 params->name, arg_name, arg_value);
487 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
488 params->name, arg_name, arg_value);
489
490 p->dma_enabled = 1;
491
492 continue;
493 }
494
495 /* dma_src_offset */
496 if (strcmp(arg_name, "dma_src_offset") == 0) {
497 int status;
498
499 PIPELINE_PARSE_ERR_DUPLICATE(
500 dma_src_offset_present == 0, params->name,
501 arg_name);
502 dma_src_offset_present = 1;
503
504 status = parser_read_uint32(&p->dma_src_offset,
505 arg_value);
506 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
507 params->name, arg_name, arg_value);
508 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
509 params->name, arg_name, arg_value);
510
511 p->dma_enabled = 1;
512
513 continue;
514 }
515
516 /* dma_size */
517 if (strcmp(arg_name, "dma_size") == 0) {
518 int status;
519
520 PIPELINE_PARSE_ERR_DUPLICATE(
521 dma_size_present == 0, params->name,
522 arg_name);
523 dma_size_present = 1;
524
525 status = parser_read_uint32(&p->dma_size,
526 arg_value);
527 PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
528 (p->dma_size != 0) &&
529 ((p->dma_size % 8) == 0)),
530 params->name, arg_name, arg_value);
531 PIPELINE_PARSE_ERR_OUT_RNG(((status != -ERANGE) &&
532 (p->dma_size <=
533 PIPELINE_PASSTHROUGH_DMA_SIZE_MAX)),
534 params->name, arg_name, arg_value);
535
536 p->dma_enabled = 1;
537
538 continue;
539 }
540
541 /* dma_src_mask */
542 if (strcmp(arg_name, "dma_src_mask") == 0) {
543 int mask_str_len = strlen(arg_value);
544
545 PIPELINE_PARSE_ERR_DUPLICATE(
546 dma_src_mask_present == 0,
547 params->name, arg_name);
548 dma_src_mask_present = 1;
549
550 PIPELINE_ARG_CHECK((mask_str_len <=
551 (PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2)),
552 "Parse error in section \"%s\": entry "
553 "\"%s\" too long", params->name,
554 arg_name);
555
556 snprintf(dma_mask_str, mask_str_len + 1,
557 "%s", arg_value);
558
559 p->dma_enabled = 1;
560
561 continue;
562 }
563
564 /* dma_hash_offset */
565 if (strcmp(arg_name, "dma_hash_offset") == 0) {
566 int status;
567
568 PIPELINE_PARSE_ERR_DUPLICATE(
569 dma_hash_offset_present == 0,
570 params->name, arg_name);
571 dma_hash_offset_present = 1;
572
573 status = parser_read_uint32(&p->dma_hash_offset,
574 arg_value);
575 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
576 params->name, arg_name, arg_value);
577 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
578 params->name, arg_name, arg_value);
579
580 p->dma_hash_enabled = 1;
581
582 continue;
583 }
584
585 /* load_balance mode */
586 if (strcmp(arg_name, "lb") == 0) {
587 PIPELINE_PARSE_ERR_DUPLICATE(
588 dma_hash_lb_present == 0,
589 params->name, arg_name);
590 dma_hash_lb_present = 1;
591
592 if (strcmp(arg_value, "hash") ||
593 strcmp(arg_value, "HASH"))
594
595 PIPELINE_PARSE_ERR_INV_VAL(0,
596 params->name,
597 arg_name,
598 arg_value);
599
600 p->dma_hash_lb_enabled = 1;
601
602 continue;
603 }
604
605 /* swap */
606 if (strcmp(arg_name, "swap") == 0) {
607 uint32_t a, b, n_args;
608 int len;
609
610 n_args = sscanf(arg_value, "%" SCNu32 " %" SCNu32 "%n",
611 &a, &b, &len);
612 PIPELINE_PARSE_ERR_INV_VAL(((n_args == 2) &&
613 ((size_t) len == strlen(arg_value))),
614 params->name, arg_name, arg_value);
615
616 p->swap_field0_offset[p->swap_n_fields] = a;
617 p->swap_field1_offset[p->swap_n_fields] = b;
618 p->swap_n_fields++;
619 p->swap_enabled = 1;
620
621 continue;
622 }
623
624 /* any other */
625 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
626 }
627
628 /* Check correlations between arguments */
629 PIPELINE_ARG_CHECK((p->dma_enabled + p->swap_enabled < 2),
630 "Parse error in section \"%s\": DMA and SWAP actions are both enabled",
631 params->name);
632 PIPELINE_ARG_CHECK((dma_dst_offset_present == p->dma_enabled),
633 "Parse error in section \"%s\": missing entry "
634 "\"dma_dst_offset\"", params->name);
635 PIPELINE_ARG_CHECK((dma_src_offset_present == p->dma_enabled),
636 "Parse error in section \"%s\": missing entry "
637 "\"dma_src_offset\"", params->name);
638 PIPELINE_ARG_CHECK((dma_size_present == p->dma_enabled),
639 "Parse error in section \"%s\": missing entry "
640 "\"dma_size\"", params->name);
641 PIPELINE_ARG_CHECK((p->dma_hash_enabled <= p->dma_enabled),
642 "Parse error in section \"%s\": missing all DMA entries",
643 params->name);
644 PIPELINE_ARG_CHECK((p->dma_hash_lb_enabled <= p->dma_hash_enabled),
645 "Parse error in section \"%s\": missing all DMA hash entries ",
646 params->name);
647
648 if (dma_src_mask_present) {
649 uint32_t dma_size = p->dma_size;
650 int status;
651
652 PIPELINE_ARG_CHECK((strlen(dma_mask_str) ==
653 (dma_size * 2)), "Parse error in section "
654 "\"%s\": dma_src_mask should have exactly %u hex "
655 "digits", params->name, (dma_size * 2));
656
657 status = parse_hex_string(dma_mask_str, p->dma_src_mask,
658 &p->dma_size);
659
660 PIPELINE_PARSE_ERR_INV_VAL(((status == 0) &&
661 (dma_size == p->dma_size)), params->name,
662 "dma_src_mask", dma_mask_str);
663 }
664
665 if (p->dma_hash_lb_enabled)
666 PIPELINE_ARG_CHECK((params->n_ports_out > 1),
667 "Parse error in section \"%s\": entry \"lb\" not "
668 "allowed for single output port pipeline",
669 params->name);
670 else
671 PIPELINE_ARG_CHECK(((params->n_ports_in >= params->n_ports_out)
672 && ((params->n_ports_in % params->n_ports_out) == 0)),
673 "Parse error in section \"%s\": n_ports_in needs to be "
674 "a multiple of n_ports_out (lb mode disabled)",
675 params->name);
676
677 return 0;
678 }
679
680 static rte_table_hash_op_hash
681 get_hash_function(struct pipeline_passthrough *p)
682 {
683 switch (p->params.dma_size) {
684
685 case 8: return hash_default_key8;
686 case 16: return hash_default_key16;
687 case 24: return hash_default_key24;
688 case 32: return hash_default_key32;
689 case 40: return hash_default_key40;
690 case 48: return hash_default_key48;
691 case 56: return hash_default_key56;
692 case 64: return hash_default_key64;
693 default: return NULL;
694 }
695 }
696
697 static int
698 pipeline_passthrough_swap_convert(struct pipeline_passthrough *p)
699 {
700 uint32_t i;
701
702 p->swap_n_fields = 0;
703
704 for (i = 0; i < p->params.swap_n_fields; i++) {
705 uint32_t offset0 = p->params.swap_field0_offset[i];
706 uint32_t offset1 = p->params.swap_field1_offset[i];
707 uint32_t size = offset1 - offset0;
708 uint32_t j;
709
710 /* Check */
711 if ((offset0 >= offset1) ||
712 (size > PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX) ||
713 (p->swap_n_fields >= SWAP_DIM))
714 return -1;
715
716 for (j = 0; j < (size / sizeof(uint64_t)); j++) {
717 p->swap_field0_offset[p->swap_n_fields] = offset0;
718 p->swap_field1_offset[p->swap_n_fields] = offset1;
719 p->swap_field_mask[p->swap_n_fields] = UINT64_MAX;
720 p->swap_n_fields++;
721 offset0 += sizeof(uint64_t);
722 offset1 += sizeof(uint64_t);
723 }
724 if (size % sizeof(uint64_t)) {
725 uint32_t n_bits = (size % sizeof(uint64_t)) * 8;
726
727 p->swap_field0_offset[p->swap_n_fields] = offset0;
728 p->swap_field1_offset[p->swap_n_fields] = offset1;
729 p->swap_field_mask[p->swap_n_fields] =
730 RTE_LEN2MASK(n_bits, uint64_t);
731 p->swap_n_fields++;
732 }
733 }
734
735 return 0;
736 }
737
738 static void*
739 pipeline_passthrough_init(struct pipeline_params *params,
740 __rte_unused void *arg)
741 {
742 struct pipeline *p;
743 struct pipeline_passthrough *p_pt;
744 uint32_t size, i;
745
746 /* Check input arguments */
747 if ((params == NULL) ||
748 (params->n_ports_in == 0) ||
749 (params->n_ports_out == 0))
750 return NULL;
751
752 /* Memory allocation */
753 size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_passthrough));
754 p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
755 p_pt = (struct pipeline_passthrough *) p;
756 if (p == NULL)
757 return NULL;
758
759 strcpy(p->name, params->name);
760 p->log_level = params->log_level;
761
762 PLOG(p, HIGH, "Pass-through");
763
764 /* Parse arguments */
765 if (pipeline_passthrough_parse_args(&p_pt->params, params))
766 return NULL;
767 if (pipeline_passthrough_swap_convert(p_pt))
768 return NULL;
769 p_pt->f_hash = get_hash_function(p_pt);
770
771 /* Pipeline */
772 {
773 struct rte_pipeline_params pipeline_params = {
774 .name = "PASS-THROUGH",
775 .socket_id = params->socket_id,
776 .offset_port_id = 0,
777 };
778
779 p->p = rte_pipeline_create(&pipeline_params);
780 if (p->p == NULL) {
781 rte_free(p);
782 return NULL;
783 }
784 }
785
786 p->n_ports_in = params->n_ports_in;
787 p->n_ports_out = params->n_ports_out;
788 p->n_tables = p->n_ports_in;
789
790 /*Input ports*/
791 for (i = 0; i < p->n_ports_in; i++) {
792 struct rte_pipeline_port_in_params port_params = {
793 .ops = pipeline_port_in_params_get_ops(
794 &params->port_in[i]),
795 .arg_create = pipeline_port_in_params_convert(
796 &params->port_in[i]),
797 .f_action = get_port_in_ah(p_pt),
798 .arg_ah = p_pt,
799 .burst_size = params->port_in[i].burst_size,
800 };
801
802 int status = rte_pipeline_port_in_create(p->p,
803 &port_params,
804 &p->port_in_id[i]);
805
806 if (status) {
807 rte_pipeline_free(p->p);
808 rte_free(p);
809 return NULL;
810 }
811 }
812
813 /* Output ports */
814 for (i = 0; i < p->n_ports_out; i++) {
815 struct rte_pipeline_port_out_params port_params = {
816 .ops = pipeline_port_out_params_get_ops(
817 &params->port_out[i]),
818 .arg_create = pipeline_port_out_params_convert(
819 &params->port_out[i]),
820 .f_action = NULL,
821 .arg_ah = NULL,
822 };
823
824 int status = rte_pipeline_port_out_create(p->p,
825 &port_params,
826 &p->port_out_id[i]);
827
828 if (status) {
829 rte_pipeline_free(p->p);
830 rte_free(p);
831 return NULL;
832 }
833 }
834
835 /* Tables */
836 for (i = 0; i < p->n_ports_in; i++) {
837 struct rte_pipeline_table_params table_params = {
838 .ops = &rte_table_stub_ops,
839 .arg_create = NULL,
840 .f_action_hit = NULL,
841 .f_action_miss = NULL,
842 .arg_ah = NULL,
843 .action_data_size = 0,
844 };
845
846 int status = rte_pipeline_table_create(p->p,
847 &table_params,
848 &p->table_id[i]);
849
850 if (status) {
851 rte_pipeline_free(p->p);
852 rte_free(p);
853 return NULL;
854 }
855 }
856
857 /* Connecting input ports to tables */
858 for (i = 0; i < p->n_ports_in; i++) {
859 int status = rte_pipeline_port_in_connect_to_table(p->p,
860 p->port_in_id[i],
861 p->table_id[i]);
862
863 if (status) {
864 rte_pipeline_free(p->p);
865 rte_free(p);
866 return NULL;
867 }
868 }
869
870 /* Add entries to tables */
871 for (i = 0; i < p->n_ports_in; i++) {
872 uint32_t port_out_id = (p_pt->params.dma_hash_lb_enabled == 0) ?
873 (i / (p->n_ports_in / p->n_ports_out)) :
874 0;
875
876 struct rte_pipeline_table_entry default_entry = {
877 .action = RTE_PIPELINE_ACTION_PORT,
878 {.port_id = p->port_out_id[port_out_id]},
879 };
880
881 struct rte_pipeline_table_entry *default_entry_ptr;
882
883 int status = rte_pipeline_table_default_entry_add(p->p,
884 p->table_id[i],
885 &default_entry,
886 &default_entry_ptr);
887
888 if (status) {
889 rte_pipeline_free(p->p);
890 rte_free(p);
891 return NULL;
892 }
893 }
894
895 /* Enable input ports */
896 for (i = 0; i < p->n_ports_in; i++) {
897 int status = rte_pipeline_port_in_enable(p->p,
898 p->port_in_id[i]);
899
900 if (status) {
901 rte_pipeline_free(p->p);
902 rte_free(p);
903 return NULL;
904 }
905 }
906
907 /* Check pipeline consistency */
908 if (rte_pipeline_check(p->p) < 0) {
909 rte_pipeline_free(p->p);
910 rte_free(p);
911 return NULL;
912 }
913
914 /* Message queues */
915 p->n_msgq = params->n_msgq;
916 for (i = 0; i < p->n_msgq; i++)
917 p->msgq_in[i] = params->msgq_in[i];
918 for (i = 0; i < p->n_msgq; i++)
919 p->msgq_out[i] = params->msgq_out[i];
920
921 /* Message handlers */
922 memcpy(p->handlers, handlers, sizeof(p->handlers));
923
924 return p;
925 }
926
927 static int
928 pipeline_passthrough_free(void *pipeline)
929 {
930 struct pipeline *p = (struct pipeline *) pipeline;
931
932 /* Check input arguments */
933 if (p == NULL)
934 return -1;
935
936 /* Free resources */
937 rte_pipeline_free(p->p);
938 rte_free(p);
939 return 0;
940 }
941
942 static int
943 pipeline_passthrough_timer(void *pipeline)
944 {
945 struct pipeline *p = (struct pipeline *) pipeline;
946
947 pipeline_msg_req_handle(p);
948 rte_pipeline_flush(p->p);
949
950 return 0;
951 }
952
953 struct pipeline_be_ops pipeline_passthrough_be_ops = {
954 .f_init = pipeline_passthrough_init,
955 .f_free = pipeline_passthrough_free,
956 .f_run = NULL,
957 .f_timer = pipeline_passthrough_timer,
958 };
Ceph