1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
7 #include <rte_cycles.h>
9 #include <rte_malloc.h>
10 #include <rte_ethdev.h>
12 #include <rte_meter.h>
13 #include <rte_eth_softnic.h>
18 #define SUBPORT_NODES_PER_PORT 1
19 #define PIPE_NODES_PER_SUBPORT 4096
20 #define TC_NODES_PER_PIPE 4
21 #define QUEUE_NODES_PER_TC 4
23 #define NUM_PIPE_NODES \
24 (SUBPORT_NODES_PER_PORT * PIPE_NODES_PER_SUBPORT)
26 #define NUM_TC_NODES \
27 (NUM_PIPE_NODES * TC_NODES_PER_PIPE)
29 #define ROOT_NODE_ID 1000000
30 #define SUBPORT_NODES_START_ID 900000
31 #define PIPE_NODES_START_ID 800000
32 #define TC_NODES_START_ID 700000
34 #define STATS_MASK_DEFAULT \
35 (RTE_TM_STATS_N_PKTS | \
36 RTE_TM_STATS_N_BYTES | \
37 RTE_TM_STATS_N_PKTS_GREEN_DROPPED | \
38 RTE_TM_STATS_N_BYTES_GREEN_DROPPED)
40 #define STATS_MASK_QUEUE \
41 (STATS_MASK_DEFAULT | \
42 RTE_TM_STATS_N_PKTS_QUEUED)
44 #define BYTES_IN_MBPS (1000 * 1000 / 8)
45 #define TOKEN_BUCKET_SIZE 1000000
47 /* TM Hierarchy Levels */
48 enum tm_hierarchy_level
{
49 TM_NODE_LEVEL_PORT
= 0,
50 TM_NODE_LEVEL_SUBPORT
,
59 uint32_t root_node_id
;
60 uint32_t subport_node_id
[SUBPORT_NODES_PER_PORT
];
61 uint32_t pipe_node_id
[SUBPORT_NODES_PER_PORT
][PIPE_NODES_PER_SUBPORT
];
62 uint32_t tc_node_id
[NUM_PIPE_NODES
][TC_NODES_PER_PIPE
];
63 uint32_t queue_node_id
[NUM_TC_NODES
][QUEUE_NODES_PER_TC
];
65 /* TM Hierarchy Nodes Shaper Rates */
66 uint32_t root_node_shaper_rate
;
67 uint32_t subport_node_shaper_rate
;
68 uint32_t pipe_node_shaper_rate
;
69 uint32_t tc_node_shaper_rate
;
70 uint32_t tc_node_shared_shaper_rate
;
75 static struct fwd_lcore
*softnic_fwd_lcore
;
76 static uint16_t softnic_port_id
;
77 struct fwd_engine softnic_fwd_engine
;
80 * Softnic packet forward
83 softnic_fwd(struct fwd_stream
*fs
)
85 struct rte_mbuf
*pkts_burst
[MAX_PKT_BURST
];
90 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
96 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
97 start_tsc
= rte_rdtsc();
100 /* Packets Receive */
101 nb_rx
= rte_eth_rx_burst(fs
->rx_port
, fs
->rx_queue
,
102 pkts_burst
, nb_pkt_per_burst
);
103 fs
->rx_packets
+= nb_rx
;
105 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
106 fs
->rx_burst_stats
.pkt_burst_spread
[nb_rx
]++;
109 nb_tx
= rte_eth_tx_burst(fs
->tx_port
, fs
->tx_queue
,
112 /* Retry if necessary */
113 if (unlikely(nb_tx
< nb_rx
) && fs
->retry_enabled
) {
115 while (nb_tx
< nb_rx
&& retry
++ < burst_tx_retry_num
) {
116 rte_delay_us(burst_tx_delay_time
);
117 nb_tx
+= rte_eth_tx_burst(fs
->tx_port
, fs
->tx_queue
,
118 &pkts_burst
[nb_tx
], nb_rx
- nb_tx
);
121 fs
->tx_packets
+= nb_tx
;
123 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
124 fs
->tx_burst_stats
.pkt_burst_spread
[nb_tx
]++;
127 if (unlikely(nb_tx
< nb_rx
)) {
128 fs
->fwd_dropped
+= (nb_rx
- nb_tx
);
130 rte_pktmbuf_free(pkts_burst
[nb_tx
]);
131 } while (++nb_tx
< nb_rx
);
133 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
134 end_tsc
= rte_rdtsc();
135 core_cycles
= (end_tsc
- start_tsc
);
136 fs
->core_cycles
= (uint64_t) (fs
->core_cycles
+ core_cycles
);
141 softnic_fwd_run(struct fwd_stream
*fs
)
143 rte_pmd_softnic_run(softnic_port_id
);
151 softnic_begin(void *arg __rte_unused
)
154 if (!softnic_fwd_lcore
->stopped
)
160 rte_pmd_softnic_run(softnic_port_id
);
161 } while (!softnic_fwd_lcore
->stopped
);
167 set_tm_hiearchy_nodes_shaper_rate(portid_t port_id
,
168 struct tm_hierarchy
*h
)
170 struct rte_eth_link link_params
;
171 uint64_t tm_port_rate
;
173 memset(&link_params
, 0, sizeof(link_params
));
175 rte_eth_link_get(port_id
, &link_params
);
176 tm_port_rate
= (uint64_t)ETH_SPEED_NUM_10G
* BYTES_IN_MBPS
;
178 /* Set tm hierarchy shapers rate */
179 h
->root_node_shaper_rate
= tm_port_rate
;
180 h
->subport_node_shaper_rate
=
181 tm_port_rate
/ SUBPORT_NODES_PER_PORT
;
182 h
->pipe_node_shaper_rate
183 = h
->subport_node_shaper_rate
/ PIPE_NODES_PER_SUBPORT
;
184 h
->tc_node_shaper_rate
= h
->pipe_node_shaper_rate
;
185 h
->tc_node_shared_shaper_rate
= h
->subport_node_shaper_rate
;
189 softport_tm_root_node_add(portid_t port_id
, struct tm_hierarchy
*h
,
190 struct rte_tm_error
*error
)
192 struct rte_tm_node_params rnp
;
193 struct rte_tm_shaper_params rsp
;
194 uint32_t priority
, weight
, level_id
, shaper_profile_id
;
196 memset(&rsp
, 0, sizeof(struct rte_tm_shaper_params
));
197 memset(&rnp
, 0, sizeof(struct rte_tm_node_params
));
199 /* Shaper profile Parameters */
200 rsp
.peak
.rate
= h
->root_node_shaper_rate
;
201 rsp
.peak
.size
= TOKEN_BUCKET_SIZE
;
202 rsp
.pkt_length_adjust
= RTE_TM_ETH_FRAMING_OVERHEAD_FCS
;
203 shaper_profile_id
= 0;
205 if (rte_tm_shaper_profile_add(port_id
, shaper_profile_id
,
207 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
208 __func__
, error
->type
, error
->message
,
213 /* Root Node Parameters */
214 h
->root_node_id
= ROOT_NODE_ID
;
217 level_id
= TM_NODE_LEVEL_PORT
;
218 rnp
.shaper_profile_id
= shaper_profile_id
;
219 rnp
.nonleaf
.n_sp_priorities
= 1;
220 rnp
.stats_mask
= STATS_MASK_DEFAULT
;
222 /* Add Node to TM Hierarchy */
223 if (rte_tm_node_add(port_id
, h
->root_node_id
, RTE_TM_NODE_ID_NULL
,
224 priority
, weight
, level_id
, &rnp
, error
)) {
225 printf("%s ERROR(%d)-%s!(node_id %u, parent_id %u, level %u)\n",
226 __func__
, error
->type
, error
->message
,
227 h
->root_node_id
, RTE_TM_NODE_ID_NULL
,
234 printf(" Root node added (Start id %u, Count %u, level %u)\n",
235 h
->root_node_id
, 1, level_id
);
241 softport_tm_subport_node_add(portid_t port_id
,
242 struct tm_hierarchy
*h
,
243 struct rte_tm_error
*error
)
245 uint32_t subport_parent_node_id
, subport_node_id
= 0;
246 struct rte_tm_node_params snp
;
247 struct rte_tm_shaper_params ssp
;
248 uint32_t priority
, weight
, level_id
, shaper_profile_id
;
251 memset(&ssp
, 0, sizeof(struct rte_tm_shaper_params
));
252 memset(&snp
, 0, sizeof(struct rte_tm_node_params
));
254 shaper_profile_id
= h
->n_shapers
;
256 /* Add Shaper Profile to TM Hierarchy */
257 for (i
= 0; i
< SUBPORT_NODES_PER_PORT
; i
++) {
258 ssp
.peak
.rate
= h
->subport_node_shaper_rate
;
259 ssp
.peak
.size
= TOKEN_BUCKET_SIZE
;
260 ssp
.pkt_length_adjust
= RTE_TM_ETH_FRAMING_OVERHEAD_FCS
;
262 if (rte_tm_shaper_profile_add(port_id
, shaper_profile_id
,
264 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
265 __func__
, error
->type
, error
->message
,
270 /* Node Parameters */
271 h
->subport_node_id
[i
] = SUBPORT_NODES_START_ID
+ i
;
272 subport_parent_node_id
= h
->root_node_id
;
275 level_id
= TM_NODE_LEVEL_SUBPORT
;
276 snp
.shaper_profile_id
= shaper_profile_id
;
277 snp
.nonleaf
.n_sp_priorities
= 1;
278 snp
.stats_mask
= STATS_MASK_DEFAULT
;
280 /* Add Node to TM Hiearchy */
281 if (rte_tm_node_add(port_id
,
282 h
->subport_node_id
[i
],
283 subport_parent_node_id
,
288 printf("%s ERROR(%d)-%s!(node %u,parent %u,level %u)\n",
292 h
->subport_node_id
[i
],
293 subport_parent_node_id
,
301 h
->n_shapers
= shaper_profile_id
;
303 printf(" Subport nodes added (Start id %u, Count %u, level %u)\n",
304 h
->subport_node_id
[0], SUBPORT_NODES_PER_PORT
, level_id
);
310 softport_tm_pipe_node_add(portid_t port_id
,
311 struct tm_hierarchy
*h
,
312 struct rte_tm_error
*error
)
314 uint32_t pipe_parent_node_id
;
315 struct rte_tm_node_params pnp
;
316 struct rte_tm_shaper_params psp
;
317 uint32_t priority
, weight
, level_id
, shaper_profile_id
;
320 memset(&psp
, 0, sizeof(struct rte_tm_shaper_params
));
321 memset(&pnp
, 0, sizeof(struct rte_tm_node_params
));
323 shaper_profile_id
= h
->n_shapers
;
325 /* Shaper Profile Parameters */
326 psp
.peak
.rate
= h
->pipe_node_shaper_rate
;
327 psp
.peak
.size
= TOKEN_BUCKET_SIZE
;
328 psp
.pkt_length_adjust
= RTE_TM_ETH_FRAMING_OVERHEAD_FCS
;
330 /* Pipe Node Parameters */
333 level_id
= TM_NODE_LEVEL_PIPE
;
334 pnp
.nonleaf
.n_sp_priorities
= 4;
335 pnp
.stats_mask
= STATS_MASK_DEFAULT
;
337 /* Add Shaper Profiles and Nodes to TM Hierarchy */
338 for (i
= 0; i
< SUBPORT_NODES_PER_PORT
; i
++) {
339 for (j
= 0; j
< PIPE_NODES_PER_SUBPORT
; j
++) {
340 if (rte_tm_shaper_profile_add(port_id
,
341 shaper_profile_id
, &psp
, error
)) {
342 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
343 __func__
, error
->type
, error
->message
,
347 pnp
.shaper_profile_id
= shaper_profile_id
;
348 pipe_parent_node_id
= h
->subport_node_id
[i
];
349 h
->pipe_node_id
[i
][j
] = PIPE_NODES_START_ID
+
350 (i
* PIPE_NODES_PER_SUBPORT
) + j
;
352 if (rte_tm_node_add(port_id
,
353 h
->pipe_node_id
[i
][j
],
355 priority
, weight
, level_id
,
358 printf("%s ERROR(%d)-%s!(node %u,parent %u )\n",
362 h
->pipe_node_id
[i
][j
],
363 pipe_parent_node_id
);
371 h
->n_shapers
= shaper_profile_id
;
373 printf(" Pipe nodes added (Start id %u, Count %u, level %u)\n",
374 h
->pipe_node_id
[0][0], NUM_PIPE_NODES
, level_id
);
380 softport_tm_tc_node_add(portid_t port_id
,
381 struct tm_hierarchy
*h
,
382 struct rte_tm_error
*error
)
384 uint32_t tc_parent_node_id
;
385 struct rte_tm_node_params tnp
;
386 struct rte_tm_shaper_params tsp
, tssp
;
387 uint32_t shared_shaper_profile_id
[TC_NODES_PER_PIPE
];
388 uint32_t priority
, weight
, level_id
, shaper_profile_id
;
389 uint32_t pos
, n_tc_nodes
, i
, j
, k
;
391 memset(&tsp
, 0, sizeof(struct rte_tm_shaper_params
));
392 memset(&tssp
, 0, sizeof(struct rte_tm_shaper_params
));
393 memset(&tnp
, 0, sizeof(struct rte_tm_node_params
));
395 shaper_profile_id
= h
->n_shapers
;
397 /* Private Shaper Profile (TC) Parameters */
398 tsp
.peak
.rate
= h
->tc_node_shaper_rate
;
399 tsp
.peak
.size
= TOKEN_BUCKET_SIZE
;
400 tsp
.pkt_length_adjust
= RTE_TM_ETH_FRAMING_OVERHEAD_FCS
;
402 /* Shared Shaper Profile (TC) Parameters */
403 tssp
.peak
.rate
= h
->tc_node_shared_shaper_rate
;
404 tssp
.peak
.size
= TOKEN_BUCKET_SIZE
;
405 tssp
.pkt_length_adjust
= RTE_TM_ETH_FRAMING_OVERHEAD_FCS
;
407 /* TC Node Parameters */
409 level_id
= TM_NODE_LEVEL_TC
;
410 tnp
.n_shared_shapers
= 1;
411 tnp
.nonleaf
.n_sp_priorities
= 1;
412 tnp
.stats_mask
= STATS_MASK_DEFAULT
;
414 /* Add Shared Shaper Profiles to TM Hierarchy */
415 for (i
= 0; i
< TC_NODES_PER_PIPE
; i
++) {
416 shared_shaper_profile_id
[i
] = shaper_profile_id
;
418 if (rte_tm_shaper_profile_add(port_id
,
419 shared_shaper_profile_id
[i
], &tssp
, error
)) {
420 printf("%s ERROR(%d)-%s!(Shared shaper profileid %u)\n",
421 __func__
, error
->type
, error
->message
,
422 shared_shaper_profile_id
[i
]);
426 if (rte_tm_shared_shaper_add_update(port_id
, i
,
427 shared_shaper_profile_id
[i
], error
)) {
428 printf("%s ERROR(%d)-%s!(Shared shaper id %u)\n",
429 __func__
, error
->type
, error
->message
, i
);
436 /* Add Shaper Profiles and Nodes to TM Hierarchy */
438 for (i
= 0; i
< SUBPORT_NODES_PER_PORT
; i
++) {
439 for (j
= 0; j
< PIPE_NODES_PER_SUBPORT
; j
++) {
440 for (k
= 0; k
< TC_NODES_PER_PIPE
; k
++) {
442 tc_parent_node_id
= h
->pipe_node_id
[i
][j
];
443 tnp
.shared_shaper_id
=
444 (uint32_t *)calloc(1, sizeof(uint32_t));
445 if (tnp
.shared_shaper_id
== NULL
) {
446 printf("Shared shaper mem alloc err\n");
449 tnp
.shared_shaper_id
[0] = k
;
450 pos
= j
+ (i
* PIPE_NODES_PER_SUBPORT
);
451 h
->tc_node_id
[pos
][k
] =
452 TC_NODES_START_ID
+ n_tc_nodes
;
454 if (rte_tm_shaper_profile_add(port_id
,
455 shaper_profile_id
, &tsp
, error
)) {
456 printf("%s ERROR(%d)-%s!(shaper %u)\n",
457 __func__
, error
->type
,
461 free(tnp
.shared_shaper_id
);
464 tnp
.shaper_profile_id
= shaper_profile_id
;
465 if (rte_tm_node_add(port_id
,
466 h
->tc_node_id
[pos
][k
],
471 printf("%s ERROR(%d)-%s!(node id %u)\n",
475 h
->tc_node_id
[pos
][k
]);
477 free(tnp
.shared_shaper_id
);
486 h
->n_shapers
= shaper_profile_id
;
488 printf(" TC nodes added (Start id %u, Count %u, level %u)\n",
489 h
->tc_node_id
[0][0], n_tc_nodes
, level_id
);
495 softport_tm_queue_node_add(portid_t port_id
, struct tm_hierarchy
*h
,
496 struct rte_tm_error
*error
)
498 uint32_t queue_parent_node_id
;
499 struct rte_tm_node_params qnp
;
500 uint32_t priority
, weight
, level_id
, pos
;
501 uint32_t n_queue_nodes
, i
, j
, k
;
503 memset(&qnp
, 0, sizeof(struct rte_tm_node_params
));
505 /* Queue Node Parameters */
508 level_id
= TM_NODE_LEVEL_QUEUE
;
509 qnp
.shaper_profile_id
= RTE_TM_SHAPER_PROFILE_ID_NONE
;
510 qnp
.leaf
.cman
= RTE_TM_CMAN_TAIL_DROP
;
511 qnp
.stats_mask
= STATS_MASK_QUEUE
;
513 /* Add Queue Nodes to TM Hierarchy */
515 for (i
= 0; i
< NUM_PIPE_NODES
; i
++) {
516 for (j
= 0; j
< TC_NODES_PER_PIPE
; j
++) {
517 queue_parent_node_id
= h
->tc_node_id
[i
][j
];
518 for (k
= 0; k
< QUEUE_NODES_PER_TC
; k
++) {
519 pos
= j
+ (i
* TC_NODES_PER_PIPE
);
520 h
->queue_node_id
[pos
][k
] = n_queue_nodes
;
521 if (rte_tm_node_add(port_id
,
522 h
->queue_node_id
[pos
][k
],
523 queue_parent_node_id
,
528 printf("%s ERROR(%d)-%s!(node %u)\n",
532 h
->queue_node_id
[pos
][k
]);
540 printf(" Queue nodes added (Start id %u, Count %u, level %u)\n",
541 h
->queue_node_id
[0][0], n_queue_nodes
, level_id
);
547 softport_tm_hierarchy_specify(portid_t port_id
,
548 struct rte_tm_error
*error
)
551 struct tm_hierarchy h
;
554 memset(&h
, 0, sizeof(struct tm_hierarchy
));
556 /* TM hierarchy shapers rate */
557 set_tm_hiearchy_nodes_shaper_rate(port_id
, &h
);
559 /* Add root node (level 0) */
560 status
= softport_tm_root_node_add(port_id
, &h
, error
);
564 /* Add subport node (level 1) */
565 status
= softport_tm_subport_node_add(port_id
, &h
, error
);
569 /* Add pipe nodes (level 2) */
570 status
= softport_tm_pipe_node_add(port_id
, &h
, error
);
574 /* Add traffic class nodes (level 3) */
575 status
= softport_tm_tc_node_add(port_id
, &h
, error
);
579 /* Add queue nodes (level 4) */
580 status
= softport_tm_queue_node_add(port_id
, &h
, error
);
588 * Softnic TM default configuration
591 softnic_tm_default_config(portid_t pi
)
593 struct rte_port
*port
= &ports
[pi
];
594 struct rte_tm_error error
;
598 rte_eth_dev_stop(pi
);
600 /* TM hierarchy specification */
601 status
= softport_tm_hierarchy_specify(pi
, &error
);
603 printf(" TM Hierarchy built error(%d) - %s\n",
604 error
.type
, error
.message
);
607 printf("\n TM Hierarchy Specified!\n");
609 /* TM hierarchy commit */
610 status
= rte_tm_hierarchy_commit(pi
, 0, &error
);
612 printf(" Hierarchy commit error(%d) - %s\n",
613 error
.type
, error
.message
);
616 printf(" Hierarchy Committed (port %u)!\n", pi
);
619 status
= rte_eth_dev_start(pi
);
621 printf("\n Port %u start error!\n", pi
);
625 /* Reset the default hierarchy flag */
626 port
->softport
.default_tm_hierarchy_enable
= 0;
630 * Softnic forwarding init
633 softnic_fwd_begin(portid_t pi
)
635 struct rte_port
*port
= &ports
[pi
];
636 uint32_t lcore
, fwd_core_present
= 0, softnic_run_launch
= 0;
639 softnic_fwd_lcore
= port
->softport
.fwd_lcore_arg
[0];
640 softnic_port_id
= pi
;
642 /* Launch softnic_run function on lcores */
643 for (lcore
= 0; lcore
< RTE_MAX_LCORE
; lcore
++) {
644 if (!rte_lcore_is_enabled(lcore
))
647 if (lcore
== rte_get_master_lcore())
650 if (fwd_core_present
== 0) {
655 status
= rte_eal_remote_launch(softnic_begin
, NULL
, lcore
);
657 printf("softnic launch on lcore %u failed (%d)\n",
660 softnic_run_launch
= 1;
663 if (!softnic_run_launch
)
664 softnic_fwd_engine
.packet_fwd
= softnic_fwd_run
;
666 /* Softnic TM default configuration */
667 if (port
->softport
.default_tm_hierarchy_enable
== 1)
668 softnic_tm_default_config(pi
);
671 struct fwd_engine softnic_fwd_engine
= {
672 .fwd_mode_name
= "softnic",
673 .port_fwd_begin
= softnic_fwd_begin
,
674 .port_fwd_end
= NULL
,
675 .packet_fwd
= softnic_fwd
,