4 * Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 #include <sys/types.h>
41 #include <sys/queue.h>
42 #include <netinet/in.h>
51 #include <rte_atomic.h>
52 #include <rte_branch_prediction.h>
53 #include <rte_common.h>
54 #include <rte_cryptodev.h>
55 #include <rte_cycles.h>
56 #include <rte_debug.h>
58 #include <rte_ether.h>
59 #include <rte_ethdev.h>
60 #include <rte_interrupts.h>
62 #include <rte_launch.h>
63 #include <rte_lcore.h>
65 #include <rte_malloc.h>
67 #include <rte_memcpy.h>
68 #include <rte_memory.h>
69 #include <rte_mempool.h>
70 #include <rte_memzone.h>
72 #include <rte_per_lcore.h>
73 #include <rte_prefetch.h>
74 #include <rte_random.h>
75 #include <rte_hexdump.h>
83 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
87 #define MAX_STR_LEN 32
88 #define MAX_KEY_SIZE 128
89 #define MAX_PKT_BURST 32
90 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
93 * Configurable number of RX/TX ring descriptors
95 #define RTE_TEST_RX_DESC_DEFAULT 128
96 #define RTE_TEST_TX_DESC_DEFAULT 512
98 static uint16_t nb_rxd
= RTE_TEST_RX_DESC_DEFAULT
;
99 static uint16_t nb_txd
= RTE_TEST_TX_DESC_DEFAULT
;
101 /* ethernet addresses of ports */
102 static struct ether_addr l2fwd_ports_eth_addr
[RTE_MAX_ETHPORTS
];
104 /* mask of enabled ports */
105 static uint64_t l2fwd_enabled_port_mask
;
106 static uint64_t l2fwd_enabled_crypto_mask
;
108 /* list of enabled ports */
109 static uint32_t l2fwd_dst_ports
[RTE_MAX_ETHPORTS
];
114 struct rte_mbuf
*buffer
[MAX_PKT_BURST
];
119 struct rte_crypto_op
*buffer
[MAX_PKT_BURST
];
122 #define MAX_RX_QUEUE_PER_LCORE 16
123 #define MAX_TX_QUEUE_PER_PORT 16
125 enum l2fwd_crypto_xform_chain
{
126 L2FWD_CRYPTO_CIPHER_HASH
,
127 L2FWD_CRYPTO_HASH_CIPHER
,
128 L2FWD_CRYPTO_CIPHER_ONLY
,
129 L2FWD_CRYPTO_HASH_ONLY
135 phys_addr_t phys_addr
;
138 char supported_auth_algo
[RTE_CRYPTO_AUTH_LIST_END
][MAX_STR_LEN
];
139 char supported_cipher_algo
[RTE_CRYPTO_CIPHER_LIST_END
][MAX_STR_LEN
];
141 /** l2fwd crypto application command line options */
142 struct l2fwd_crypto_options
{
144 unsigned nb_ports_per_lcore
;
145 unsigned refresh_period
;
146 unsigned single_lcore
:1;
149 unsigned sessionless
:1;
151 enum l2fwd_crypto_xform_chain xform_chain
;
153 struct rte_crypto_sym_xform cipher_xform
;
155 int ckey_random_size
;
161 struct rte_crypto_sym_xform auth_xform
;
163 int akey_random_size
;
165 struct l2fwd_key aad
;
172 char string_type
[MAX_STR_LEN
];
175 /** l2fwd crypto lcore params */
176 struct l2fwd_crypto_params
{
180 unsigned digest_length
;
184 struct l2fwd_key aad
;
185 struct rte_cryptodev_sym_session
*session
;
191 enum rte_crypto_cipher_algorithm cipher_algo
;
192 enum rte_crypto_auth_algorithm auth_algo
;
195 /** lcore configuration */
196 struct lcore_queue_conf
{
197 unsigned nb_rx_ports
;
198 unsigned rx_port_list
[MAX_RX_QUEUE_PER_LCORE
];
200 unsigned nb_crypto_devs
;
201 unsigned cryptodev_list
[MAX_RX_QUEUE_PER_LCORE
];
203 struct op_buffer op_buf
[RTE_MAX_ETHPORTS
];
204 struct pkt_buffer pkt_buf
[RTE_MAX_ETHPORTS
];
205 } __rte_cache_aligned
;
207 struct lcore_queue_conf lcore_queue_conf
[RTE_MAX_LCORE
];
209 static const struct rte_eth_conf port_conf
= {
211 .mq_mode
= ETH_MQ_RX_NONE
,
212 .max_rx_pkt_len
= ETHER_MAX_LEN
,
214 .header_split
= 0, /**< Header Split disabled */
215 .hw_ip_checksum
= 0, /**< IP checksum offload disabled */
216 .hw_vlan_filter
= 0, /**< VLAN filtering disabled */
217 .jumbo_frame
= 0, /**< Jumbo Frame Support disabled */
218 .hw_strip_crc
= 0, /**< CRC stripped by hardware */
221 .mq_mode
= ETH_MQ_TX_NONE
,
225 struct rte_mempool
*l2fwd_pktmbuf_pool
;
226 struct rte_mempool
*l2fwd_crypto_op_pool
;
228 /* Per-port statistics struct */
229 struct l2fwd_port_statistics
{
233 uint64_t crypto_enqueued
;
234 uint64_t crypto_dequeued
;
237 } __rte_cache_aligned
;
239 struct l2fwd_crypto_statistics
{
244 } __rte_cache_aligned
;
246 struct l2fwd_port_statistics port_statistics
[RTE_MAX_ETHPORTS
];
247 struct l2fwd_crypto_statistics crypto_statistics
[RTE_CRYPTO_MAX_DEVS
];
249 /* A tsc-based timer responsible for triggering statistics printout */
250 #define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */
251 #define MAX_TIMER_PERIOD 86400UL /* 1 day max */
253 /* default period is 10 seconds */
254 static int64_t timer_period
= 10 * TIMER_MILLISECOND
* 1000;
256 /* Print out statistics on packets dropped */
260 uint64_t total_packets_dropped
, total_packets_tx
, total_packets_rx
;
261 uint64_t total_packets_enqueued
, total_packets_dequeued
,
262 total_packets_errors
;
266 total_packets_dropped
= 0;
267 total_packets_tx
= 0;
268 total_packets_rx
= 0;
269 total_packets_enqueued
= 0;
270 total_packets_dequeued
= 0;
271 total_packets_errors
= 0;
273 const char clr
[] = { 27, '[', '2', 'J', '\0' };
274 const char topLeft
[] = { 27, '[', '1', ';', '1', 'H', '\0' };
276 /* Clear screen and move to top left */
277 printf("%s%s", clr
, topLeft
);
279 printf("\nPort statistics ====================================");
281 for (portid
= 0; portid
< RTE_MAX_ETHPORTS
; portid
++) {
282 /* skip disabled ports */
283 if ((l2fwd_enabled_port_mask
& (1 << portid
)) == 0)
285 printf("\nStatistics for port %u ------------------------------"
286 "\nPackets sent: %32"PRIu64
287 "\nPackets received: %28"PRIu64
288 "\nPackets dropped: %29"PRIu64
,
290 port_statistics
[portid
].tx
,
291 port_statistics
[portid
].rx
,
292 port_statistics
[portid
].dropped
);
294 total_packets_dropped
+= port_statistics
[portid
].dropped
;
295 total_packets_tx
+= port_statistics
[portid
].tx
;
296 total_packets_rx
+= port_statistics
[portid
].rx
;
298 printf("\nCrypto statistics ==================================");
300 for (cdevid
= 0; cdevid
< RTE_CRYPTO_MAX_DEVS
; cdevid
++) {
301 /* skip disabled ports */
302 if ((l2fwd_enabled_crypto_mask
& (1lu << cdevid
)) == 0)
304 printf("\nStatistics for cryptodev %"PRIu64
305 " -------------------------"
306 "\nPackets enqueued: %28"PRIu64
307 "\nPackets dequeued: %28"PRIu64
308 "\nPackets errors: %30"PRIu64
,
310 crypto_statistics
[cdevid
].enqueued
,
311 crypto_statistics
[cdevid
].dequeued
,
312 crypto_statistics
[cdevid
].errors
);
314 total_packets_enqueued
+= crypto_statistics
[cdevid
].enqueued
;
315 total_packets_dequeued
+= crypto_statistics
[cdevid
].dequeued
;
316 total_packets_errors
+= crypto_statistics
[cdevid
].errors
;
318 printf("\nAggregate statistics ==============================="
319 "\nTotal packets received: %22"PRIu64
320 "\nTotal packets enqueued: %22"PRIu64
321 "\nTotal packets dequeued: %22"PRIu64
322 "\nTotal packets sent: %26"PRIu64
323 "\nTotal packets dropped: %23"PRIu64
324 "\nTotal packets crypto errors: %17"PRIu64
,
326 total_packets_enqueued
,
327 total_packets_dequeued
,
329 total_packets_dropped
,
330 total_packets_errors
);
331 printf("\n====================================================\n");
335 fill_supported_algorithm_tables(void)
339 for (i
= 0; i
< RTE_CRYPTO_AUTH_LIST_END
; i
++)
340 strcpy(supported_auth_algo
[i
], "NOT_SUPPORTED");
342 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_AES_GCM
], "AES_GCM");
343 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_AES_GMAC
], "AES_GMAC");
344 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_MD5_HMAC
], "MD5_HMAC");
345 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_MD5
], "MD5");
346 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_NULL
], "NULL");
347 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_AES_XCBC_MAC
],
349 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SHA1_HMAC
], "SHA1_HMAC");
350 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SHA1
], "SHA1");
351 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SHA224_HMAC
], "SHA224_HMAC");
352 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SHA224
], "SHA224");
353 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SHA256_HMAC
], "SHA256_HMAC");
354 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SHA256
], "SHA256");
355 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SHA384_HMAC
], "SHA384_HMAC");
356 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SHA384
], "SHA384");
357 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SHA512_HMAC
], "SHA512_HMAC");
358 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SHA512
], "SHA512");
359 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_SNOW3G_UIA2
], "SNOW3G_UIA2");
360 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_ZUC_EIA3
], "ZUC_EIA3");
361 strcpy(supported_auth_algo
[RTE_CRYPTO_AUTH_KASUMI_F9
], "KASUMI_F9");
363 for (i
= 0; i
< RTE_CRYPTO_CIPHER_LIST_END
; i
++)
364 strcpy(supported_cipher_algo
[i
], "NOT_SUPPORTED");
366 strcpy(supported_cipher_algo
[RTE_CRYPTO_CIPHER_AES_CBC
], "AES_CBC");
367 strcpy(supported_cipher_algo
[RTE_CRYPTO_CIPHER_AES_CTR
], "AES_CTR");
368 strcpy(supported_cipher_algo
[RTE_CRYPTO_CIPHER_AES_GCM
], "AES_GCM");
369 strcpy(supported_cipher_algo
[RTE_CRYPTO_CIPHER_NULL
], "NULL");
370 strcpy(supported_cipher_algo
[RTE_CRYPTO_CIPHER_SNOW3G_UEA2
], "SNOW3G_UEA2");
371 strcpy(supported_cipher_algo
[RTE_CRYPTO_CIPHER_ZUC_EEA3
], "ZUC_EEA3");
372 strcpy(supported_cipher_algo
[RTE_CRYPTO_CIPHER_KASUMI_F8
], "KASUMI_F8");
373 strcpy(supported_cipher_algo
[RTE_CRYPTO_CIPHER_3DES_CTR
], "3DES_CTR");
374 strcpy(supported_cipher_algo
[RTE_CRYPTO_CIPHER_3DES_CBC
], "3DES_CBC");
379 l2fwd_crypto_send_burst(struct lcore_queue_conf
*qconf
, unsigned n
,
380 struct l2fwd_crypto_params
*cparams
)
382 struct rte_crypto_op
**op_buffer
;
385 op_buffer
= (struct rte_crypto_op
**)
386 qconf
->op_buf
[cparams
->dev_id
].buffer
;
388 ret
= rte_cryptodev_enqueue_burst(cparams
->dev_id
,
389 cparams
->qp_id
, op_buffer
, (uint16_t) n
);
391 crypto_statistics
[cparams
->dev_id
].enqueued
+= ret
;
392 if (unlikely(ret
< n
)) {
393 crypto_statistics
[cparams
->dev_id
].errors
+= (n
- ret
);
395 rte_pktmbuf_free(op_buffer
[ret
]->sym
->m_src
);
396 rte_crypto_op_free(op_buffer
[ret
]);
404 l2fwd_crypto_enqueue(struct rte_crypto_op
*op
,
405 struct l2fwd_crypto_params
*cparams
)
407 unsigned lcore_id
, len
;
408 struct lcore_queue_conf
*qconf
;
410 lcore_id
= rte_lcore_id();
412 qconf
= &lcore_queue_conf
[lcore_id
];
413 len
= qconf
->op_buf
[cparams
->dev_id
].len
;
414 qconf
->op_buf
[cparams
->dev_id
].buffer
[len
] = op
;
417 /* enough ops to be sent */
418 if (len
== MAX_PKT_BURST
) {
419 l2fwd_crypto_send_burst(qconf
, MAX_PKT_BURST
, cparams
);
423 qconf
->op_buf
[cparams
->dev_id
].len
= len
;
428 l2fwd_simple_crypto_enqueue(struct rte_mbuf
*m
,
429 struct rte_crypto_op
*op
,
430 struct l2fwd_crypto_params
*cparams
)
432 struct ether_hdr
*eth_hdr
;
433 struct ipv4_hdr
*ip_hdr
;
435 unsigned ipdata_offset
, pad_len
, data_len
;
438 eth_hdr
= rte_pktmbuf_mtod(m
, struct ether_hdr
*);
440 if (eth_hdr
->ether_type
!= rte_cpu_to_be_16(ETHER_TYPE_IPv4
))
443 ipdata_offset
= sizeof(struct ether_hdr
);
445 ip_hdr
= (struct ipv4_hdr
*)(rte_pktmbuf_mtod(m
, char *) +
448 ipdata_offset
+= (ip_hdr
->version_ihl
& IPV4_HDR_IHL_MASK
)
449 * IPV4_IHL_MULTIPLIER
;
452 /* Zero pad data to be crypto'd so it is block aligned */
453 data_len
= rte_pktmbuf_data_len(m
) - ipdata_offset
;
455 if (cparams
->do_hash
&& cparams
->hash_verify
)
456 data_len
-= cparams
->digest_length
;
458 pad_len
= data_len
% cparams
->block_size
? cparams
->block_size
-
459 (data_len
% cparams
->block_size
) : 0;
462 padding
= rte_pktmbuf_append(m
, pad_len
);
463 if (unlikely(!padding
))
467 memset(padding
, 0, pad_len
);
470 /* Set crypto operation data parameters */
471 rte_crypto_op_attach_sym_session(op
, cparams
->session
);
473 if (cparams
->do_hash
) {
474 if (!cparams
->hash_verify
) {
475 /* Append space for digest to end of packet */
476 op
->sym
->auth
.digest
.data
= (uint8_t *)rte_pktmbuf_append(m
,
477 cparams
->digest_length
);
479 op
->sym
->auth
.digest
.data
= rte_pktmbuf_mtod(m
,
480 uint8_t *) + ipdata_offset
+ data_len
;
483 op
->sym
->auth
.digest
.phys_addr
= rte_pktmbuf_mtophys_offset(m
,
484 rte_pktmbuf_pkt_len(m
) - cparams
->digest_length
);
485 op
->sym
->auth
.digest
.length
= cparams
->digest_length
;
487 /* For wireless algorithms, offset/length must be in bits */
488 if (cparams
->auth_algo
== RTE_CRYPTO_AUTH_SNOW3G_UIA2
||
489 cparams
->auth_algo
== RTE_CRYPTO_AUTH_KASUMI_F9
||
490 cparams
->auth_algo
== RTE_CRYPTO_AUTH_ZUC_EIA3
) {
491 op
->sym
->auth
.data
.offset
= ipdata_offset
<< 3;
492 op
->sym
->auth
.data
.length
= data_len
<< 3;
494 op
->sym
->auth
.data
.offset
= ipdata_offset
;
495 op
->sym
->auth
.data
.length
= data_len
;
498 if (cparams
->aad
.length
) {
499 op
->sym
->auth
.aad
.data
= cparams
->aad
.data
;
500 op
->sym
->auth
.aad
.phys_addr
= cparams
->aad
.phys_addr
;
501 op
->sym
->auth
.aad
.length
= cparams
->aad
.length
;
505 if (cparams
->do_cipher
) {
506 op
->sym
->cipher
.iv
.data
= cparams
->iv
.data
;
507 op
->sym
->cipher
.iv
.phys_addr
= cparams
->iv
.phys_addr
;
508 op
->sym
->cipher
.iv
.length
= cparams
->iv
.length
;
510 /* For wireless algorithms, offset/length must be in bits */
511 if (cparams
->cipher_algo
== RTE_CRYPTO_CIPHER_SNOW3G_UEA2
||
512 cparams
->cipher_algo
== RTE_CRYPTO_CIPHER_KASUMI_F8
||
513 cparams
->cipher_algo
== RTE_CRYPTO_CIPHER_ZUC_EEA3
) {
514 op
->sym
->cipher
.data
.offset
= ipdata_offset
<< 3;
515 op
->sym
->cipher
.data
.length
= data_len
<< 3;
517 op
->sym
->cipher
.data
.offset
= ipdata_offset
;
518 op
->sym
->cipher
.data
.length
= data_len
;
524 return l2fwd_crypto_enqueue(op
, cparams
);
528 /* Send the burst of packets on an output interface */
530 l2fwd_send_burst(struct lcore_queue_conf
*qconf
, unsigned n
,
533 struct rte_mbuf
**pkt_buffer
;
536 pkt_buffer
= (struct rte_mbuf
**)qconf
->pkt_buf
[port
].buffer
;
538 ret
= rte_eth_tx_burst(port
, 0, pkt_buffer
, (uint16_t)n
);
539 port_statistics
[port
].tx
+= ret
;
540 if (unlikely(ret
< n
)) {
541 port_statistics
[port
].dropped
+= (n
- ret
);
543 rte_pktmbuf_free(pkt_buffer
[ret
]);
550 /* Enqueue packets for TX and prepare them to be sent */
552 l2fwd_send_packet(struct rte_mbuf
*m
, uint8_t port
)
554 unsigned lcore_id
, len
;
555 struct lcore_queue_conf
*qconf
;
557 lcore_id
= rte_lcore_id();
559 qconf
= &lcore_queue_conf
[lcore_id
];
560 len
= qconf
->pkt_buf
[port
].len
;
561 qconf
->pkt_buf
[port
].buffer
[len
] = m
;
564 /* enough pkts to be sent */
565 if (unlikely(len
== MAX_PKT_BURST
)) {
566 l2fwd_send_burst(qconf
, MAX_PKT_BURST
, port
);
570 qconf
->pkt_buf
[port
].len
= len
;
575 l2fwd_simple_forward(struct rte_mbuf
*m
, unsigned portid
)
577 struct ether_hdr
*eth
;
581 dst_port
= l2fwd_dst_ports
[portid
];
582 eth
= rte_pktmbuf_mtod(m
, struct ether_hdr
*);
584 /* 02:00:00:00:00:xx */
585 tmp
= ð
->d_addr
.addr_bytes
[0];
586 *((uint64_t *)tmp
) = 0x000000000002 + ((uint64_t)dst_port
<< 40);
589 ether_addr_copy(&l2fwd_ports_eth_addr
[dst_port
], ð
->s_addr
);
591 l2fwd_send_packet(m
, (uint8_t) dst_port
);
594 /** Generate random key */
596 generate_random_key(uint8_t *key
, unsigned length
)
601 fd
= open("/dev/urandom", O_RDONLY
);
603 rte_exit(EXIT_FAILURE
, "Failed to generate random key\n");
605 ret
= read(fd
, key
, length
);
608 if (ret
!= (signed)length
)
609 rte_exit(EXIT_FAILURE
, "Failed to generate random key\n");
612 static struct rte_cryptodev_sym_session
*
613 initialize_crypto_session(struct l2fwd_crypto_options
*options
,
616 struct rte_crypto_sym_xform
*first_xform
;
618 if (options
->xform_chain
== L2FWD_CRYPTO_CIPHER_HASH
) {
619 first_xform
= &options
->cipher_xform
;
620 first_xform
->next
= &options
->auth_xform
;
621 } else if (options
->xform_chain
== L2FWD_CRYPTO_HASH_CIPHER
) {
622 first_xform
= &options
->auth_xform
;
623 first_xform
->next
= &options
->cipher_xform
;
624 } else if (options
->xform_chain
== L2FWD_CRYPTO_CIPHER_ONLY
) {
625 first_xform
= &options
->cipher_xform
;
627 first_xform
= &options
->auth_xform
;
630 /* Setup Cipher Parameters */
631 return rte_cryptodev_sym_session_create(cdev_id
, first_xform
);
635 l2fwd_crypto_options_print(struct l2fwd_crypto_options
*options
);
637 /* main processing loop */
639 l2fwd_main_loop(struct l2fwd_crypto_options
*options
)
641 struct rte_mbuf
*m
, *pkts_burst
[MAX_PKT_BURST
];
642 struct rte_crypto_op
*ops_burst
[MAX_PKT_BURST
];
644 unsigned lcore_id
= rte_lcore_id();
645 uint64_t prev_tsc
= 0, diff_tsc
, cur_tsc
, timer_tsc
= 0;
646 unsigned i
, j
, portid
, nb_rx
, len
;
647 struct lcore_queue_conf
*qconf
= &lcore_queue_conf
[lcore_id
];
648 const uint64_t drain_tsc
= (rte_get_tsc_hz() + US_PER_S
- 1) /
649 US_PER_S
* BURST_TX_DRAIN_US
;
650 struct l2fwd_crypto_params
*cparams
;
651 struct l2fwd_crypto_params port_cparams
[qconf
->nb_crypto_devs
];
653 if (qconf
->nb_rx_ports
== 0) {
654 RTE_LOG(INFO
, L2FWD
, "lcore %u has nothing to do\n", lcore_id
);
658 RTE_LOG(INFO
, L2FWD
, "entering main loop on lcore %u\n", lcore_id
);
660 for (i
= 0; i
< qconf
->nb_rx_ports
; i
++) {
662 portid
= qconf
->rx_port_list
[i
];
663 RTE_LOG(INFO
, L2FWD
, " -- lcoreid=%u portid=%u\n", lcore_id
,
667 for (i
= 0; i
< qconf
->nb_crypto_devs
; i
++) {
668 port_cparams
[i
].do_cipher
= 0;
669 port_cparams
[i
].do_hash
= 0;
671 switch (options
->xform_chain
) {
672 case L2FWD_CRYPTO_CIPHER_HASH
:
673 case L2FWD_CRYPTO_HASH_CIPHER
:
674 port_cparams
[i
].do_cipher
= 1;
675 port_cparams
[i
].do_hash
= 1;
677 case L2FWD_CRYPTO_HASH_ONLY
:
678 port_cparams
[i
].do_hash
= 1;
680 case L2FWD_CRYPTO_CIPHER_ONLY
:
681 port_cparams
[i
].do_cipher
= 1;
685 port_cparams
[i
].dev_id
= qconf
->cryptodev_list
[i
];
686 port_cparams
[i
].qp_id
= 0;
688 port_cparams
[i
].block_size
= options
->block_size
;
690 if (port_cparams
[i
].do_hash
) {
691 port_cparams
[i
].digest_length
=
692 options
->auth_xform
.auth
.digest_length
;
693 if (options
->auth_xform
.auth
.add_auth_data_length
) {
694 port_cparams
[i
].aad
.data
= options
->aad
.data
;
695 port_cparams
[i
].aad
.length
=
696 options
->auth_xform
.auth
.add_auth_data_length
;
697 port_cparams
[i
].aad
.phys_addr
= options
->aad
.phys_addr
;
698 if (!options
->aad_param
)
699 generate_random_key(port_cparams
[i
].aad
.data
,
700 port_cparams
[i
].aad
.length
);
704 if (options
->auth_xform
.auth
.op
== RTE_CRYPTO_AUTH_OP_VERIFY
)
705 port_cparams
[i
].hash_verify
= 1;
707 port_cparams
[i
].hash_verify
= 0;
709 port_cparams
[i
].auth_algo
= options
->auth_xform
.auth
.algo
;
712 if (port_cparams
[i
].do_cipher
) {
713 port_cparams
[i
].iv
.data
= options
->iv
.data
;
714 port_cparams
[i
].iv
.length
= options
->iv
.length
;
715 port_cparams
[i
].iv
.phys_addr
= options
->iv
.phys_addr
;
716 if (!options
->iv_param
)
717 generate_random_key(port_cparams
[i
].iv
.data
,
718 port_cparams
[i
].iv
.length
);
720 port_cparams
[i
].cipher_algo
= options
->cipher_xform
.cipher
.algo
;
723 port_cparams
[i
].session
= initialize_crypto_session(options
,
724 port_cparams
[i
].dev_id
);
726 if (port_cparams
[i
].session
== NULL
)
728 RTE_LOG(INFO
, L2FWD
, " -- lcoreid=%u cryptoid=%u\n", lcore_id
,
729 port_cparams
[i
].dev_id
);
732 l2fwd_crypto_options_print(options
);
735 * Initialize previous tsc timestamp before the loop,
736 * to avoid showing the port statistics immediately,
737 * so user can see the crypto information.
739 prev_tsc
= rte_rdtsc();
742 cur_tsc
= rte_rdtsc();
745 * Crypto device/TX burst queue drain
747 diff_tsc
= cur_tsc
- prev_tsc
;
748 if (unlikely(diff_tsc
> drain_tsc
)) {
749 /* Enqueue all crypto ops remaining in buffers */
750 for (i
= 0; i
< qconf
->nb_crypto_devs
; i
++) {
751 cparams
= &port_cparams
[i
];
752 len
= qconf
->op_buf
[cparams
->dev_id
].len
;
753 l2fwd_crypto_send_burst(qconf
, len
, cparams
);
754 qconf
->op_buf
[cparams
->dev_id
].len
= 0;
756 /* Transmit all packets remaining in buffers */
757 for (portid
= 0; portid
< RTE_MAX_ETHPORTS
; portid
++) {
758 if (qconf
->pkt_buf
[portid
].len
== 0)
760 l2fwd_send_burst(&lcore_queue_conf
[lcore_id
],
761 qconf
->pkt_buf
[portid
].len
,
763 qconf
->pkt_buf
[portid
].len
= 0;
766 /* if timer is enabled */
767 if (timer_period
> 0) {
769 /* advance the timer */
770 timer_tsc
+= diff_tsc
;
772 /* if timer has reached its timeout */
773 if (unlikely(timer_tsc
>=
774 (uint64_t)timer_period
)) {
776 /* do this only on master core */
777 if (lcore_id
== rte_get_master_lcore()
778 && options
->refresh_period
) {
789 * Read packet from RX queues
791 for (i
= 0; i
< qconf
->nb_rx_ports
; i
++) {
792 portid
= qconf
->rx_port_list
[i
];
794 cparams
= &port_cparams
[i
];
796 nb_rx
= rte_eth_rx_burst((uint8_t) portid
, 0,
797 pkts_burst
, MAX_PKT_BURST
);
799 port_statistics
[portid
].rx
+= nb_rx
;
803 * If we can't allocate a crypto_ops, then drop
804 * the rest of the burst and dequeue and
805 * process the packets to free offload structs
807 if (rte_crypto_op_bulk_alloc(
808 l2fwd_crypto_op_pool
,
809 RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
812 for (j
= 0; j
< nb_rx
; j
++)
813 rte_pktmbuf_free(pkts_burst
[i
]);
818 /* Enqueue packets from Crypto device*/
819 for (j
= 0; j
< nb_rx
; j
++) {
822 l2fwd_simple_crypto_enqueue(m
,
823 ops_burst
[j
], cparams
);
827 /* Dequeue packets from Crypto device */
829 nb_rx
= rte_cryptodev_dequeue_burst(
830 cparams
->dev_id
, cparams
->qp_id
,
831 ops_burst
, MAX_PKT_BURST
);
833 crypto_statistics
[cparams
->dev_id
].dequeued
+=
836 /* Forward crypto'd packets */
837 for (j
= 0; j
< nb_rx
; j
++) {
838 m
= ops_burst
[j
]->sym
->m_src
;
840 rte_crypto_op_free(ops_burst
[j
]);
841 l2fwd_simple_forward(m
, portid
);
843 } while (nb_rx
== MAX_PKT_BURST
);
849 l2fwd_launch_one_lcore(void *arg
)
851 l2fwd_main_loop((struct l2fwd_crypto_options
*)arg
);
855 /* Display command line arguments usage */
857 l2fwd_crypto_usage(const char *prgname
)
859 printf("%s [EAL options] --\n"
860 " -p PORTMASK: hexadecimal bitmask of ports to configure\n"
861 " -q NQ: number of queue (=ports) per lcore (default is 1)\n"
862 " -s manage all ports from single lcore\n"
863 " -T PERIOD: statistics will be refreshed each PERIOD seconds"
864 " (0 to disable, 10 default, 86400 maximum)\n"
866 " --cdev_type HW / SW / ANY\n"
867 " --chain HASH_CIPHER / CIPHER_HASH\n"
869 " --cipher_algo ALGO\n"
870 " --cipher_op ENCRYPT / DECRYPT\n"
871 " --cipher_key KEY (bytes separated with \":\")\n"
872 " --cipher_key_random_size SIZE: size of cipher key when generated randomly\n"
873 " --iv IV (bytes separated with \":\")\n"
874 " --iv_random_size SIZE: size of IV when generated randomly\n"
876 " --auth_algo ALGO\n"
877 " --auth_op GENERATE / VERIFY\n"
878 " --auth_key KEY (bytes separated with \":\")\n"
879 " --auth_key_random_size SIZE: size of auth key when generated randomly\n"
880 " --aad AAD (bytes separated with \":\")\n"
881 " --aad_random_size SIZE: size of AAD when generated randomly\n"
882 " --digest_size SIZE: size of digest to be generated/verified\n"
888 /** Parse crypto device type command line argument */
890 parse_cryptodev_type(enum cdev_type
*type
, char *optarg
)
892 if (strcmp("HW", optarg
) == 0) {
893 *type
= CDEV_TYPE_HW
;
895 } else if (strcmp("SW", optarg
) == 0) {
896 *type
= CDEV_TYPE_SW
;
898 } else if (strcmp("ANY", optarg
) == 0) {
899 *type
= CDEV_TYPE_ANY
;
906 /** Parse crypto chain xform command line argument */
908 parse_crypto_opt_chain(struct l2fwd_crypto_options
*options
, char *optarg
)
910 if (strcmp("CIPHER_HASH", optarg
) == 0) {
911 options
->xform_chain
= L2FWD_CRYPTO_CIPHER_HASH
;
913 } else if (strcmp("HASH_CIPHER", optarg
) == 0) {
914 options
->xform_chain
= L2FWD_CRYPTO_HASH_CIPHER
;
916 } else if (strcmp("CIPHER_ONLY", optarg
) == 0) {
917 options
->xform_chain
= L2FWD_CRYPTO_CIPHER_ONLY
;
919 } else if (strcmp("HASH_ONLY", optarg
) == 0) {
920 options
->xform_chain
= L2FWD_CRYPTO_HASH_ONLY
;
927 /** Parse crypto cipher algo option command line argument */
929 parse_cipher_algo(enum rte_crypto_cipher_algorithm
*algo
, char *optarg
)
933 for (i
= 0; i
< RTE_CRYPTO_CIPHER_LIST_END
; i
++) {
934 if (!strcmp(supported_cipher_algo
[i
], optarg
)) {
935 *algo
= (enum rte_crypto_cipher_algorithm
)i
;
940 printf("Cipher algorithm not supported!\n");
944 /** Parse crypto cipher operation command line argument */
946 parse_cipher_op(enum rte_crypto_cipher_operation
*op
, char *optarg
)
948 if (strcmp("ENCRYPT", optarg
) == 0) {
949 *op
= RTE_CRYPTO_CIPHER_OP_ENCRYPT
;
951 } else if (strcmp("DECRYPT", optarg
) == 0) {
952 *op
= RTE_CRYPTO_CIPHER_OP_DECRYPT
;
956 printf("Cipher operation not supported!\n");
960 /** Parse crypto key command line argument */
962 parse_key(uint8_t *data
, char *input_arg
)
967 for (byte_count
= 0, token
= strtok(input_arg
, ":");
968 (byte_count
< MAX_KEY_SIZE
) && (token
!= NULL
);
969 token
= strtok(NULL
, ":")) {
971 int number
= (int)strtol(token
, NULL
, 16);
973 if (errno
== EINVAL
|| errno
== ERANGE
|| number
> 0xFF)
976 data
[byte_count
++] = (uint8_t)number
;
982 /** Parse size param*/
984 parse_size(int *size
, const char *q_arg
)
989 /* parse hexadecimal string */
990 n
= strtoul(q_arg
, &end
, 10);
991 if ((q_arg
[0] == '\0') || (end
== NULL
) || (*end
!= '\0'))
995 printf("invalid size\n");
1003 /** Parse crypto cipher operation command line argument */
1005 parse_auth_algo(enum rte_crypto_auth_algorithm
*algo
, char *optarg
)
1009 for (i
= 0; i
< RTE_CRYPTO_AUTH_LIST_END
; i
++) {
1010 if (!strcmp(supported_auth_algo
[i
], optarg
)) {
1011 *algo
= (enum rte_crypto_auth_algorithm
)i
;
1016 printf("Authentication algorithm specified not supported!\n");
1021 parse_auth_op(enum rte_crypto_auth_operation
*op
, char *optarg
)
1023 if (strcmp("VERIFY", optarg
) == 0) {
1024 *op
= RTE_CRYPTO_AUTH_OP_VERIFY
;
1026 } else if (strcmp("GENERATE", optarg
) == 0) {
1027 *op
= RTE_CRYPTO_AUTH_OP_GENERATE
;
1031 printf("Authentication operation specified not supported!\n");
1035 /** Parse long options */
1037 l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options
*options
,
1038 struct option
*lgopts
, int option_index
)
1042 if (strcmp(lgopts
[option_index
].name
, "cdev_type") == 0) {
1043 retval
= parse_cryptodev_type(&options
->type
, optarg
);
1045 snprintf(options
->string_type
, MAX_STR_LEN
,
1050 else if (strcmp(lgopts
[option_index
].name
, "chain") == 0)
1051 return parse_crypto_opt_chain(options
, optarg
);
1053 /* Cipher options */
1054 else if (strcmp(lgopts
[option_index
].name
, "cipher_algo") == 0)
1055 return parse_cipher_algo(&options
->cipher_xform
.cipher
.algo
,
1058 else if (strcmp(lgopts
[option_index
].name
, "cipher_op") == 0)
1059 return parse_cipher_op(&options
->cipher_xform
.cipher
.op
,
1062 else if (strcmp(lgopts
[option_index
].name
, "cipher_key") == 0) {
1063 options
->ckey_param
= 1;
1064 options
->cipher_xform
.cipher
.key
.length
=
1065 parse_key(options
->cipher_xform
.cipher
.key
.data
, optarg
);
1066 if (options
->cipher_xform
.cipher
.key
.length
> 0)
1072 else if (strcmp(lgopts
[option_index
].name
, "cipher_key_random_size") == 0)
1073 return parse_size(&options
->ckey_random_size
, optarg
);
1075 else if (strcmp(lgopts
[option_index
].name
, "iv") == 0) {
1076 options
->iv_param
= 1;
1077 options
->iv
.length
=
1078 parse_key(options
->iv
.data
, optarg
);
1079 if (options
->iv
.length
> 0)
1085 else if (strcmp(lgopts
[option_index
].name
, "iv_random_size") == 0)
1086 return parse_size(&options
->iv_random_size
, optarg
);
1088 /* Authentication options */
1089 else if (strcmp(lgopts
[option_index
].name
, "auth_algo") == 0) {
1090 return parse_auth_algo(&options
->auth_xform
.auth
.algo
,
1094 else if (strcmp(lgopts
[option_index
].name
, "auth_op") == 0)
1095 return parse_auth_op(&options
->auth_xform
.auth
.op
,
1098 else if (strcmp(lgopts
[option_index
].name
, "auth_key") == 0) {
1099 options
->akey_param
= 1;
1100 options
->auth_xform
.auth
.key
.length
=
1101 parse_key(options
->auth_xform
.auth
.key
.data
, optarg
);
1102 if (options
->auth_xform
.auth
.key
.length
> 0)
1108 else if (strcmp(lgopts
[option_index
].name
, "auth_key_random_size") == 0) {
1109 return parse_size(&options
->akey_random_size
, optarg
);
1112 else if (strcmp(lgopts
[option_index
].name
, "aad") == 0) {
1113 options
->aad_param
= 1;
1114 options
->aad
.length
=
1115 parse_key(options
->aad
.data
, optarg
);
1116 if (options
->aad
.length
> 0)
1122 else if (strcmp(lgopts
[option_index
].name
, "aad_random_size") == 0) {
1123 return parse_size(&options
->aad_random_size
, optarg
);
1126 else if (strcmp(lgopts
[option_index
].name
, "digest_size") == 0) {
1127 return parse_size(&options
->digest_size
, optarg
);
1130 else if (strcmp(lgopts
[option_index
].name
, "sessionless") == 0) {
1131 options
->sessionless
= 1;
1138 /** Parse port mask */
1140 l2fwd_crypto_parse_portmask(struct l2fwd_crypto_options
*options
,
1146 /* parse hexadecimal string */
1147 pm
= strtoul(q_arg
, &end
, 16);
1148 if ((pm
== '\0') || (end
== NULL
) || (*end
!= '\0'))
1151 options
->portmask
= pm
;
1152 if (options
->portmask
== 0) {
1153 printf("invalid portmask specified\n");
1160 /** Parse number of queues */
1162 l2fwd_crypto_parse_nqueue(struct l2fwd_crypto_options
*options
,
1168 /* parse hexadecimal string */
1169 n
= strtoul(q_arg
, &end
, 10);
1170 if ((q_arg
[0] == '\0') || (end
== NULL
) || (*end
!= '\0'))
1172 else if (n
>= MAX_RX_QUEUE_PER_LCORE
)
1175 options
->nb_ports_per_lcore
= n
;
1176 if (options
->nb_ports_per_lcore
== 0) {
1177 printf("invalid number of ports selected\n");
1184 /** Parse timer period */
1186 l2fwd_crypto_parse_timer_period(struct l2fwd_crypto_options
*options
,
1192 /* parse number string */
1193 n
= (unsigned)strtol(q_arg
, &end
, 10);
1194 if ((q_arg
[0] == '\0') || (end
== NULL
) || (*end
!= '\0'))
1197 if (n
>= MAX_TIMER_PERIOD
) {
1198 printf("Warning refresh period specified %lu is greater than "
1199 "max value %lu! using max value",
1200 n
, MAX_TIMER_PERIOD
);
1201 n
= MAX_TIMER_PERIOD
;
1204 options
->refresh_period
= n
* 1000 * TIMER_MILLISECOND
;
1209 /** Generate default options for application */
1211 l2fwd_crypto_default_options(struct l2fwd_crypto_options
*options
)
1213 options
->portmask
= 0xffffffff;
1214 options
->nb_ports_per_lcore
= 1;
1215 options
->refresh_period
= 10000;
1216 options
->single_lcore
= 0;
1217 options
->sessionless
= 0;
1219 options
->xform_chain
= L2FWD_CRYPTO_CIPHER_HASH
;
1222 options
->cipher_xform
.type
= RTE_CRYPTO_SYM_XFORM_CIPHER
;
1223 options
->cipher_xform
.next
= NULL
;
1224 options
->ckey_param
= 0;
1225 options
->ckey_random_size
= -1;
1226 options
->cipher_xform
.cipher
.key
.length
= 0;
1227 options
->iv_param
= 0;
1228 options
->iv_random_size
= -1;
1229 options
->iv
.length
= 0;
1231 options
->cipher_xform
.cipher
.algo
= RTE_CRYPTO_CIPHER_AES_CBC
;
1232 options
->cipher_xform
.cipher
.op
= RTE_CRYPTO_CIPHER_OP_ENCRYPT
;
1234 /* Authentication Data */
1235 options
->auth_xform
.type
= RTE_CRYPTO_SYM_XFORM_AUTH
;
1236 options
->auth_xform
.next
= NULL
;
1237 options
->akey_param
= 0;
1238 options
->akey_random_size
= -1;
1239 options
->auth_xform
.auth
.key
.length
= 0;
1240 options
->aad_param
= 0;
1241 options
->aad_random_size
= -1;
1242 options
->aad
.length
= 0;
1243 options
->digest_size
= -1;
1245 options
->auth_xform
.auth
.algo
= RTE_CRYPTO_AUTH_SHA1_HMAC
;
1246 options
->auth_xform
.auth
.op
= RTE_CRYPTO_AUTH_OP_GENERATE
;
1248 options
->type
= CDEV_TYPE_ANY
;
1252 display_cipher_info(struct l2fwd_crypto_options
*options
)
1254 printf("\n---- Cipher information ---\n");
1255 printf("Algorithm: %s\n",
1256 supported_cipher_algo
[options
->cipher_xform
.cipher
.algo
]);
1257 rte_hexdump(stdout
, "Cipher key:",
1258 options
->cipher_xform
.cipher
.key
.data
,
1259 options
->cipher_xform
.cipher
.key
.length
);
1260 rte_hexdump(stdout
, "IV:", options
->iv
.data
, options
->iv
.length
);
1264 display_auth_info(struct l2fwd_crypto_options
*options
)
1266 printf("\n---- Authentication information ---\n");
1267 printf("Algorithm: %s\n",
1268 supported_auth_algo
[options
->auth_xform
.auth
.algo
]);
1269 rte_hexdump(stdout
, "Auth key:",
1270 options
->auth_xform
.auth
.key
.data
,
1271 options
->auth_xform
.auth
.key
.length
);
1272 rte_hexdump(stdout
, "AAD:", options
->aad
.data
, options
->aad
.length
);
1276 l2fwd_crypto_options_print(struct l2fwd_crypto_options
*options
)
1278 char string_cipher_op
[MAX_STR_LEN
];
1279 char string_auth_op
[MAX_STR_LEN
];
1281 if (options
->cipher_xform
.cipher
.op
== RTE_CRYPTO_CIPHER_OP_ENCRYPT
)
1282 strcpy(string_cipher_op
, "Encrypt");
1284 strcpy(string_cipher_op
, "Decrypt");
1286 if (options
->auth_xform
.auth
.op
== RTE_CRYPTO_AUTH_OP_GENERATE
)
1287 strcpy(string_auth_op
, "Auth generate");
1289 strcpy(string_auth_op
, "Auth verify");
1291 printf("Options:-\nn");
1292 printf("portmask: %x\n", options
->portmask
);
1293 printf("ports per lcore: %u\n", options
->nb_ports_per_lcore
);
1294 printf("refresh period : %u\n", options
->refresh_period
);
1295 printf("single lcore mode: %s\n",
1296 options
->single_lcore
? "enabled" : "disabled");
1297 printf("stats_printing: %s\n",
1298 options
->refresh_period
== 0 ? "disabled" : "enabled");
1300 printf("sessionless crypto: %s\n",
1301 options
->sessionless
? "enabled" : "disabled");
1303 if (options
->ckey_param
&& (options
->ckey_random_size
!= -1))
1304 printf("Cipher key already parsed, ignoring size of random key\n");
1306 if (options
->akey_param
&& (options
->akey_random_size
!= -1))
1307 printf("Auth key already parsed, ignoring size of random key\n");
1309 if (options
->iv_param
&& (options
->iv_random_size
!= -1))
1310 printf("IV already parsed, ignoring size of random IV\n");
1312 if (options
->aad_param
&& (options
->aad_random_size
!= -1))
1313 printf("AAD already parsed, ignoring size of random AAD\n");
1315 printf("\nCrypto chain: ");
1316 switch (options
->xform_chain
) {
1317 case L2FWD_CRYPTO_CIPHER_HASH
:
1318 printf("Input --> %s --> %s --> Output\n",
1319 string_cipher_op
, string_auth_op
);
1320 display_cipher_info(options
);
1321 display_auth_info(options
);
1323 case L2FWD_CRYPTO_HASH_CIPHER
:
1324 printf("Input --> %s --> %s --> Output\n",
1325 string_auth_op
, string_cipher_op
);
1326 display_cipher_info(options
);
1327 display_auth_info(options
);
1329 case L2FWD_CRYPTO_HASH_ONLY
:
1330 printf("Input --> %s --> Output\n", string_auth_op
);
1331 display_auth_info(options
);
1333 case L2FWD_CRYPTO_CIPHER_ONLY
:
1334 printf("Input --> %s --> Output\n", string_cipher_op
);
1335 display_cipher_info(options
);
1340 /* Parse the argument given in the command line of the application */
1342 l2fwd_crypto_parse_args(struct l2fwd_crypto_options
*options
,
1343 int argc
, char **argv
)
1345 int opt
, retval
, option_index
;
1346 char **argvopt
= argv
, *prgname
= argv
[0];
1348 static struct option lgopts
[] = {
1349 { "sessionless", no_argument
, 0, 0 },
1351 { "cdev_type", required_argument
, 0, 0 },
1352 { "chain", required_argument
, 0, 0 },
1354 { "cipher_algo", required_argument
, 0, 0 },
1355 { "cipher_op", required_argument
, 0, 0 },
1356 { "cipher_key", required_argument
, 0, 0 },
1357 { "cipher_key_random_size", required_argument
, 0, 0 },
1359 { "auth_algo", required_argument
, 0, 0 },
1360 { "auth_op", required_argument
, 0, 0 },
1361 { "auth_key", required_argument
, 0, 0 },
1362 { "auth_key_random_size", required_argument
, 0, 0 },
1364 { "iv", required_argument
, 0, 0 },
1365 { "iv_random_size", required_argument
, 0, 0 },
1366 { "aad", required_argument
, 0, 0 },
1367 { "aad_random_size", required_argument
, 0, 0 },
1368 { "digest_size", required_argument
, 0, 0 },
1370 { "sessionless", no_argument
, 0, 0 },
1375 l2fwd_crypto_default_options(options
);
1377 while ((opt
= getopt_long(argc
, argvopt
, "p:q:st:", lgopts
,
1378 &option_index
)) != EOF
) {
1382 retval
= l2fwd_crypto_parse_args_long_options(options
,
1383 lgopts
, option_index
);
1385 l2fwd_crypto_usage(prgname
);
1392 retval
= l2fwd_crypto_parse_portmask(options
, optarg
);
1394 l2fwd_crypto_usage(prgname
);
1401 retval
= l2fwd_crypto_parse_nqueue(options
, optarg
);
1403 l2fwd_crypto_usage(prgname
);
1410 options
->single_lcore
= 1;
1416 retval
= l2fwd_crypto_parse_timer_period(options
,
1419 l2fwd_crypto_usage(prgname
);
1425 l2fwd_crypto_usage(prgname
);
1432 argv
[optind
-1] = prgname
;
1435 optind
= 0; /* reset getopt lib */
1440 /* Check the link status of all ports in up to 9s, and print them finally */
1442 check_all_ports_link_status(uint8_t port_num
, uint32_t port_mask
)
1444 #define CHECK_INTERVAL 100 /* 100ms */
1445 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1446 uint8_t portid
, count
, all_ports_up
, print_flag
= 0;
1447 struct rte_eth_link link
;
1449 printf("\nChecking link status");
1451 for (count
= 0; count
<= MAX_CHECK_TIME
; count
++) {
1453 for (portid
= 0; portid
< port_num
; portid
++) {
1454 if ((port_mask
& (1 << portid
)) == 0)
1456 memset(&link
, 0, sizeof(link
));
1457 rte_eth_link_get_nowait(portid
, &link
);
1458 /* print link status if flag set */
1459 if (print_flag
== 1) {
1460 if (link
.link_status
)
1461 printf("Port %d Link Up - speed %u "
1462 "Mbps - %s\n", (uint8_t)portid
,
1463 (unsigned)link
.link_speed
,
1464 (link
.link_duplex
== ETH_LINK_FULL_DUPLEX
) ?
1465 ("full-duplex") : ("half-duplex\n"));
1467 printf("Port %d Link Down\n",
1471 /* clear all_ports_up flag if any link down */
1472 if (link
.link_status
== ETH_LINK_DOWN
) {
1477 /* after finally printing all link status, get out */
1478 if (print_flag
== 1)
1481 if (all_ports_up
== 0) {
1484 rte_delay_ms(CHECK_INTERVAL
);
1487 /* set the print_flag if all ports up or timeout */
1488 if (all_ports_up
== 1 || count
== (MAX_CHECK_TIME
- 1)) {
1495 /* Check if device has to be HW/SW or any */
1497 check_type(struct l2fwd_crypto_options
*options
, struct rte_cryptodev_info
*dev_info
)
1499 if (options
->type
== CDEV_TYPE_HW
&&
1500 (dev_info
->feature_flags
& RTE_CRYPTODEV_FF_HW_ACCELERATED
))
1502 if (options
->type
== CDEV_TYPE_SW
&&
1503 !(dev_info
->feature_flags
& RTE_CRYPTODEV_FF_HW_ACCELERATED
))
1505 if (options
->type
== CDEV_TYPE_ANY
)
1512 check_supported_size(uint16_t length
, uint16_t min
, uint16_t max
,
1518 if (increment
== 0) {
1525 /* Range of values */
1526 for (supp_size
= min
; supp_size
<= max
; supp_size
+= increment
) {
1527 if (length
== supp_size
)
1534 initialize_cryptodevs(struct l2fwd_crypto_options
*options
, unsigned nb_ports
,
1535 uint8_t *enabled_cdevs
)
1537 unsigned i
, cdev_id
, cdev_count
, enabled_cdev_count
= 0;
1538 const struct rte_cryptodev_capabilities
*cap
;
1539 enum rte_crypto_auth_algorithm cap_auth_algo
;
1540 enum rte_crypto_auth_algorithm opt_auth_algo
;
1541 enum rte_crypto_cipher_algorithm cap_cipher_algo
;
1542 enum rte_crypto_cipher_algorithm opt_cipher_algo
;
1545 cdev_count
= rte_cryptodev_count();
1546 if (cdev_count
== 0) {
1547 printf("No crypto devices available\n");
1551 for (cdev_id
= 0; cdev_id
< cdev_count
&& enabled_cdev_count
< nb_ports
;
1553 struct rte_cryptodev_qp_conf qp_conf
;
1554 struct rte_cryptodev_info dev_info
;
1556 struct rte_cryptodev_config conf
= {
1557 .nb_queue_pairs
= 1,
1558 .socket_id
= SOCKET_ID_ANY
,
1565 rte_cryptodev_info_get(cdev_id
, &dev_info
);
1567 /* Set cipher parameters */
1568 if (options
->xform_chain
== L2FWD_CRYPTO_CIPHER_HASH
||
1569 options
->xform_chain
== L2FWD_CRYPTO_HASH_CIPHER
||
1570 options
->xform_chain
== L2FWD_CRYPTO_CIPHER_ONLY
) {
1571 /* Check if device supports cipher algo */
1573 opt_cipher_algo
= options
->cipher_xform
.cipher
.algo
;
1574 cap
= &dev_info
.capabilities
[i
];
1575 while (cap
->op
!= RTE_CRYPTO_OP_TYPE_UNDEFINED
) {
1576 cap_cipher_algo
= cap
->sym
.cipher
.algo
;
1577 if (cap
->sym
.xform_type
==
1578 RTE_CRYPTO_SYM_XFORM_CIPHER
) {
1579 if (cap_cipher_algo
== opt_cipher_algo
) {
1580 if (check_type(options
, &dev_info
) == 0)
1584 cap
= &dev_info
.capabilities
[++i
];
1587 if (cap
->op
== RTE_CRYPTO_OP_TYPE_UNDEFINED
) {
1588 printf("Algorithm %s not supported by cryptodev %u"
1589 " or device not of preferred type (%s)\n",
1590 supported_cipher_algo
[opt_cipher_algo
],
1592 options
->string_type
);
1596 options
->block_size
= cap
->sym
.cipher
.block_size
;
1598 * Check if length of provided IV is supported
1599 * by the algorithm chosen.
1601 if (options
->iv_param
) {
1602 if (check_supported_size(options
->iv
.length
,
1603 cap
->sym
.cipher
.iv_size
.min
,
1604 cap
->sym
.cipher
.iv_size
.max
,
1605 cap
->sym
.cipher
.iv_size
.increment
)
1607 printf("Unsupported IV length\n");
1611 * Check if length of IV to be randomly generated
1612 * is supported by the algorithm chosen.
1614 } else if (options
->iv_random_size
!= -1) {
1615 if (check_supported_size(options
->iv_random_size
,
1616 cap
->sym
.cipher
.iv_size
.min
,
1617 cap
->sym
.cipher
.iv_size
.max
,
1618 cap
->sym
.cipher
.iv_size
.increment
)
1620 printf("Unsupported IV length\n");
1623 options
->iv
.length
= options
->iv_random_size
;
1624 /* No size provided, use minimum size. */
1626 options
->iv
.length
= cap
->sym
.cipher
.iv_size
.min
;
1629 * Check if length of provided cipher key is supported
1630 * by the algorithm chosen.
1632 if (options
->ckey_param
) {
1633 if (check_supported_size(
1634 options
->cipher_xform
.cipher
.key
.length
,
1635 cap
->sym
.cipher
.key_size
.min
,
1636 cap
->sym
.cipher
.key_size
.max
,
1637 cap
->sym
.cipher
.key_size
.increment
)
1639 printf("Unsupported cipher key length\n");
1643 * Check if length of the cipher key to be randomly generated
1644 * is supported by the algorithm chosen.
1646 } else if (options
->ckey_random_size
!= -1) {
1647 if (check_supported_size(options
->ckey_random_size
,
1648 cap
->sym
.cipher
.key_size
.min
,
1649 cap
->sym
.cipher
.key_size
.max
,
1650 cap
->sym
.cipher
.key_size
.increment
)
1652 printf("Unsupported cipher key length\n");
1655 options
->cipher_xform
.cipher
.key
.length
=
1656 options
->ckey_random_size
;
1657 /* No size provided, use minimum size. */
1659 options
->cipher_xform
.cipher
.key
.length
=
1660 cap
->sym
.cipher
.key_size
.min
;
1662 if (!options
->ckey_param
)
1663 generate_random_key(
1664 options
->cipher_xform
.cipher
.key
.data
,
1665 options
->cipher_xform
.cipher
.key
.length
);
1669 /* Set auth parameters */
1670 if (options
->xform_chain
== L2FWD_CRYPTO_CIPHER_HASH
||
1671 options
->xform_chain
== L2FWD_CRYPTO_HASH_CIPHER
||
1672 options
->xform_chain
== L2FWD_CRYPTO_HASH_ONLY
) {
1673 /* Check if device supports auth algo */
1675 opt_auth_algo
= options
->auth_xform
.auth
.algo
;
1676 cap
= &dev_info
.capabilities
[i
];
1677 while (cap
->op
!= RTE_CRYPTO_OP_TYPE_UNDEFINED
) {
1678 cap_auth_algo
= cap
->sym
.auth
.algo
;
1679 if ((cap
->sym
.xform_type
== RTE_CRYPTO_SYM_XFORM_AUTH
) &&
1680 (cap_auth_algo
== opt_auth_algo
) &&
1681 (check_type(options
, &dev_info
) == 0)) {
1684 cap
= &dev_info
.capabilities
[++i
];
1687 if (cap
->op
== RTE_CRYPTO_OP_TYPE_UNDEFINED
) {
1688 printf("Algorithm %s not supported by cryptodev %u"
1689 " or device not of preferred type (%s)\n",
1690 supported_auth_algo
[opt_auth_algo
],
1692 options
->string_type
);
1696 options
->block_size
= cap
->sym
.auth
.block_size
;
1698 * Check if length of provided AAD is supported
1699 * by the algorithm chosen.
1701 if (options
->aad_param
) {
1702 if (check_supported_size(options
->aad
.length
,
1703 cap
->sym
.auth
.aad_size
.min
,
1704 cap
->sym
.auth
.aad_size
.max
,
1705 cap
->sym
.auth
.aad_size
.increment
)
1707 printf("Unsupported AAD length\n");
1711 * Check if length of AAD to be randomly generated
1712 * is supported by the algorithm chosen.
1714 } else if (options
->aad_random_size
!= -1) {
1715 if (check_supported_size(options
->aad_random_size
,
1716 cap
->sym
.auth
.aad_size
.min
,
1717 cap
->sym
.auth
.aad_size
.max
,
1718 cap
->sym
.auth
.aad_size
.increment
)
1720 printf("Unsupported AAD length\n");
1723 options
->aad
.length
= options
->aad_random_size
;
1724 /* No size provided, use minimum size. */
1726 options
->aad
.length
= cap
->sym
.auth
.aad_size
.min
;
1728 options
->auth_xform
.auth
.add_auth_data_length
=
1729 options
->aad
.length
;
1732 * Check if length of provided auth key is supported
1733 * by the algorithm chosen.
1735 if (options
->akey_param
) {
1736 if (check_supported_size(
1737 options
->auth_xform
.auth
.key
.length
,
1738 cap
->sym
.auth
.key_size
.min
,
1739 cap
->sym
.auth
.key_size
.max
,
1740 cap
->sym
.auth
.key_size
.increment
)
1742 printf("Unsupported auth key length\n");
1746 * Check if length of the auth key to be randomly generated
1747 * is supported by the algorithm chosen.
1749 } else if (options
->akey_random_size
!= -1) {
1750 if (check_supported_size(options
->akey_random_size
,
1751 cap
->sym
.auth
.key_size
.min
,
1752 cap
->sym
.auth
.key_size
.max
,
1753 cap
->sym
.auth
.key_size
.increment
)
1755 printf("Unsupported auth key length\n");
1758 options
->auth_xform
.auth
.key
.length
=
1759 options
->akey_random_size
;
1760 /* No size provided, use minimum size. */
1762 options
->auth_xform
.auth
.key
.length
=
1763 cap
->sym
.auth
.key_size
.min
;
1765 if (!options
->akey_param
)
1766 generate_random_key(
1767 options
->auth_xform
.auth
.key
.data
,
1768 options
->auth_xform
.auth
.key
.length
);
1770 /* Check if digest size is supported by the algorithm. */
1771 if (options
->digest_size
!= -1) {
1772 if (check_supported_size(options
->digest_size
,
1773 cap
->sym
.auth
.digest_size
.min
,
1774 cap
->sym
.auth
.digest_size
.max
,
1775 cap
->sym
.auth
.digest_size
.increment
)
1777 printf("Unsupported digest length\n");
1780 options
->auth_xform
.auth
.digest_length
=
1781 options
->digest_size
;
1782 /* No size provided, use minimum size. */
1784 options
->auth_xform
.auth
.digest_length
=
1785 cap
->sym
.auth
.digest_size
.min
;
1788 retval
= rte_cryptodev_configure(cdev_id
, &conf
);
1790 printf("Failed to configure cryptodev %u", cdev_id
);
1794 qp_conf
.nb_descriptors
= 2048;
1796 retval
= rte_cryptodev_queue_pair_setup(cdev_id
, 0, &qp_conf
,
1799 printf("Failed to setup queue pair %u on cryptodev %u",
1804 retval
= rte_cryptodev_start(cdev_id
);
1806 printf("Failed to start device %u: error %d\n",
1811 l2fwd_enabled_crypto_mask
|= (1 << cdev_id
);
1813 enabled_cdevs
[cdev_id
] = 1;
1814 enabled_cdev_count
++;
1817 return enabled_cdev_count
;
1821 initialize_ports(struct l2fwd_crypto_options
*options
)
1823 uint8_t last_portid
, portid
;
1824 unsigned enabled_portcount
= 0;
1825 unsigned nb_ports
= rte_eth_dev_count();
1827 if (nb_ports
== 0) {
1828 printf("No Ethernet ports - bye\n");
1832 /* Reset l2fwd_dst_ports */
1833 for (portid
= 0; portid
< RTE_MAX_ETHPORTS
; portid
++)
1834 l2fwd_dst_ports
[portid
] = 0;
1836 for (last_portid
= 0, portid
= 0; portid
< nb_ports
; portid
++) {
1839 /* Skip ports that are not enabled */
1840 if ((options
->portmask
& (1 << portid
)) == 0)
1844 printf("Initializing port %u... ", (unsigned) portid
);
1846 retval
= rte_eth_dev_configure(portid
, 1, 1, &port_conf
);
1848 printf("Cannot configure device: err=%d, port=%u\n",
1849 retval
, (unsigned) portid
);
1853 /* init one RX queue */
1855 retval
= rte_eth_rx_queue_setup(portid
, 0, nb_rxd
,
1856 rte_eth_dev_socket_id(portid
),
1857 NULL
, l2fwd_pktmbuf_pool
);
1859 printf("rte_eth_rx_queue_setup:err=%d, port=%u\n",
1860 retval
, (unsigned) portid
);
1864 /* init one TX queue on each port */
1866 retval
= rte_eth_tx_queue_setup(portid
, 0, nb_txd
,
1867 rte_eth_dev_socket_id(portid
),
1870 printf("rte_eth_tx_queue_setup:err=%d, port=%u\n",
1871 retval
, (unsigned) portid
);
1877 retval
= rte_eth_dev_start(portid
);
1879 printf("rte_eth_dev_start:err=%d, port=%u\n",
1880 retval
, (unsigned) portid
);
1884 rte_eth_promiscuous_enable(portid
);
1886 rte_eth_macaddr_get(portid
, &l2fwd_ports_eth_addr
[portid
]);
1888 printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
1890 l2fwd_ports_eth_addr
[portid
].addr_bytes
[0],
1891 l2fwd_ports_eth_addr
[portid
].addr_bytes
[1],
1892 l2fwd_ports_eth_addr
[portid
].addr_bytes
[2],
1893 l2fwd_ports_eth_addr
[portid
].addr_bytes
[3],
1894 l2fwd_ports_eth_addr
[portid
].addr_bytes
[4],
1895 l2fwd_ports_eth_addr
[portid
].addr_bytes
[5]);
1897 /* initialize port stats */
1898 memset(&port_statistics
, 0, sizeof(port_statistics
));
1900 /* Setup port forwarding table */
1901 if (enabled_portcount
% 2) {
1902 l2fwd_dst_ports
[portid
] = last_portid
;
1903 l2fwd_dst_ports
[last_portid
] = portid
;
1905 last_portid
= portid
;
1908 l2fwd_enabled_port_mask
|= (1 << portid
);
1909 enabled_portcount
++;
1912 if (enabled_portcount
== 1) {
1913 l2fwd_dst_ports
[last_portid
] = last_portid
;
1914 } else if (enabled_portcount
% 2) {
1915 printf("odd number of ports in portmask- bye\n");
1919 check_all_ports_link_status(nb_ports
, l2fwd_enabled_port_mask
);
1921 return enabled_portcount
;
1925 reserve_key_memory(struct l2fwd_crypto_options
*options
)
1927 options
->cipher_xform
.cipher
.key
.data
= rte_malloc("crypto key",
1929 if (options
->cipher_xform
.cipher
.key
.data
== NULL
)
1930 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for cipher key");
1933 options
->auth_xform
.auth
.key
.data
= rte_malloc("auth key",
1935 if (options
->auth_xform
.auth
.key
.data
== NULL
)
1936 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for auth key");
1938 options
->iv
.data
= rte_malloc("iv", MAX_KEY_SIZE
, 0);
1939 if (options
->iv
.data
== NULL
)
1940 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for IV");
1941 options
->iv
.phys_addr
= rte_malloc_virt2phy(options
->iv
.data
);
1943 options
->aad
.data
= rte_malloc("aad", MAX_KEY_SIZE
, 0);
1944 if (options
->aad
.data
== NULL
)
1945 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for AAD");
1946 options
->aad
.phys_addr
= rte_malloc_virt2phy(options
->aad
.data
);
1950 main(int argc
, char **argv
)
1952 struct lcore_queue_conf
*qconf
;
1953 struct l2fwd_crypto_options options
;
1955 uint8_t nb_ports
, nb_cryptodevs
, portid
, cdev_id
;
1956 unsigned lcore_id
, rx_lcore_id
;
1957 int ret
, enabled_cdevcount
, enabled_portcount
;
1958 uint8_t enabled_cdevs
[RTE_CRYPTO_MAX_DEVS
] = {0};
1961 ret
= rte_eal_init(argc
, argv
);
1963 rte_exit(EXIT_FAILURE
, "Invalid EAL arguments\n");
1967 /* reserve memory for Cipher/Auth key and IV */
1968 reserve_key_memory(&options
);
1970 /* fill out the supported algorithm tables */
1971 fill_supported_algorithm_tables();
1973 /* parse application arguments (after the EAL ones) */
1974 ret
= l2fwd_crypto_parse_args(&options
, argc
, argv
);
1976 rte_exit(EXIT_FAILURE
, "Invalid L2FWD-CRYPTO arguments\n");
1978 /* create the mbuf pool */
1979 l2fwd_pktmbuf_pool
= rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF
, 512,
1980 sizeof(struct rte_crypto_op
),
1981 RTE_MBUF_DEFAULT_BUF_SIZE
, rte_socket_id());
1982 if (l2fwd_pktmbuf_pool
== NULL
)
1983 rte_exit(EXIT_FAILURE
, "Cannot create mbuf pool\n");
1985 /* create crypto op pool */
1986 l2fwd_crypto_op_pool
= rte_crypto_op_pool_create("crypto_op_pool",
1987 RTE_CRYPTO_OP_TYPE_SYMMETRIC
, NB_MBUF
, 128, 0,
1989 if (l2fwd_crypto_op_pool
== NULL
)
1990 rte_exit(EXIT_FAILURE
, "Cannot create crypto op pool\n");
1992 /* Enable Ethernet ports */
1993 enabled_portcount
= initialize_ports(&options
);
1994 if (enabled_portcount
< 1)
1995 rte_exit(EXIT_FAILURE
, "Failed to initial Ethernet ports\n");
1997 nb_ports
= rte_eth_dev_count();
1998 /* Initialize the port/queue configuration of each logical core */
1999 for (rx_lcore_id
= 0, qconf
= NULL
, portid
= 0;
2000 portid
< nb_ports
; portid
++) {
2002 /* skip ports that are not enabled */
2003 if ((options
.portmask
& (1 << portid
)) == 0)
2006 if (options
.single_lcore
&& qconf
== NULL
) {
2007 while (rte_lcore_is_enabled(rx_lcore_id
) == 0) {
2009 if (rx_lcore_id
>= RTE_MAX_LCORE
)
2010 rte_exit(EXIT_FAILURE
,
2011 "Not enough cores\n");
2013 } else if (!options
.single_lcore
) {
2014 /* get the lcore_id for this port */
2015 while (rte_lcore_is_enabled(rx_lcore_id
) == 0 ||
2016 lcore_queue_conf
[rx_lcore_id
].nb_rx_ports
==
2017 options
.nb_ports_per_lcore
) {
2019 if (rx_lcore_id
>= RTE_MAX_LCORE
)
2020 rte_exit(EXIT_FAILURE
,
2021 "Not enough cores\n");
2025 /* Assigned a new logical core in the loop above. */
2026 if (qconf
!= &lcore_queue_conf
[rx_lcore_id
])
2027 qconf
= &lcore_queue_conf
[rx_lcore_id
];
2029 qconf
->rx_port_list
[qconf
->nb_rx_ports
] = portid
;
2030 qconf
->nb_rx_ports
++;
2032 printf("Lcore %u: RX port %u\n", rx_lcore_id
, (unsigned)portid
);
2035 /* Enable Crypto devices */
2036 enabled_cdevcount
= initialize_cryptodevs(&options
, enabled_portcount
,
2038 if (enabled_cdevcount
< 0)
2039 rte_exit(EXIT_FAILURE
, "Failed to initialize crypto devices\n");
2041 if (enabled_cdevcount
< enabled_portcount
)
2042 rte_exit(EXIT_FAILURE
, "Number of capable crypto devices (%d) "
2043 "has to be more or equal to number of ports (%d)\n",
2044 enabled_cdevcount
, enabled_portcount
);
2046 nb_cryptodevs
= rte_cryptodev_count();
2048 /* Initialize the port/cryptodev configuration of each logical core */
2049 for (rx_lcore_id
= 0, qconf
= NULL
, cdev_id
= 0;
2050 cdev_id
< nb_cryptodevs
&& enabled_cdevcount
;
2052 /* Crypto op not supported by crypto device */
2053 if (!enabled_cdevs
[cdev_id
])
2056 if (options
.single_lcore
&& qconf
== NULL
) {
2057 while (rte_lcore_is_enabled(rx_lcore_id
) == 0) {
2059 if (rx_lcore_id
>= RTE_MAX_LCORE
)
2060 rte_exit(EXIT_FAILURE
,
2061 "Not enough cores\n");
2063 } else if (!options
.single_lcore
) {
2064 /* get the lcore_id for this port */
2065 while (rte_lcore_is_enabled(rx_lcore_id
) == 0 ||
2066 lcore_queue_conf
[rx_lcore_id
].nb_crypto_devs
==
2067 options
.nb_ports_per_lcore
) {
2069 if (rx_lcore_id
>= RTE_MAX_LCORE
)
2070 rte_exit(EXIT_FAILURE
,
2071 "Not enough cores\n");
2075 /* Assigned a new logical core in the loop above. */
2076 if (qconf
!= &lcore_queue_conf
[rx_lcore_id
])
2077 qconf
= &lcore_queue_conf
[rx_lcore_id
];
2079 qconf
->cryptodev_list
[qconf
->nb_crypto_devs
] = cdev_id
;
2080 qconf
->nb_crypto_devs
++;
2082 enabled_cdevcount
--;
2084 printf("Lcore %u: cryptodev %u\n", rx_lcore_id
,
2088 /* launch per-lcore init on every lcore */
2089 rte_eal_mp_remote_launch(l2fwd_launch_one_lcore
, (void *)&options
,
2091 RTE_LCORE_FOREACH_SLAVE(lcore_id
) {
2092 if (rte_eal_wait_lcore(lcore_id
) < 0)