1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2016 Intel Corporation
11 #include <sys/types.h>
12 #include <sys/queue.h>
13 #include <netinet/in.h>
22 #include <rte_atomic.h>
23 #include <rte_branch_prediction.h>
24 #include <rte_common.h>
25 #include <rte_cryptodev.h>
26 #include <rte_cycles.h>
27 #include <rte_debug.h>
29 #include <rte_ether.h>
30 #include <rte_ethdev.h>
31 #include <rte_interrupts.h>
33 #include <rte_launch.h>
34 #include <rte_lcore.h>
36 #include <rte_malloc.h>
38 #include <rte_memcpy.h>
39 #include <rte_memory.h>
40 #include <rte_mempool.h>
41 #include <rte_per_lcore.h>
42 #include <rte_prefetch.h>
43 #include <rte_random.h>
44 #include <rte_hexdump.h>
45 #ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER
46 #include <rte_cryptodev_scheduler.h>
55 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
59 #define MAX_STR_LEN 32
60 #define MAX_KEY_SIZE 128
61 #define MAX_IV_SIZE 16
62 #define MAX_AAD_SIZE 65535
63 #define MAX_PKT_BURST 32
64 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
65 #define SESSION_POOL_CACHE_SIZE 0
67 #define MAXIMUM_IV_LENGTH 16
68 #define IV_OFFSET (sizeof(struct rte_crypto_op) + \
69 sizeof(struct rte_crypto_sym_op))
72 * Configurable number of RX/TX ring descriptors
74 #define RTE_TEST_RX_DESC_DEFAULT 1024
75 #define RTE_TEST_TX_DESC_DEFAULT 1024
77 static uint16_t nb_rxd
= RTE_TEST_RX_DESC_DEFAULT
;
78 static uint16_t nb_txd
= RTE_TEST_TX_DESC_DEFAULT
;
80 /* ethernet addresses of ports */
81 static struct ether_addr l2fwd_ports_eth_addr
[RTE_MAX_ETHPORTS
];
83 /* mask of enabled ports */
84 static uint64_t l2fwd_enabled_port_mask
;
85 static uint64_t l2fwd_enabled_crypto_mask
;
87 /* list of enabled ports */
88 static uint16_t l2fwd_dst_ports
[RTE_MAX_ETHPORTS
];
93 struct rte_mbuf
*buffer
[MAX_PKT_BURST
];
98 struct rte_crypto_op
*buffer
[MAX_PKT_BURST
];
101 #define MAX_RX_QUEUE_PER_LCORE 16
102 #define MAX_TX_QUEUE_PER_PORT 16
104 enum l2fwd_crypto_xform_chain
{
105 L2FWD_CRYPTO_CIPHER_HASH
,
106 L2FWD_CRYPTO_HASH_CIPHER
,
107 L2FWD_CRYPTO_CIPHER_ONLY
,
108 L2FWD_CRYPTO_HASH_ONLY
,
115 rte_iova_t phys_addr
;
123 /** l2fwd crypto application command line options */
124 struct l2fwd_crypto_options
{
126 unsigned nb_ports_per_lcore
;
127 unsigned refresh_period
;
128 unsigned single_lcore
:1;
131 unsigned sessionless
:1;
133 enum l2fwd_crypto_xform_chain xform_chain
;
135 struct rte_crypto_sym_xform cipher_xform
;
137 int ckey_random_size
;
139 struct l2fwd_iv cipher_iv
;
140 unsigned int cipher_iv_param
;
141 int cipher_iv_random_size
;
143 struct rte_crypto_sym_xform auth_xform
;
145 int akey_random_size
;
147 struct l2fwd_iv auth_iv
;
148 unsigned int auth_iv_param
;
149 int auth_iv_random_size
;
151 struct rte_crypto_sym_xform aead_xform
;
152 unsigned int aead_key_param
;
153 int aead_key_random_size
;
155 struct l2fwd_iv aead_iv
;
156 unsigned int aead_iv_param
;
157 int aead_iv_random_size
;
159 struct l2fwd_key aad
;
166 char string_type
[MAX_STR_LEN
];
168 uint64_t cryptodev_mask
;
170 unsigned int mac_updating
;
173 /** l2fwd crypto lcore params */
174 struct l2fwd_crypto_params
{
178 unsigned digest_length
;
181 struct l2fwd_iv cipher_iv
;
182 struct l2fwd_iv auth_iv
;
183 struct l2fwd_iv aead_iv
;
184 struct l2fwd_key aad
;
185 struct rte_cryptodev_sym_session
*session
;
192 enum rte_crypto_cipher_algorithm cipher_algo
;
193 enum rte_crypto_auth_algorithm auth_algo
;
194 enum rte_crypto_aead_algorithm aead_algo
;
197 /** lcore configuration */
198 struct lcore_queue_conf
{
199 unsigned nb_rx_ports
;
200 uint16_t rx_port_list
[MAX_RX_QUEUE_PER_LCORE
];
202 unsigned nb_crypto_devs
;
203 unsigned cryptodev_list
[MAX_RX_QUEUE_PER_LCORE
];
205 struct op_buffer op_buf
[RTE_CRYPTO_MAX_DEVS
];
206 struct pkt_buffer pkt_buf
[RTE_MAX_ETHPORTS
];
207 } __rte_cache_aligned
;
209 struct lcore_queue_conf lcore_queue_conf
[RTE_MAX_LCORE
];
211 static struct rte_eth_conf port_conf
= {
213 .mq_mode
= ETH_MQ_RX_NONE
,
214 .max_rx_pkt_len
= ETHER_MAX_LEN
,
216 .offloads
= DEV_RX_OFFLOAD_CRC_STRIP
,
219 .mq_mode
= ETH_MQ_TX_NONE
,
223 struct rte_mempool
*l2fwd_pktmbuf_pool
;
224 struct rte_mempool
*l2fwd_crypto_op_pool
;
225 struct rte_mempool
*session_pool_socket
[RTE_MAX_NUMA_NODES
] = { 0 };
227 /* Per-port statistics struct */
228 struct l2fwd_port_statistics
{
232 uint64_t crypto_enqueued
;
233 uint64_t crypto_dequeued
;
236 } __rte_cache_aligned
;
238 struct l2fwd_crypto_statistics
{
243 } __rte_cache_aligned
;
245 struct l2fwd_port_statistics port_statistics
[RTE_MAX_ETHPORTS
];
246 struct l2fwd_crypto_statistics crypto_statistics
[RTE_CRYPTO_MAX_DEVS
];
248 /* A tsc-based timer responsible for triggering statistics printout */
249 #define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */
250 #define MAX_TIMER_PERIOD 86400UL /* 1 day max */
252 /* default period is 10 seconds */
253 static int64_t timer_period
= 10 * TIMER_MILLISECOND
* 1000;
255 /* Print out statistics on packets dropped */
259 uint64_t total_packets_dropped
, total_packets_tx
, total_packets_rx
;
260 uint64_t total_packets_enqueued
, total_packets_dequeued
,
261 total_packets_errors
;
265 total_packets_dropped
= 0;
266 total_packets_tx
= 0;
267 total_packets_rx
= 0;
268 total_packets_enqueued
= 0;
269 total_packets_dequeued
= 0;
270 total_packets_errors
= 0;
272 const char clr
[] = { 27, '[', '2', 'J', '\0' };
273 const char topLeft
[] = { 27, '[', '1', ';', '1', 'H', '\0' };
275 /* Clear screen and move to top left */
276 printf("%s%s", clr
, topLeft
);
278 printf("\nPort statistics ====================================");
280 for (portid
= 0; portid
< RTE_MAX_ETHPORTS
; portid
++) {
281 /* skip disabled ports */
282 if ((l2fwd_enabled_port_mask
& (1 << portid
)) == 0)
284 printf("\nStatistics for port %u ------------------------------"
285 "\nPackets sent: %32"PRIu64
286 "\nPackets received: %28"PRIu64
287 "\nPackets dropped: %29"PRIu64
,
289 port_statistics
[portid
].tx
,
290 port_statistics
[portid
].rx
,
291 port_statistics
[portid
].dropped
);
293 total_packets_dropped
+= port_statistics
[portid
].dropped
;
294 total_packets_tx
+= port_statistics
[portid
].tx
;
295 total_packets_rx
+= port_statistics
[portid
].rx
;
297 printf("\nCrypto statistics ==================================");
299 for (cdevid
= 0; cdevid
< RTE_CRYPTO_MAX_DEVS
; cdevid
++) {
300 /* skip disabled ports */
301 if ((l2fwd_enabled_crypto_mask
& (((uint64_t)1) << cdevid
)) == 0)
303 printf("\nStatistics for cryptodev %"PRIu64
304 " -------------------------"
305 "\nPackets enqueued: %28"PRIu64
306 "\nPackets dequeued: %28"PRIu64
307 "\nPackets errors: %30"PRIu64
,
309 crypto_statistics
[cdevid
].enqueued
,
310 crypto_statistics
[cdevid
].dequeued
,
311 crypto_statistics
[cdevid
].errors
);
313 total_packets_enqueued
+= crypto_statistics
[cdevid
].enqueued
;
314 total_packets_dequeued
+= crypto_statistics
[cdevid
].dequeued
;
315 total_packets_errors
+= crypto_statistics
[cdevid
].errors
;
317 printf("\nAggregate statistics ==============================="
318 "\nTotal packets received: %22"PRIu64
319 "\nTotal packets enqueued: %22"PRIu64
320 "\nTotal packets dequeued: %22"PRIu64
321 "\nTotal packets sent: %26"PRIu64
322 "\nTotal packets dropped: %23"PRIu64
323 "\nTotal packets crypto errors: %17"PRIu64
,
325 total_packets_enqueued
,
326 total_packets_dequeued
,
328 total_packets_dropped
,
329 total_packets_errors
);
330 printf("\n====================================================\n");
334 l2fwd_crypto_send_burst(struct lcore_queue_conf
*qconf
, unsigned n
,
335 struct l2fwd_crypto_params
*cparams
)
337 struct rte_crypto_op
**op_buffer
;
340 op_buffer
= (struct rte_crypto_op
**)
341 qconf
->op_buf
[cparams
->dev_id
].buffer
;
343 ret
= rte_cryptodev_enqueue_burst(cparams
->dev_id
,
344 cparams
->qp_id
, op_buffer
, (uint16_t) n
);
346 crypto_statistics
[cparams
->dev_id
].enqueued
+= ret
;
347 if (unlikely(ret
< n
)) {
348 crypto_statistics
[cparams
->dev_id
].errors
+= (n
- ret
);
350 rte_pktmbuf_free(op_buffer
[ret
]->sym
->m_src
);
351 rte_crypto_op_free(op_buffer
[ret
]);
359 l2fwd_crypto_enqueue(struct rte_crypto_op
*op
,
360 struct l2fwd_crypto_params
*cparams
)
362 unsigned lcore_id
, len
;
363 struct lcore_queue_conf
*qconf
;
365 lcore_id
= rte_lcore_id();
367 qconf
= &lcore_queue_conf
[lcore_id
];
368 len
= qconf
->op_buf
[cparams
->dev_id
].len
;
369 qconf
->op_buf
[cparams
->dev_id
].buffer
[len
] = op
;
372 /* enough ops to be sent */
373 if (len
== MAX_PKT_BURST
) {
374 l2fwd_crypto_send_burst(qconf
, MAX_PKT_BURST
, cparams
);
378 qconf
->op_buf
[cparams
->dev_id
].len
= len
;
383 l2fwd_simple_crypto_enqueue(struct rte_mbuf
*m
,
384 struct rte_crypto_op
*op
,
385 struct l2fwd_crypto_params
*cparams
)
387 struct ether_hdr
*eth_hdr
;
388 struct ipv4_hdr
*ip_hdr
;
390 uint32_t ipdata_offset
, data_len
;
391 uint32_t pad_len
= 0;
394 eth_hdr
= rte_pktmbuf_mtod(m
, struct ether_hdr
*);
396 if (eth_hdr
->ether_type
!= rte_cpu_to_be_16(ETHER_TYPE_IPv4
))
399 ipdata_offset
= sizeof(struct ether_hdr
);
401 ip_hdr
= (struct ipv4_hdr
*)(rte_pktmbuf_mtod(m
, char *) +
404 ipdata_offset
+= (ip_hdr
->version_ihl
& IPV4_HDR_IHL_MASK
)
405 * IPV4_IHL_MULTIPLIER
;
408 /* Zero pad data to be crypto'd so it is block aligned */
409 data_len
= rte_pktmbuf_data_len(m
) - ipdata_offset
;
411 if ((cparams
->do_hash
|| cparams
->do_aead
) && cparams
->hash_verify
)
412 data_len
-= cparams
->digest_length
;
414 if (cparams
->do_cipher
) {
416 * Following algorithms are block cipher algorithms,
417 * and might need padding
419 switch (cparams
->cipher_algo
) {
420 case RTE_CRYPTO_CIPHER_AES_CBC
:
421 case RTE_CRYPTO_CIPHER_AES_ECB
:
422 case RTE_CRYPTO_CIPHER_DES_CBC
:
423 case RTE_CRYPTO_CIPHER_3DES_CBC
:
424 case RTE_CRYPTO_CIPHER_3DES_ECB
:
425 if (data_len
% cparams
->block_size
)
426 pad_len
= cparams
->block_size
-
427 (data_len
% cparams
->block_size
);
434 padding
= rte_pktmbuf_append(m
, pad_len
);
435 if (unlikely(!padding
))
439 memset(padding
, 0, pad_len
);
443 /* Set crypto operation data parameters */
444 rte_crypto_op_attach_sym_session(op
, cparams
->session
);
446 if (cparams
->do_hash
) {
447 if (cparams
->auth_iv
.length
) {
448 uint8_t *iv_ptr
= rte_crypto_op_ctod_offset(op
,
451 cparams
->cipher_iv
.length
);
453 * Copy IV at the end of the crypto operation,
454 * after the cipher IV, if added
456 rte_memcpy(iv_ptr
, cparams
->auth_iv
.data
,
457 cparams
->auth_iv
.length
);
459 if (!cparams
->hash_verify
) {
460 /* Append space for digest to end of packet */
461 op
->sym
->auth
.digest
.data
= (uint8_t *)rte_pktmbuf_append(m
,
462 cparams
->digest_length
);
464 op
->sym
->auth
.digest
.data
= rte_pktmbuf_mtod(m
,
465 uint8_t *) + ipdata_offset
+ data_len
;
468 op
->sym
->auth
.digest
.phys_addr
= rte_pktmbuf_iova_offset(m
,
469 rte_pktmbuf_pkt_len(m
) - cparams
->digest_length
);
471 /* For wireless algorithms, offset/length must be in bits */
472 if (cparams
->auth_algo
== RTE_CRYPTO_AUTH_SNOW3G_UIA2
||
473 cparams
->auth_algo
== RTE_CRYPTO_AUTH_KASUMI_F9
||
474 cparams
->auth_algo
== RTE_CRYPTO_AUTH_ZUC_EIA3
) {
475 op
->sym
->auth
.data
.offset
= ipdata_offset
<< 3;
476 op
->sym
->auth
.data
.length
= data_len
<< 3;
478 op
->sym
->auth
.data
.offset
= ipdata_offset
;
479 op
->sym
->auth
.data
.length
= data_len
;
483 if (cparams
->do_cipher
) {
484 uint8_t *iv_ptr
= rte_crypto_op_ctod_offset(op
, uint8_t *,
486 /* Copy IV at the end of the crypto operation */
487 rte_memcpy(iv_ptr
, cparams
->cipher_iv
.data
,
488 cparams
->cipher_iv
.length
);
490 /* For wireless algorithms, offset/length must be in bits */
491 if (cparams
->cipher_algo
== RTE_CRYPTO_CIPHER_SNOW3G_UEA2
||
492 cparams
->cipher_algo
== RTE_CRYPTO_CIPHER_KASUMI_F8
||
493 cparams
->cipher_algo
== RTE_CRYPTO_CIPHER_ZUC_EEA3
) {
494 op
->sym
->cipher
.data
.offset
= ipdata_offset
<< 3;
495 op
->sym
->cipher
.data
.length
= data_len
<< 3;
497 op
->sym
->cipher
.data
.offset
= ipdata_offset
;
498 op
->sym
->cipher
.data
.length
= data_len
;
502 if (cparams
->do_aead
) {
503 uint8_t *iv_ptr
= rte_crypto_op_ctod_offset(op
, uint8_t *,
505 /* Copy IV at the end of the crypto operation */
507 * If doing AES-CCM, nonce is copied one byte
508 * after the start of IV field
510 if (cparams
->aead_algo
== RTE_CRYPTO_AEAD_AES_CCM
)
511 rte_memcpy(iv_ptr
+ 1, cparams
->aead_iv
.data
,
512 cparams
->aead_iv
.length
);
514 rte_memcpy(iv_ptr
, cparams
->aead_iv
.data
,
515 cparams
->aead_iv
.length
);
517 op
->sym
->aead
.data
.offset
= ipdata_offset
;
518 op
->sym
->aead
.data
.length
= data_len
;
520 if (!cparams
->hash_verify
) {
521 /* Append space for digest to end of packet */
522 op
->sym
->aead
.digest
.data
= (uint8_t *)rte_pktmbuf_append(m
,
523 cparams
->digest_length
);
525 op
->sym
->aead
.digest
.data
= rte_pktmbuf_mtod(m
,
526 uint8_t *) + ipdata_offset
+ data_len
;
529 op
->sym
->aead
.digest
.phys_addr
= rte_pktmbuf_iova_offset(m
,
530 rte_pktmbuf_pkt_len(m
) - cparams
->digest_length
);
532 if (cparams
->aad
.length
) {
533 op
->sym
->aead
.aad
.data
= cparams
->aad
.data
;
534 op
->sym
->aead
.aad
.phys_addr
= cparams
->aad
.phys_addr
;
540 return l2fwd_crypto_enqueue(op
, cparams
);
544 /* Send the burst of packets on an output interface */
546 l2fwd_send_burst(struct lcore_queue_conf
*qconf
, unsigned n
,
549 struct rte_mbuf
**pkt_buffer
;
552 pkt_buffer
= (struct rte_mbuf
**)qconf
->pkt_buf
[port
].buffer
;
554 ret
= rte_eth_tx_burst(port
, 0, pkt_buffer
, (uint16_t)n
);
555 port_statistics
[port
].tx
+= ret
;
556 if (unlikely(ret
< n
)) {
557 port_statistics
[port
].dropped
+= (n
- ret
);
559 rte_pktmbuf_free(pkt_buffer
[ret
]);
566 /* Enqueue packets for TX and prepare them to be sent */
568 l2fwd_send_packet(struct rte_mbuf
*m
, uint16_t port
)
570 unsigned lcore_id
, len
;
571 struct lcore_queue_conf
*qconf
;
573 lcore_id
= rte_lcore_id();
575 qconf
= &lcore_queue_conf
[lcore_id
];
576 len
= qconf
->pkt_buf
[port
].len
;
577 qconf
->pkt_buf
[port
].buffer
[len
] = m
;
580 /* enough pkts to be sent */
581 if (unlikely(len
== MAX_PKT_BURST
)) {
582 l2fwd_send_burst(qconf
, MAX_PKT_BURST
, port
);
586 qconf
->pkt_buf
[port
].len
= len
;
591 l2fwd_mac_updating(struct rte_mbuf
*m
, uint16_t dest_portid
)
593 struct ether_hdr
*eth
;
596 eth
= rte_pktmbuf_mtod(m
, struct ether_hdr
*);
598 /* 02:00:00:00:00:xx */
599 tmp
= ð
->d_addr
.addr_bytes
[0];
600 *((uint64_t *)tmp
) = 0x000000000002 + ((uint64_t)dest_portid
<< 40);
603 ether_addr_copy(&l2fwd_ports_eth_addr
[dest_portid
], ð
->s_addr
);
607 l2fwd_simple_forward(struct rte_mbuf
*m
, uint16_t portid
,
608 struct l2fwd_crypto_options
*options
)
612 dst_port
= l2fwd_dst_ports
[portid
];
614 if (options
->mac_updating
)
615 l2fwd_mac_updating(m
, dst_port
);
617 l2fwd_send_packet(m
, dst_port
);
620 /** Generate random key */
622 generate_random_key(uint8_t *key
, unsigned length
)
627 fd
= open("/dev/urandom", O_RDONLY
);
629 rte_exit(EXIT_FAILURE
, "Failed to generate random key\n");
631 ret
= read(fd
, key
, length
);
634 if (ret
!= (signed)length
)
635 rte_exit(EXIT_FAILURE
, "Failed to generate random key\n");
638 static struct rte_cryptodev_sym_session
*
639 initialize_crypto_session(struct l2fwd_crypto_options
*options
, uint8_t cdev_id
)
641 struct rte_crypto_sym_xform
*first_xform
;
642 struct rte_cryptodev_sym_session
*session
;
643 int retval
= rte_cryptodev_socket_id(cdev_id
);
648 uint8_t socket_id
= (uint8_t) retval
;
649 struct rte_mempool
*sess_mp
= session_pool_socket
[socket_id
];
651 if (options
->xform_chain
== L2FWD_CRYPTO_AEAD
) {
652 first_xform
= &options
->aead_xform
;
653 } else if (options
->xform_chain
== L2FWD_CRYPTO_CIPHER_HASH
) {
654 first_xform
= &options
->cipher_xform
;
655 first_xform
->next
= &options
->auth_xform
;
656 } else if (options
->xform_chain
== L2FWD_CRYPTO_HASH_CIPHER
) {
657 first_xform
= &options
->auth_xform
;
658 first_xform
->next
= &options
->cipher_xform
;
659 } else if (options
->xform_chain
== L2FWD_CRYPTO_CIPHER_ONLY
) {
660 first_xform
= &options
->cipher_xform
;
662 first_xform
= &options
->auth_xform
;
665 session
= rte_cryptodev_sym_session_create(sess_mp
);
670 if (rte_cryptodev_sym_session_init(cdev_id
, session
,
671 first_xform
, sess_mp
) < 0)
678 l2fwd_crypto_options_print(struct l2fwd_crypto_options
*options
);
680 /* main processing loop */
682 l2fwd_main_loop(struct l2fwd_crypto_options
*options
)
684 struct rte_mbuf
*m
, *pkts_burst
[MAX_PKT_BURST
];
685 struct rte_crypto_op
*ops_burst
[MAX_PKT_BURST
];
687 unsigned lcore_id
= rte_lcore_id();
688 uint64_t prev_tsc
= 0, diff_tsc
, cur_tsc
, timer_tsc
= 0;
689 unsigned int i
, j
, nb_rx
, len
;
691 struct lcore_queue_conf
*qconf
= &lcore_queue_conf
[lcore_id
];
692 const uint64_t drain_tsc
= (rte_get_tsc_hz() + US_PER_S
- 1) /
693 US_PER_S
* BURST_TX_DRAIN_US
;
694 struct l2fwd_crypto_params
*cparams
;
695 struct l2fwd_crypto_params port_cparams
[qconf
->nb_crypto_devs
];
696 struct rte_cryptodev_sym_session
*session
;
698 if (qconf
->nb_rx_ports
== 0) {
699 RTE_LOG(INFO
, L2FWD
, "lcore %u has nothing to do\n", lcore_id
);
703 RTE_LOG(INFO
, L2FWD
, "entering main loop on lcore %u\n", lcore_id
);
705 for (i
= 0; i
< qconf
->nb_rx_ports
; i
++) {
707 portid
= qconf
->rx_port_list
[i
];
708 RTE_LOG(INFO
, L2FWD
, " -- lcoreid=%u portid=%u\n", lcore_id
,
712 for (i
= 0; i
< qconf
->nb_crypto_devs
; i
++) {
713 port_cparams
[i
].do_cipher
= 0;
714 port_cparams
[i
].do_hash
= 0;
715 port_cparams
[i
].do_aead
= 0;
717 switch (options
->xform_chain
) {
718 case L2FWD_CRYPTO_AEAD
:
719 port_cparams
[i
].do_aead
= 1;
721 case L2FWD_CRYPTO_CIPHER_HASH
:
722 case L2FWD_CRYPTO_HASH_CIPHER
:
723 port_cparams
[i
].do_cipher
= 1;
724 port_cparams
[i
].do_hash
= 1;
726 case L2FWD_CRYPTO_HASH_ONLY
:
727 port_cparams
[i
].do_hash
= 1;
729 case L2FWD_CRYPTO_CIPHER_ONLY
:
730 port_cparams
[i
].do_cipher
= 1;
734 port_cparams
[i
].dev_id
= qconf
->cryptodev_list
[i
];
735 port_cparams
[i
].qp_id
= 0;
737 port_cparams
[i
].block_size
= options
->block_size
;
739 if (port_cparams
[i
].do_hash
) {
740 port_cparams
[i
].auth_iv
.data
= options
->auth_iv
.data
;
741 port_cparams
[i
].auth_iv
.length
= options
->auth_iv
.length
;
742 if (!options
->auth_iv_param
)
743 generate_random_key(port_cparams
[i
].auth_iv
.data
,
744 port_cparams
[i
].auth_iv
.length
);
745 if (options
->auth_xform
.auth
.op
== RTE_CRYPTO_AUTH_OP_VERIFY
)
746 port_cparams
[i
].hash_verify
= 1;
748 port_cparams
[i
].hash_verify
= 0;
750 port_cparams
[i
].auth_algo
= options
->auth_xform
.auth
.algo
;
751 port_cparams
[i
].digest_length
=
752 options
->auth_xform
.auth
.digest_length
;
753 /* Set IV parameters */
754 if (options
->auth_iv
.length
) {
755 options
->auth_xform
.auth
.iv
.offset
=
756 IV_OFFSET
+ options
->cipher_iv
.length
;
757 options
->auth_xform
.auth
.iv
.length
=
758 options
->auth_iv
.length
;
762 if (port_cparams
[i
].do_aead
) {
763 port_cparams
[i
].aead_iv
.data
= options
->aead_iv
.data
;
764 port_cparams
[i
].aead_iv
.length
= options
->aead_iv
.length
;
765 if (!options
->aead_iv_param
)
766 generate_random_key(port_cparams
[i
].aead_iv
.data
,
767 port_cparams
[i
].aead_iv
.length
);
768 port_cparams
[i
].aead_algo
= options
->aead_xform
.aead
.algo
;
769 port_cparams
[i
].digest_length
=
770 options
->aead_xform
.aead
.digest_length
;
771 if (options
->aead_xform
.aead
.aad_length
) {
772 port_cparams
[i
].aad
.data
= options
->aad
.data
;
773 port_cparams
[i
].aad
.phys_addr
= options
->aad
.phys_addr
;
774 port_cparams
[i
].aad
.length
= options
->aad
.length
;
775 if (!options
->aad_param
)
776 generate_random_key(port_cparams
[i
].aad
.data
,
777 port_cparams
[i
].aad
.length
);
779 * If doing AES-CCM, first 18 bytes has to be reserved,
780 * and actual AAD should start from byte 18
782 if (port_cparams
[i
].aead_algo
== RTE_CRYPTO_AEAD_AES_CCM
)
783 memmove(port_cparams
[i
].aad
.data
+ 18,
784 port_cparams
[i
].aad
.data
,
785 port_cparams
[i
].aad
.length
);
788 port_cparams
[i
].aad
.length
= 0;
790 if (options
->aead_xform
.aead
.op
== RTE_CRYPTO_AEAD_OP_DECRYPT
)
791 port_cparams
[i
].hash_verify
= 1;
793 port_cparams
[i
].hash_verify
= 0;
795 /* Set IV parameters */
796 options
->aead_xform
.aead
.iv
.offset
= IV_OFFSET
;
797 options
->aead_xform
.aead
.iv
.length
= options
->aead_iv
.length
;
800 if (port_cparams
[i
].do_cipher
) {
801 port_cparams
[i
].cipher_iv
.data
= options
->cipher_iv
.data
;
802 port_cparams
[i
].cipher_iv
.length
= options
->cipher_iv
.length
;
803 if (!options
->cipher_iv_param
)
804 generate_random_key(port_cparams
[i
].cipher_iv
.data
,
805 port_cparams
[i
].cipher_iv
.length
);
807 port_cparams
[i
].cipher_algo
= options
->cipher_xform
.cipher
.algo
;
808 /* Set IV parameters */
809 options
->cipher_xform
.cipher
.iv
.offset
= IV_OFFSET
;
810 options
->cipher_xform
.cipher
.iv
.length
=
811 options
->cipher_iv
.length
;
814 session
= initialize_crypto_session(options
,
815 port_cparams
[i
].dev_id
);
817 rte_exit(EXIT_FAILURE
, "Failed to initialize crypto session\n");
819 port_cparams
[i
].session
= session
;
821 RTE_LOG(INFO
, L2FWD
, " -- lcoreid=%u cryptoid=%u\n", lcore_id
,
822 port_cparams
[i
].dev_id
);
825 l2fwd_crypto_options_print(options
);
828 * Initialize previous tsc timestamp before the loop,
829 * to avoid showing the port statistics immediately,
830 * so user can see the crypto information.
832 prev_tsc
= rte_rdtsc();
835 cur_tsc
= rte_rdtsc();
838 * Crypto device/TX burst queue drain
840 diff_tsc
= cur_tsc
- prev_tsc
;
841 if (unlikely(diff_tsc
> drain_tsc
)) {
842 /* Enqueue all crypto ops remaining in buffers */
843 for (i
= 0; i
< qconf
->nb_crypto_devs
; i
++) {
844 cparams
= &port_cparams
[i
];
845 len
= qconf
->op_buf
[cparams
->dev_id
].len
;
846 l2fwd_crypto_send_burst(qconf
, len
, cparams
);
847 qconf
->op_buf
[cparams
->dev_id
].len
= 0;
849 /* Transmit all packets remaining in buffers */
850 for (portid
= 0; portid
< RTE_MAX_ETHPORTS
; portid
++) {
851 if (qconf
->pkt_buf
[portid
].len
== 0)
853 l2fwd_send_burst(&lcore_queue_conf
[lcore_id
],
854 qconf
->pkt_buf
[portid
].len
,
856 qconf
->pkt_buf
[portid
].len
= 0;
859 /* if timer is enabled */
860 if (timer_period
> 0) {
862 /* advance the timer */
863 timer_tsc
+= diff_tsc
;
865 /* if timer has reached its timeout */
866 if (unlikely(timer_tsc
>=
867 (uint64_t)timer_period
)) {
869 /* do this only on master core */
870 if (lcore_id
== rte_get_master_lcore()
871 && options
->refresh_period
) {
882 * Read packet from RX queues
884 for (i
= 0; i
< qconf
->nb_rx_ports
; i
++) {
885 portid
= qconf
->rx_port_list
[i
];
887 cparams
= &port_cparams
[i
];
889 nb_rx
= rte_eth_rx_burst(portid
, 0,
890 pkts_burst
, MAX_PKT_BURST
);
892 port_statistics
[portid
].rx
+= nb_rx
;
896 * If we can't allocate a crypto_ops, then drop
897 * the rest of the burst and dequeue and
898 * process the packets to free offload structs
900 if (rte_crypto_op_bulk_alloc(
901 l2fwd_crypto_op_pool
,
902 RTE_CRYPTO_OP_TYPE_SYMMETRIC
,
905 for (j
= 0; j
< nb_rx
; j
++)
906 rte_pktmbuf_free(pkts_burst
[j
]);
911 /* Enqueue packets from Crypto device*/
912 for (j
= 0; j
< nb_rx
; j
++) {
915 l2fwd_simple_crypto_enqueue(m
,
916 ops_burst
[j
], cparams
);
920 /* Dequeue packets from Crypto device */
922 nb_rx
= rte_cryptodev_dequeue_burst(
923 cparams
->dev_id
, cparams
->qp_id
,
924 ops_burst
, MAX_PKT_BURST
);
926 crypto_statistics
[cparams
->dev_id
].dequeued
+=
929 /* Forward crypto'd packets */
930 for (j
= 0; j
< nb_rx
; j
++) {
931 m
= ops_burst
[j
]->sym
->m_src
;
933 rte_crypto_op_free(ops_burst
[j
]);
934 l2fwd_simple_forward(m
, portid
,
937 } while (nb_rx
== MAX_PKT_BURST
);
943 l2fwd_launch_one_lcore(void *arg
)
945 l2fwd_main_loop((struct l2fwd_crypto_options
*)arg
);
949 /* Display command line arguments usage */
951 l2fwd_crypto_usage(const char *prgname
)
953 printf("%s [EAL options] --\n"
954 " -p PORTMASK: hexadecimal bitmask of ports to configure\n"
955 " -q NQ: number of queue (=ports) per lcore (default is 1)\n"
956 " -s manage all ports from single lcore\n"
957 " -T PERIOD: statistics will be refreshed each PERIOD seconds"
958 " (0 to disable, 10 default, 86400 maximum)\n"
960 " --cdev_type HW / SW / ANY\n"
961 " --chain HASH_CIPHER / CIPHER_HASH / CIPHER_ONLY /"
962 " HASH_ONLY / AEAD\n"
964 " --cipher_algo ALGO\n"
965 " --cipher_op ENCRYPT / DECRYPT\n"
966 " --cipher_key KEY (bytes separated with \":\")\n"
967 " --cipher_key_random_size SIZE: size of cipher key when generated randomly\n"
968 " --cipher_iv IV (bytes separated with \":\")\n"
969 " --cipher_iv_random_size SIZE: size of cipher IV when generated randomly\n"
971 " --auth_algo ALGO\n"
972 " --auth_op GENERATE / VERIFY\n"
973 " --auth_key KEY (bytes separated with \":\")\n"
974 " --auth_key_random_size SIZE: size of auth key when generated randomly\n"
975 " --auth_iv IV (bytes separated with \":\")\n"
976 " --auth_iv_random_size SIZE: size of auth IV when generated randomly\n"
978 " --aead_algo ALGO\n"
979 " --aead_op ENCRYPT / DECRYPT\n"
980 " --aead_key KEY (bytes separated with \":\")\n"
981 " --aead_key_random_size SIZE: size of AEAD key when generated randomly\n"
982 " --aead_iv IV (bytes separated with \":\")\n"
983 " --aead_iv_random_size SIZE: size of AEAD IV when generated randomly\n"
984 " --aad AAD (bytes separated with \":\")\n"
985 " --aad_random_size SIZE: size of AAD when generated randomly\n"
987 " --digest_size SIZE: size of digest to be generated/verified\n"
990 " --cryptodev_mask MASK: hexadecimal bitmask of crypto devices to configure\n"
992 " --[no-]mac-updating: Enable or disable MAC addresses updating (enabled by default)\n"
994 " - The source MAC address is replaced by the TX port MAC address\n"
995 " - The destination MAC address is replaced by 02:00:00:00:00:TX_PORT_ID\n",
999 /** Parse crypto device type command line argument */
1001 parse_cryptodev_type(enum cdev_type
*type
, char *optarg
)
1003 if (strcmp("HW", optarg
) == 0) {
1004 *type
= CDEV_TYPE_HW
;
1006 } else if (strcmp("SW", optarg
) == 0) {
1007 *type
= CDEV_TYPE_SW
;
1009 } else if (strcmp("ANY", optarg
) == 0) {
1010 *type
= CDEV_TYPE_ANY
;
1017 /** Parse crypto chain xform command line argument */
1019 parse_crypto_opt_chain(struct l2fwd_crypto_options
*options
, char *optarg
)
1021 if (strcmp("CIPHER_HASH", optarg
) == 0) {
1022 options
->xform_chain
= L2FWD_CRYPTO_CIPHER_HASH
;
1024 } else if (strcmp("HASH_CIPHER", optarg
) == 0) {
1025 options
->xform_chain
= L2FWD_CRYPTO_HASH_CIPHER
;
1027 } else if (strcmp("CIPHER_ONLY", optarg
) == 0) {
1028 options
->xform_chain
= L2FWD_CRYPTO_CIPHER_ONLY
;
1030 } else if (strcmp("HASH_ONLY", optarg
) == 0) {
1031 options
->xform_chain
= L2FWD_CRYPTO_HASH_ONLY
;
1033 } else if (strcmp("AEAD", optarg
) == 0) {
1034 options
->xform_chain
= L2FWD_CRYPTO_AEAD
;
1041 /** Parse crypto cipher algo option command line argument */
1043 parse_cipher_algo(enum rte_crypto_cipher_algorithm
*algo
, char *optarg
)
1046 if (rte_cryptodev_get_cipher_algo_enum(algo
, optarg
) < 0) {
1047 RTE_LOG(ERR
, USER1
, "Cipher algorithm specified "
1048 "not supported!\n");
1055 /** Parse crypto cipher operation command line argument */
1057 parse_cipher_op(enum rte_crypto_cipher_operation
*op
, char *optarg
)
1059 if (strcmp("ENCRYPT", optarg
) == 0) {
1060 *op
= RTE_CRYPTO_CIPHER_OP_ENCRYPT
;
1062 } else if (strcmp("DECRYPT", optarg
) == 0) {
1063 *op
= RTE_CRYPTO_CIPHER_OP_DECRYPT
;
1067 printf("Cipher operation not supported!\n");
1071 /** Parse bytes from command line argument */
1073 parse_bytes(uint8_t *data
, char *input_arg
, uint16_t max_size
)
1075 unsigned byte_count
;
1079 for (byte_count
= 0, token
= strtok(input_arg
, ":");
1080 (byte_count
< max_size
) && (token
!= NULL
);
1081 token
= strtok(NULL
, ":")) {
1083 int number
= (int)strtol(token
, NULL
, 16);
1085 if (errno
== EINVAL
|| errno
== ERANGE
|| number
> 0xFF)
1088 data
[byte_count
++] = (uint8_t)number
;
1094 /** Parse size param*/
1096 parse_size(int *size
, const char *q_arg
)
1101 /* parse hexadecimal string */
1102 n
= strtoul(q_arg
, &end
, 10);
1103 if ((q_arg
[0] == '\0') || (end
== NULL
) || (*end
!= '\0'))
1107 printf("invalid size\n");
1115 /** Parse crypto cipher operation command line argument */
1117 parse_auth_algo(enum rte_crypto_auth_algorithm
*algo
, char *optarg
)
1119 if (rte_cryptodev_get_auth_algo_enum(algo
, optarg
) < 0) {
1120 RTE_LOG(ERR
, USER1
, "Authentication algorithm specified "
1121 "not supported!\n");
1129 parse_auth_op(enum rte_crypto_auth_operation
*op
, char *optarg
)
1131 if (strcmp("VERIFY", optarg
) == 0) {
1132 *op
= RTE_CRYPTO_AUTH_OP_VERIFY
;
1134 } else if (strcmp("GENERATE", optarg
) == 0) {
1135 *op
= RTE_CRYPTO_AUTH_OP_GENERATE
;
1139 printf("Authentication operation specified not supported!\n");
1144 parse_aead_algo(enum rte_crypto_aead_algorithm
*algo
, char *optarg
)
1146 if (rte_cryptodev_get_aead_algo_enum(algo
, optarg
) < 0) {
1147 RTE_LOG(ERR
, USER1
, "AEAD algorithm specified "
1148 "not supported!\n");
1156 parse_aead_op(enum rte_crypto_aead_operation
*op
, char *optarg
)
1158 if (strcmp("ENCRYPT", optarg
) == 0) {
1159 *op
= RTE_CRYPTO_AEAD_OP_ENCRYPT
;
1161 } else if (strcmp("DECRYPT", optarg
) == 0) {
1162 *op
= RTE_CRYPTO_AEAD_OP_DECRYPT
;
1166 printf("AEAD operation specified not supported!\n");
1170 parse_cryptodev_mask(struct l2fwd_crypto_options
*options
,
1176 /* parse hexadecimal string */
1177 pm
= strtoul(q_arg
, &end
, 16);
1178 if ((pm
== '\0') || (end
== NULL
) || (*end
!= '\0'))
1181 options
->cryptodev_mask
= pm
;
1182 if (options
->cryptodev_mask
== 0) {
1183 printf("invalid cryptodev_mask specified\n");
1190 /** Parse long options */
1192 l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options
*options
,
1193 struct option
*lgopts
, int option_index
)
1197 if (strcmp(lgopts
[option_index
].name
, "cdev_type") == 0) {
1198 retval
= parse_cryptodev_type(&options
->type
, optarg
);
1200 snprintf(options
->string_type
, MAX_STR_LEN
,
1205 else if (strcmp(lgopts
[option_index
].name
, "chain") == 0)
1206 return parse_crypto_opt_chain(options
, optarg
);
1208 /* Cipher options */
1209 else if (strcmp(lgopts
[option_index
].name
, "cipher_algo") == 0)
1210 return parse_cipher_algo(&options
->cipher_xform
.cipher
.algo
,
1213 else if (strcmp(lgopts
[option_index
].name
, "cipher_op") == 0)
1214 return parse_cipher_op(&options
->cipher_xform
.cipher
.op
,
1217 else if (strcmp(lgopts
[option_index
].name
, "cipher_key") == 0) {
1218 options
->ckey_param
= 1;
1219 options
->cipher_xform
.cipher
.key
.length
=
1220 parse_bytes(options
->cipher_xform
.cipher
.key
.data
, optarg
,
1222 if (options
->cipher_xform
.cipher
.key
.length
> 0)
1228 else if (strcmp(lgopts
[option_index
].name
, "cipher_key_random_size") == 0)
1229 return parse_size(&options
->ckey_random_size
, optarg
);
1231 else if (strcmp(lgopts
[option_index
].name
, "cipher_iv") == 0) {
1232 options
->cipher_iv_param
= 1;
1233 options
->cipher_iv
.length
=
1234 parse_bytes(options
->cipher_iv
.data
, optarg
, MAX_IV_SIZE
);
1235 if (options
->cipher_iv
.length
> 0)
1241 else if (strcmp(lgopts
[option_index
].name
, "cipher_iv_random_size") == 0)
1242 return parse_size(&options
->cipher_iv_random_size
, optarg
);
1244 /* Authentication options */
1245 else if (strcmp(lgopts
[option_index
].name
, "auth_algo") == 0) {
1246 return parse_auth_algo(&options
->auth_xform
.auth
.algo
,
1250 else if (strcmp(lgopts
[option_index
].name
, "auth_op") == 0)
1251 return parse_auth_op(&options
->auth_xform
.auth
.op
,
1254 else if (strcmp(lgopts
[option_index
].name
, "auth_key") == 0) {
1255 options
->akey_param
= 1;
1256 options
->auth_xform
.auth
.key
.length
=
1257 parse_bytes(options
->auth_xform
.auth
.key
.data
, optarg
,
1259 if (options
->auth_xform
.auth
.key
.length
> 0)
1265 else if (strcmp(lgopts
[option_index
].name
, "auth_key_random_size") == 0) {
1266 return parse_size(&options
->akey_random_size
, optarg
);
1269 else if (strcmp(lgopts
[option_index
].name
, "auth_iv") == 0) {
1270 options
->auth_iv_param
= 1;
1271 options
->auth_iv
.length
=
1272 parse_bytes(options
->auth_iv
.data
, optarg
, MAX_IV_SIZE
);
1273 if (options
->auth_iv
.length
> 0)
1279 else if (strcmp(lgopts
[option_index
].name
, "auth_iv_random_size") == 0)
1280 return parse_size(&options
->auth_iv_random_size
, optarg
);
1283 else if (strcmp(lgopts
[option_index
].name
, "aead_algo") == 0) {
1284 return parse_aead_algo(&options
->aead_xform
.aead
.algo
,
1288 else if (strcmp(lgopts
[option_index
].name
, "aead_op") == 0)
1289 return parse_aead_op(&options
->aead_xform
.aead
.op
,
1292 else if (strcmp(lgopts
[option_index
].name
, "aead_key") == 0) {
1293 options
->aead_key_param
= 1;
1294 options
->aead_xform
.aead
.key
.length
=
1295 parse_bytes(options
->aead_xform
.aead
.key
.data
, optarg
,
1297 if (options
->aead_xform
.aead
.key
.length
> 0)
1303 else if (strcmp(lgopts
[option_index
].name
, "aead_key_random_size") == 0)
1304 return parse_size(&options
->aead_key_random_size
, optarg
);
1307 else if (strcmp(lgopts
[option_index
].name
, "aead_iv") == 0) {
1308 options
->aead_iv_param
= 1;
1309 options
->aead_iv
.length
=
1310 parse_bytes(options
->aead_iv
.data
, optarg
, MAX_IV_SIZE
);
1311 if (options
->aead_iv
.length
> 0)
1317 else if (strcmp(lgopts
[option_index
].name
, "aead_iv_random_size") == 0)
1318 return parse_size(&options
->aead_iv_random_size
, optarg
);
1320 else if (strcmp(lgopts
[option_index
].name
, "aad") == 0) {
1321 options
->aad_param
= 1;
1322 options
->aad
.length
=
1323 parse_bytes(options
->aad
.data
, optarg
, MAX_AAD_SIZE
);
1324 if (options
->aad
.length
> 0)
1330 else if (strcmp(lgopts
[option_index
].name
, "aad_random_size") == 0) {
1331 return parse_size(&options
->aad_random_size
, optarg
);
1334 else if (strcmp(lgopts
[option_index
].name
, "digest_size") == 0) {
1335 return parse_size(&options
->digest_size
, optarg
);
1338 else if (strcmp(lgopts
[option_index
].name
, "sessionless") == 0) {
1339 options
->sessionless
= 1;
1343 else if (strcmp(lgopts
[option_index
].name
, "cryptodev_mask") == 0)
1344 return parse_cryptodev_mask(options
, optarg
);
1346 else if (strcmp(lgopts
[option_index
].name
, "mac-updating") == 0) {
1347 options
->mac_updating
= 1;
1351 else if (strcmp(lgopts
[option_index
].name
, "no-mac-updating") == 0) {
1352 options
->mac_updating
= 0;
1359 /** Parse port mask */
1361 l2fwd_crypto_parse_portmask(struct l2fwd_crypto_options
*options
,
1367 /* parse hexadecimal string */
1368 pm
= strtoul(q_arg
, &end
, 16);
1369 if ((pm
== '\0') || (end
== NULL
) || (*end
!= '\0'))
1372 options
->portmask
= pm
;
1373 if (options
->portmask
== 0) {
1374 printf("invalid portmask specified\n");
1381 /** Parse number of queues */
1383 l2fwd_crypto_parse_nqueue(struct l2fwd_crypto_options
*options
,
1389 /* parse hexadecimal string */
1390 n
= strtoul(q_arg
, &end
, 10);
1391 if ((q_arg
[0] == '\0') || (end
== NULL
) || (*end
!= '\0'))
1393 else if (n
>= MAX_RX_QUEUE_PER_LCORE
)
1396 options
->nb_ports_per_lcore
= n
;
1397 if (options
->nb_ports_per_lcore
== 0) {
1398 printf("invalid number of ports selected\n");
1405 /** Parse timer period */
1407 l2fwd_crypto_parse_timer_period(struct l2fwd_crypto_options
*options
,
1413 /* parse number string */
1414 n
= (unsigned)strtol(q_arg
, &end
, 10);
1415 if ((q_arg
[0] == '\0') || (end
== NULL
) || (*end
!= '\0'))
1418 if (n
>= MAX_TIMER_PERIOD
) {
1419 printf("Warning refresh period specified %lu is greater than "
1420 "max value %lu! using max value",
1421 n
, MAX_TIMER_PERIOD
);
1422 n
= MAX_TIMER_PERIOD
;
1425 options
->refresh_period
= n
* 1000 * TIMER_MILLISECOND
;
1430 /** Generate default options for application */
1432 l2fwd_crypto_default_options(struct l2fwd_crypto_options
*options
)
1434 options
->portmask
= 0xffffffff;
1435 options
->nb_ports_per_lcore
= 1;
1436 options
->refresh_period
= 10000;
1437 options
->single_lcore
= 0;
1438 options
->sessionless
= 0;
1440 options
->xform_chain
= L2FWD_CRYPTO_CIPHER_HASH
;
1443 options
->cipher_xform
.type
= RTE_CRYPTO_SYM_XFORM_CIPHER
;
1444 options
->cipher_xform
.next
= NULL
;
1445 options
->ckey_param
= 0;
1446 options
->ckey_random_size
= -1;
1447 options
->cipher_xform
.cipher
.key
.length
= 0;
1448 options
->cipher_iv_param
= 0;
1449 options
->cipher_iv_random_size
= -1;
1450 options
->cipher_iv
.length
= 0;
1452 options
->cipher_xform
.cipher
.algo
= RTE_CRYPTO_CIPHER_AES_CBC
;
1453 options
->cipher_xform
.cipher
.op
= RTE_CRYPTO_CIPHER_OP_ENCRYPT
;
1455 /* Authentication Data */
1456 options
->auth_xform
.type
= RTE_CRYPTO_SYM_XFORM_AUTH
;
1457 options
->auth_xform
.next
= NULL
;
1458 options
->akey_param
= 0;
1459 options
->akey_random_size
= -1;
1460 options
->auth_xform
.auth
.key
.length
= 0;
1461 options
->auth_iv_param
= 0;
1462 options
->auth_iv_random_size
= -1;
1463 options
->auth_iv
.length
= 0;
1465 options
->auth_xform
.auth
.algo
= RTE_CRYPTO_AUTH_SHA1_HMAC
;
1466 options
->auth_xform
.auth
.op
= RTE_CRYPTO_AUTH_OP_GENERATE
;
1469 options
->aead_xform
.type
= RTE_CRYPTO_SYM_XFORM_AEAD
;
1470 options
->aead_xform
.next
= NULL
;
1471 options
->aead_key_param
= 0;
1472 options
->aead_key_random_size
= -1;
1473 options
->aead_xform
.aead
.key
.length
= 0;
1474 options
->aead_iv_param
= 0;
1475 options
->aead_iv_random_size
= -1;
1476 options
->aead_iv
.length
= 0;
1478 options
->aead_xform
.aead
.algo
= RTE_CRYPTO_AEAD_AES_GCM
;
1479 options
->aead_xform
.aead
.op
= RTE_CRYPTO_AEAD_OP_ENCRYPT
;
1481 options
->aad_param
= 0;
1482 options
->aad_random_size
= -1;
1483 options
->aad
.length
= 0;
1485 options
->digest_size
= -1;
1487 options
->type
= CDEV_TYPE_ANY
;
1488 options
->cryptodev_mask
= UINT64_MAX
;
1490 options
->mac_updating
= 1;
1494 display_cipher_info(struct l2fwd_crypto_options
*options
)
1496 printf("\n---- Cipher information ---\n");
1497 printf("Algorithm: %s\n",
1498 rte_crypto_cipher_algorithm_strings
[options
->cipher_xform
.cipher
.algo
]);
1499 rte_hexdump(stdout
, "Cipher key:",
1500 options
->cipher_xform
.cipher
.key
.data
,
1501 options
->cipher_xform
.cipher
.key
.length
);
1502 rte_hexdump(stdout
, "IV:", options
->cipher_iv
.data
, options
->cipher_iv
.length
);
1506 display_auth_info(struct l2fwd_crypto_options
*options
)
1508 printf("\n---- Authentication information ---\n");
1509 printf("Algorithm: %s\n",
1510 rte_crypto_auth_algorithm_strings
[options
->auth_xform
.auth
.algo
]);
1511 rte_hexdump(stdout
, "Auth key:",
1512 options
->auth_xform
.auth
.key
.data
,
1513 options
->auth_xform
.auth
.key
.length
);
1514 rte_hexdump(stdout
, "IV:", options
->auth_iv
.data
, options
->auth_iv
.length
);
1518 display_aead_info(struct l2fwd_crypto_options
*options
)
1520 printf("\n---- AEAD information ---\n");
1521 printf("Algorithm: %s\n",
1522 rte_crypto_aead_algorithm_strings
[options
->aead_xform
.aead
.algo
]);
1523 rte_hexdump(stdout
, "AEAD key:",
1524 options
->aead_xform
.aead
.key
.data
,
1525 options
->aead_xform
.aead
.key
.length
);
1526 rte_hexdump(stdout
, "IV:", options
->aead_iv
.data
, options
->aead_iv
.length
);
1527 rte_hexdump(stdout
, "AAD:", options
->aad
.data
, options
->aad
.length
);
1531 l2fwd_crypto_options_print(struct l2fwd_crypto_options
*options
)
1533 char string_cipher_op
[MAX_STR_LEN
];
1534 char string_auth_op
[MAX_STR_LEN
];
1535 char string_aead_op
[MAX_STR_LEN
];
1537 if (options
->cipher_xform
.cipher
.op
== RTE_CRYPTO_CIPHER_OP_ENCRYPT
)
1538 strcpy(string_cipher_op
, "Encrypt");
1540 strcpy(string_cipher_op
, "Decrypt");
1542 if (options
->auth_xform
.auth
.op
== RTE_CRYPTO_AUTH_OP_GENERATE
)
1543 strcpy(string_auth_op
, "Auth generate");
1545 strcpy(string_auth_op
, "Auth verify");
1547 if (options
->aead_xform
.aead
.op
== RTE_CRYPTO_AEAD_OP_ENCRYPT
)
1548 strcpy(string_aead_op
, "Authenticated encryption");
1550 strcpy(string_aead_op
, "Authenticated decryption");
1553 printf("Options:-\nn");
1554 printf("portmask: %x\n", options
->portmask
);
1555 printf("ports per lcore: %u\n", options
->nb_ports_per_lcore
);
1556 printf("refresh period : %u\n", options
->refresh_period
);
1557 printf("single lcore mode: %s\n",
1558 options
->single_lcore
? "enabled" : "disabled");
1559 printf("stats_printing: %s\n",
1560 options
->refresh_period
== 0 ? "disabled" : "enabled");
1562 printf("sessionless crypto: %s\n",
1563 options
->sessionless
? "enabled" : "disabled");
1565 if (options
->ckey_param
&& (options
->ckey_random_size
!= -1))
1566 printf("Cipher key already parsed, ignoring size of random key\n");
1568 if (options
->akey_param
&& (options
->akey_random_size
!= -1))
1569 printf("Auth key already parsed, ignoring size of random key\n");
1571 if (options
->cipher_iv_param
&& (options
->cipher_iv_random_size
!= -1))
1572 printf("Cipher IV already parsed, ignoring size of random IV\n");
1574 if (options
->auth_iv_param
&& (options
->auth_iv_random_size
!= -1))
1575 printf("Auth IV already parsed, ignoring size of random IV\n");
1577 if (options
->aad_param
&& (options
->aad_random_size
!= -1))
1578 printf("AAD already parsed, ignoring size of random AAD\n");
1580 printf("\nCrypto chain: ");
1581 switch (options
->xform_chain
) {
1582 case L2FWD_CRYPTO_AEAD
:
1583 printf("Input --> %s --> Output\n", string_aead_op
);
1584 display_aead_info(options
);
1586 case L2FWD_CRYPTO_CIPHER_HASH
:
1587 printf("Input --> %s --> %s --> Output\n",
1588 string_cipher_op
, string_auth_op
);
1589 display_cipher_info(options
);
1590 display_auth_info(options
);
1592 case L2FWD_CRYPTO_HASH_CIPHER
:
1593 printf("Input --> %s --> %s --> Output\n",
1594 string_auth_op
, string_cipher_op
);
1595 display_cipher_info(options
);
1596 display_auth_info(options
);
1598 case L2FWD_CRYPTO_HASH_ONLY
:
1599 printf("Input --> %s --> Output\n", string_auth_op
);
1600 display_auth_info(options
);
1602 case L2FWD_CRYPTO_CIPHER_ONLY
:
1603 printf("Input --> %s --> Output\n", string_cipher_op
);
1604 display_cipher_info(options
);
1609 /* Parse the argument given in the command line of the application */
1611 l2fwd_crypto_parse_args(struct l2fwd_crypto_options
*options
,
1612 int argc
, char **argv
)
1614 int opt
, retval
, option_index
;
1615 char **argvopt
= argv
, *prgname
= argv
[0];
1617 static struct option lgopts
[] = {
1618 { "sessionless", no_argument
, 0, 0 },
1620 { "cdev_type", required_argument
, 0, 0 },
1621 { "chain", required_argument
, 0, 0 },
1623 { "cipher_algo", required_argument
, 0, 0 },
1624 { "cipher_op", required_argument
, 0, 0 },
1625 { "cipher_key", required_argument
, 0, 0 },
1626 { "cipher_key_random_size", required_argument
, 0, 0 },
1627 { "cipher_iv", required_argument
, 0, 0 },
1628 { "cipher_iv_random_size", required_argument
, 0, 0 },
1630 { "auth_algo", required_argument
, 0, 0 },
1631 { "auth_op", required_argument
, 0, 0 },
1632 { "auth_key", required_argument
, 0, 0 },
1633 { "auth_key_random_size", required_argument
, 0, 0 },
1634 { "auth_iv", required_argument
, 0, 0 },
1635 { "auth_iv_random_size", required_argument
, 0, 0 },
1637 { "aead_algo", required_argument
, 0, 0 },
1638 { "aead_op", required_argument
, 0, 0 },
1639 { "aead_key", required_argument
, 0, 0 },
1640 { "aead_key_random_size", required_argument
, 0, 0 },
1641 { "aead_iv", required_argument
, 0, 0 },
1642 { "aead_iv_random_size", required_argument
, 0, 0 },
1644 { "aad", required_argument
, 0, 0 },
1645 { "aad_random_size", required_argument
, 0, 0 },
1647 { "digest_size", required_argument
, 0, 0 },
1649 { "sessionless", no_argument
, 0, 0 },
1650 { "cryptodev_mask", required_argument
, 0, 0},
1652 { "mac-updating", no_argument
, 0, 0},
1653 { "no-mac-updating", no_argument
, 0, 0},
1658 l2fwd_crypto_default_options(options
);
1660 while ((opt
= getopt_long(argc
, argvopt
, "p:q:sT:", lgopts
,
1661 &option_index
)) != EOF
) {
1665 retval
= l2fwd_crypto_parse_args_long_options(options
,
1666 lgopts
, option_index
);
1668 l2fwd_crypto_usage(prgname
);
1675 retval
= l2fwd_crypto_parse_portmask(options
, optarg
);
1677 l2fwd_crypto_usage(prgname
);
1684 retval
= l2fwd_crypto_parse_nqueue(options
, optarg
);
1686 l2fwd_crypto_usage(prgname
);
1693 options
->single_lcore
= 1;
1699 retval
= l2fwd_crypto_parse_timer_period(options
,
1702 l2fwd_crypto_usage(prgname
);
1708 l2fwd_crypto_usage(prgname
);
1715 argv
[optind
-1] = prgname
;
1718 optind
= 1; /* reset getopt lib */
1723 /* Check the link status of all ports in up to 9s, and print them finally */
1725 check_all_ports_link_status(uint32_t port_mask
)
1727 #define CHECK_INTERVAL 100 /* 100ms */
1728 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1730 uint8_t count
, all_ports_up
, print_flag
= 0;
1731 struct rte_eth_link link
;
1733 printf("\nChecking link status");
1735 for (count
= 0; count
<= MAX_CHECK_TIME
; count
++) {
1737 RTE_ETH_FOREACH_DEV(portid
) {
1738 if ((port_mask
& (1 << portid
)) == 0)
1740 memset(&link
, 0, sizeof(link
));
1741 rte_eth_link_get_nowait(portid
, &link
);
1742 /* print link status if flag set */
1743 if (print_flag
== 1) {
1744 if (link
.link_status
)
1746 "Port%d Link Up. Speed %u Mbps - %s\n",
1747 portid
, link
.link_speed
,
1748 (link
.link_duplex
== ETH_LINK_FULL_DUPLEX
) ?
1749 ("full-duplex") : ("half-duplex\n"));
1751 printf("Port %d Link Down\n", portid
);
1754 /* clear all_ports_up flag if any link down */
1755 if (link
.link_status
== ETH_LINK_DOWN
) {
1760 /* after finally printing all link status, get out */
1761 if (print_flag
== 1)
1764 if (all_ports_up
== 0) {
1767 rte_delay_ms(CHECK_INTERVAL
);
1770 /* set the print_flag if all ports up or timeout */
1771 if (all_ports_up
== 1 || count
== (MAX_CHECK_TIME
- 1)) {
1778 /* Check if device has to be HW/SW or any */
1780 check_type(const struct l2fwd_crypto_options
*options
,
1781 const struct rte_cryptodev_info
*dev_info
)
1783 if (options
->type
== CDEV_TYPE_HW
&&
1784 (dev_info
->feature_flags
& RTE_CRYPTODEV_FF_HW_ACCELERATED
))
1786 if (options
->type
== CDEV_TYPE_SW
&&
1787 !(dev_info
->feature_flags
& RTE_CRYPTODEV_FF_HW_ACCELERATED
))
1789 if (options
->type
== CDEV_TYPE_ANY
)
1795 static const struct rte_cryptodev_capabilities
*
1796 check_device_support_cipher_algo(const struct l2fwd_crypto_options
*options
,
1797 const struct rte_cryptodev_info
*dev_info
,
1801 const struct rte_cryptodev_capabilities
*cap
= &dev_info
->capabilities
[0];
1802 enum rte_crypto_cipher_algorithm cap_cipher_algo
;
1803 enum rte_crypto_cipher_algorithm opt_cipher_algo
=
1804 options
->cipher_xform
.cipher
.algo
;
1806 while (cap
->op
!= RTE_CRYPTO_OP_TYPE_UNDEFINED
) {
1807 cap_cipher_algo
= cap
->sym
.cipher
.algo
;
1808 if (cap
->sym
.xform_type
== RTE_CRYPTO_SYM_XFORM_CIPHER
) {
1809 if (cap_cipher_algo
== opt_cipher_algo
) {
1810 if (check_type(options
, dev_info
) == 0)
1814 cap
= &dev_info
->capabilities
[++i
];
1817 if (cap
->op
== RTE_CRYPTO_OP_TYPE_UNDEFINED
) {
1818 printf("Algorithm %s not supported by cryptodev %u"
1819 " or device not of preferred type (%s)\n",
1820 rte_crypto_cipher_algorithm_strings
[opt_cipher_algo
],
1822 options
->string_type
);
1829 static const struct rte_cryptodev_capabilities
*
1830 check_device_support_auth_algo(const struct l2fwd_crypto_options
*options
,
1831 const struct rte_cryptodev_info
*dev_info
,
1835 const struct rte_cryptodev_capabilities
*cap
= &dev_info
->capabilities
[0];
1836 enum rte_crypto_auth_algorithm cap_auth_algo
;
1837 enum rte_crypto_auth_algorithm opt_auth_algo
=
1838 options
->auth_xform
.auth
.algo
;
1840 while (cap
->op
!= RTE_CRYPTO_OP_TYPE_UNDEFINED
) {
1841 cap_auth_algo
= cap
->sym
.auth
.algo
;
1842 if (cap
->sym
.xform_type
== RTE_CRYPTO_SYM_XFORM_AUTH
) {
1843 if (cap_auth_algo
== opt_auth_algo
) {
1844 if (check_type(options
, dev_info
) == 0)
1848 cap
= &dev_info
->capabilities
[++i
];
1851 if (cap
->op
== RTE_CRYPTO_OP_TYPE_UNDEFINED
) {
1852 printf("Algorithm %s not supported by cryptodev %u"
1853 " or device not of preferred type (%s)\n",
1854 rte_crypto_auth_algorithm_strings
[opt_auth_algo
],
1856 options
->string_type
);
1863 static const struct rte_cryptodev_capabilities
*
1864 check_device_support_aead_algo(const struct l2fwd_crypto_options
*options
,
1865 const struct rte_cryptodev_info
*dev_info
,
1869 const struct rte_cryptodev_capabilities
*cap
= &dev_info
->capabilities
[0];
1870 enum rte_crypto_aead_algorithm cap_aead_algo
;
1871 enum rte_crypto_aead_algorithm opt_aead_algo
=
1872 options
->aead_xform
.aead
.algo
;
1874 while (cap
->op
!= RTE_CRYPTO_OP_TYPE_UNDEFINED
) {
1875 cap_aead_algo
= cap
->sym
.aead
.algo
;
1876 if (cap
->sym
.xform_type
== RTE_CRYPTO_SYM_XFORM_AEAD
) {
1877 if (cap_aead_algo
== opt_aead_algo
) {
1878 if (check_type(options
, dev_info
) == 0)
1882 cap
= &dev_info
->capabilities
[++i
];
1885 if (cap
->op
== RTE_CRYPTO_OP_TYPE_UNDEFINED
) {
1886 printf("Algorithm %s not supported by cryptodev %u"
1887 " or device not of preferred type (%s)\n",
1888 rte_crypto_aead_algorithm_strings
[opt_aead_algo
],
1890 options
->string_type
);
1897 /* Check if the device is enabled by cryptodev_mask */
1899 check_cryptodev_mask(struct l2fwd_crypto_options
*options
,
1902 if (options
->cryptodev_mask
& (1 << cdev_id
))
1909 check_supported_size(uint16_t length
, uint16_t min
, uint16_t max
,
1915 if (increment
== 0) {
1922 /* Range of values */
1923 for (supp_size
= min
; supp_size
<= max
; supp_size
+= increment
) {
1924 if (length
== supp_size
)
1932 check_iv_param(const struct rte_crypto_param_range
*iv_range_size
,
1933 unsigned int iv_param
, int iv_random_size
,
1937 * Check if length of provided IV is supported
1938 * by the algorithm chosen.
1941 if (check_supported_size(iv_length
,
1944 iv_range_size
->increment
)
1948 * Check if length of IV to be randomly generated
1949 * is supported by the algorithm chosen.
1951 } else if (iv_random_size
!= -1) {
1952 if (check_supported_size(iv_random_size
,
1955 iv_range_size
->increment
)
1964 check_capabilities(struct l2fwd_crypto_options
*options
, uint8_t cdev_id
)
1966 struct rte_cryptodev_info dev_info
;
1967 const struct rte_cryptodev_capabilities
*cap
;
1969 rte_cryptodev_info_get(cdev_id
, &dev_info
);
1971 /* Set AEAD parameters */
1972 if (options
->xform_chain
== L2FWD_CRYPTO_AEAD
) {
1973 /* Check if device supports AEAD algo */
1974 cap
= check_device_support_aead_algo(options
, &dev_info
,
1979 if (check_iv_param(&cap
->sym
.aead
.iv_size
,
1980 options
->aead_iv_param
,
1981 options
->aead_iv_random_size
,
1982 options
->aead_iv
.length
) != 0) {
1983 RTE_LOG(DEBUG
, USER1
,
1984 "Device %u does not support IV length\n",
1990 * Check if length of provided AEAD key is supported
1991 * by the algorithm chosen.
1993 if (options
->aead_key_param
) {
1994 if (check_supported_size(
1995 options
->aead_xform
.aead
.key
.length
,
1996 cap
->sym
.aead
.key_size
.min
,
1997 cap
->sym
.aead
.key_size
.max
,
1998 cap
->sym
.aead
.key_size
.increment
)
2000 RTE_LOG(DEBUG
, USER1
,
2001 "Device %u does not support "
2002 "AEAD key length\n",
2007 * Check if length of the aead key to be randomly generated
2008 * is supported by the algorithm chosen.
2010 } else if (options
->aead_key_random_size
!= -1) {
2011 if (check_supported_size(options
->aead_key_random_size
,
2012 cap
->sym
.aead
.key_size
.min
,
2013 cap
->sym
.aead
.key_size
.max
,
2014 cap
->sym
.aead
.key_size
.increment
)
2016 RTE_LOG(DEBUG
, USER1
,
2017 "Device %u does not support "
2018 "AEAD key length\n",
2026 * Check if length of provided AAD is supported
2027 * by the algorithm chosen.
2029 if (options
->aad_param
) {
2030 if (check_supported_size(options
->aad
.length
,
2031 cap
->sym
.aead
.aad_size
.min
,
2032 cap
->sym
.aead
.aad_size
.max
,
2033 cap
->sym
.aead
.aad_size
.increment
)
2035 RTE_LOG(DEBUG
, USER1
,
2036 "Device %u does not support "
2042 * Check if length of AAD to be randomly generated
2043 * is supported by the algorithm chosen.
2045 } else if (options
->aad_random_size
!= -1) {
2046 if (check_supported_size(options
->aad_random_size
,
2047 cap
->sym
.aead
.aad_size
.min
,
2048 cap
->sym
.aead
.aad_size
.max
,
2049 cap
->sym
.aead
.aad_size
.increment
)
2051 RTE_LOG(DEBUG
, USER1
,
2052 "Device %u does not support "
2059 /* Check if digest size is supported by the algorithm. */
2060 if (options
->digest_size
!= -1) {
2061 if (check_supported_size(options
->digest_size
,
2062 cap
->sym
.aead
.digest_size
.min
,
2063 cap
->sym
.aead
.digest_size
.max
,
2064 cap
->sym
.aead
.digest_size
.increment
)
2066 RTE_LOG(DEBUG
, USER1
,
2067 "Device %u does not support "
2075 /* Set cipher parameters */
2076 if (options
->xform_chain
== L2FWD_CRYPTO_CIPHER_HASH
||
2077 options
->xform_chain
== L2FWD_CRYPTO_HASH_CIPHER
||
2078 options
->xform_chain
== L2FWD_CRYPTO_CIPHER_ONLY
) {
2079 /* Check if device supports cipher algo */
2080 cap
= check_device_support_cipher_algo(options
, &dev_info
,
2085 if (check_iv_param(&cap
->sym
.cipher
.iv_size
,
2086 options
->cipher_iv_param
,
2087 options
->cipher_iv_random_size
,
2088 options
->cipher_iv
.length
) != 0) {
2089 RTE_LOG(DEBUG
, USER1
,
2090 "Device %u does not support IV length\n",
2096 * Check if length of provided cipher key is supported
2097 * by the algorithm chosen.
2099 if (options
->ckey_param
) {
2100 if (check_supported_size(
2101 options
->cipher_xform
.cipher
.key
.length
,
2102 cap
->sym
.cipher
.key_size
.min
,
2103 cap
->sym
.cipher
.key_size
.max
,
2104 cap
->sym
.cipher
.key_size
.increment
)
2106 RTE_LOG(DEBUG
, USER1
,
2107 "Device %u does not support cipher "
2113 * Check if length of the cipher key to be randomly generated
2114 * is supported by the algorithm chosen.
2116 } else if (options
->ckey_random_size
!= -1) {
2117 if (check_supported_size(options
->ckey_random_size
,
2118 cap
->sym
.cipher
.key_size
.min
,
2119 cap
->sym
.cipher
.key_size
.max
,
2120 cap
->sym
.cipher
.key_size
.increment
)
2122 RTE_LOG(DEBUG
, USER1
,
2123 "Device %u does not support cipher "
2131 /* Set auth parameters */
2132 if (options
->xform_chain
== L2FWD_CRYPTO_CIPHER_HASH
||
2133 options
->xform_chain
== L2FWD_CRYPTO_HASH_CIPHER
||
2134 options
->xform_chain
== L2FWD_CRYPTO_HASH_ONLY
) {
2135 /* Check if device supports auth algo */
2136 cap
= check_device_support_auth_algo(options
, &dev_info
,
2141 if (check_iv_param(&cap
->sym
.auth
.iv_size
,
2142 options
->auth_iv_param
,
2143 options
->auth_iv_random_size
,
2144 options
->auth_iv
.length
) != 0) {
2145 RTE_LOG(DEBUG
, USER1
,
2146 "Device %u does not support IV length\n",
2151 * Check if length of provided auth key is supported
2152 * by the algorithm chosen.
2154 if (options
->akey_param
) {
2155 if (check_supported_size(
2156 options
->auth_xform
.auth
.key
.length
,
2157 cap
->sym
.auth
.key_size
.min
,
2158 cap
->sym
.auth
.key_size
.max
,
2159 cap
->sym
.auth
.key_size
.increment
)
2161 RTE_LOG(DEBUG
, USER1
,
2162 "Device %u does not support auth "
2168 * Check if length of the auth key to be randomly generated
2169 * is supported by the algorithm chosen.
2171 } else if (options
->akey_random_size
!= -1) {
2172 if (check_supported_size(options
->akey_random_size
,
2173 cap
->sym
.auth
.key_size
.min
,
2174 cap
->sym
.auth
.key_size
.max
,
2175 cap
->sym
.auth
.key_size
.increment
)
2177 RTE_LOG(DEBUG
, USER1
,
2178 "Device %u does not support auth "
2185 /* Check if digest size is supported by the algorithm. */
2186 if (options
->digest_size
!= -1) {
2187 if (check_supported_size(options
->digest_size
,
2188 cap
->sym
.auth
.digest_size
.min
,
2189 cap
->sym
.auth
.digest_size
.max
,
2190 cap
->sym
.auth
.digest_size
.increment
)
2192 RTE_LOG(DEBUG
, USER1
,
2193 "Device %u does not support "
2205 initialize_cryptodevs(struct l2fwd_crypto_options
*options
, unsigned nb_ports
,
2206 uint8_t *enabled_cdevs
)
2208 uint8_t cdev_id
, cdev_count
, enabled_cdev_count
= 0;
2209 const struct rte_cryptodev_capabilities
*cap
;
2210 unsigned int sess_sz
, max_sess_sz
= 0;
2211 uint32_t sessions_needed
= 0;
2214 cdev_count
= rte_cryptodev_count();
2215 if (cdev_count
== 0) {
2216 printf("No crypto devices available\n");
2220 for (cdev_id
= 0; cdev_id
< cdev_count
&& enabled_cdev_count
< nb_ports
;
2222 if (check_cryptodev_mask(options
, cdev_id
) < 0)
2225 if (check_capabilities(options
, cdev_id
) < 0)
2228 sess_sz
= rte_cryptodev_sym_get_private_session_size(cdev_id
);
2229 if (sess_sz
> max_sess_sz
)
2230 max_sess_sz
= sess_sz
;
2232 l2fwd_enabled_crypto_mask
|= (((uint64_t)1) << cdev_id
);
2234 enabled_cdevs
[cdev_id
] = 1;
2235 enabled_cdev_count
++;
2238 for (cdev_id
= 0; cdev_id
< cdev_count
; cdev_id
++) {
2239 struct rte_cryptodev_qp_conf qp_conf
;
2240 struct rte_cryptodev_info dev_info
;
2242 if (enabled_cdevs
[cdev_id
] == 0)
2245 retval
= rte_cryptodev_socket_id(cdev_id
);
2248 printf("Invalid crypto device id used\n");
2252 uint8_t socket_id
= (uint8_t) retval
;
2254 struct rte_cryptodev_config conf
= {
2255 .nb_queue_pairs
= 1,
2256 .socket_id
= socket_id
,
2259 rte_cryptodev_info_get(cdev_id
, &dev_info
);
2262 * Two sessions objects are required for each session
2263 * (one for the header, one for the private data)
2265 if (!strcmp(dev_info
.driver_name
, "crypto_scheduler")) {
2266 #ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER
2267 uint32_t nb_slaves
=
2268 rte_cryptodev_scheduler_slaves_get(cdev_id
,
2271 sessions_needed
= 2 * enabled_cdev_count
* nb_slaves
;
2274 sessions_needed
= 2 * enabled_cdev_count
;
2276 if (session_pool_socket
[socket_id
] == NULL
) {
2277 char mp_name
[RTE_MEMPOOL_NAMESIZE
];
2278 struct rte_mempool
*sess_mp
;
2280 snprintf(mp_name
, RTE_MEMPOOL_NAMESIZE
,
2281 "sess_mp_%u", socket_id
);
2284 * Create enough objects for session headers and
2285 * device private data
2287 sess_mp
= rte_mempool_create(mp_name
,
2290 SESSION_POOL_CACHE_SIZE
,
2291 0, NULL
, NULL
, NULL
,
2295 if (sess_mp
== NULL
) {
2296 printf("Cannot create session pool on socket %d\n",
2301 printf("Allocated session pool on socket %d\n", socket_id
);
2302 session_pool_socket
[socket_id
] = sess_mp
;
2305 /* Set AEAD parameters */
2306 if (options
->xform_chain
== L2FWD_CRYPTO_AEAD
) {
2307 cap
= check_device_support_aead_algo(options
, &dev_info
,
2310 options
->block_size
= cap
->sym
.aead
.block_size
;
2312 /* Set IV if not provided from command line */
2313 if (options
->aead_iv_param
== 0) {
2314 if (options
->aead_iv_random_size
!= -1)
2315 options
->aead_iv
.length
=
2316 options
->aead_iv_random_size
;
2317 /* No size provided, use minimum size. */
2319 options
->aead_iv
.length
=
2320 cap
->sym
.aead
.iv_size
.min
;
2323 /* Set key if not provided from command line */
2324 if (options
->aead_key_param
== 0) {
2325 if (options
->aead_key_random_size
!= -1)
2326 options
->aead_xform
.aead
.key
.length
=
2327 options
->aead_key_random_size
;
2328 /* No size provided, use minimum size. */
2330 options
->aead_xform
.aead
.key
.length
=
2331 cap
->sym
.aead
.key_size
.min
;
2333 generate_random_key(
2334 options
->aead_xform
.aead
.key
.data
,
2335 options
->aead_xform
.aead
.key
.length
);
2338 /* Set AAD if not provided from command line */
2339 if (options
->aad_param
== 0) {
2340 if (options
->aad_random_size
!= -1)
2341 options
->aad
.length
=
2342 options
->aad_random_size
;
2343 /* No size provided, use minimum size. */
2345 options
->aad
.length
=
2346 cap
->sym
.auth
.aad_size
.min
;
2349 options
->aead_xform
.aead
.aad_length
=
2350 options
->aad
.length
;
2352 /* Set digest size if not provided from command line */
2353 if (options
->digest_size
!= -1)
2354 options
->aead_xform
.aead
.digest_length
=
2355 options
->digest_size
;
2356 /* No size provided, use minimum size. */
2358 options
->aead_xform
.aead
.digest_length
=
2359 cap
->sym
.aead
.digest_size
.min
;
2362 /* Set cipher parameters */
2363 if (options
->xform_chain
== L2FWD_CRYPTO_CIPHER_HASH
||
2364 options
->xform_chain
== L2FWD_CRYPTO_HASH_CIPHER
||
2365 options
->xform_chain
== L2FWD_CRYPTO_CIPHER_ONLY
) {
2366 cap
= check_device_support_cipher_algo(options
, &dev_info
,
2368 options
->block_size
= cap
->sym
.cipher
.block_size
;
2370 /* Set IV if not provided from command line */
2371 if (options
->cipher_iv_param
== 0) {
2372 if (options
->cipher_iv_random_size
!= -1)
2373 options
->cipher_iv
.length
=
2374 options
->cipher_iv_random_size
;
2375 /* No size provided, use minimum size. */
2377 options
->cipher_iv
.length
=
2378 cap
->sym
.cipher
.iv_size
.min
;
2381 /* Set key if not provided from command line */
2382 if (options
->ckey_param
== 0) {
2383 if (options
->ckey_random_size
!= -1)
2384 options
->cipher_xform
.cipher
.key
.length
=
2385 options
->ckey_random_size
;
2386 /* No size provided, use minimum size. */
2388 options
->cipher_xform
.cipher
.key
.length
=
2389 cap
->sym
.cipher
.key_size
.min
;
2391 generate_random_key(
2392 options
->cipher_xform
.cipher
.key
.data
,
2393 options
->cipher_xform
.cipher
.key
.length
);
2397 /* Set auth parameters */
2398 if (options
->xform_chain
== L2FWD_CRYPTO_CIPHER_HASH
||
2399 options
->xform_chain
== L2FWD_CRYPTO_HASH_CIPHER
||
2400 options
->xform_chain
== L2FWD_CRYPTO_HASH_ONLY
) {
2401 cap
= check_device_support_auth_algo(options
, &dev_info
,
2404 /* Set IV if not provided from command line */
2405 if (options
->auth_iv_param
== 0) {
2406 if (options
->auth_iv_random_size
!= -1)
2407 options
->auth_iv
.length
=
2408 options
->auth_iv_random_size
;
2409 /* No size provided, use minimum size. */
2411 options
->auth_iv
.length
=
2412 cap
->sym
.auth
.iv_size
.min
;
2415 /* Set key if not provided from command line */
2416 if (options
->akey_param
== 0) {
2417 if (options
->akey_random_size
!= -1)
2418 options
->auth_xform
.auth
.key
.length
=
2419 options
->akey_random_size
;
2420 /* No size provided, use minimum size. */
2422 options
->auth_xform
.auth
.key
.length
=
2423 cap
->sym
.auth
.key_size
.min
;
2425 generate_random_key(
2426 options
->auth_xform
.auth
.key
.data
,
2427 options
->auth_xform
.auth
.key
.length
);
2430 /* Set digest size if not provided from command line */
2431 if (options
->digest_size
!= -1)
2432 options
->auth_xform
.auth
.digest_length
=
2433 options
->digest_size
;
2434 /* No size provided, use minimum size. */
2436 options
->auth_xform
.auth
.digest_length
=
2437 cap
->sym
.auth
.digest_size
.min
;
2440 retval
= rte_cryptodev_configure(cdev_id
, &conf
);
2442 printf("Failed to configure cryptodev %u", cdev_id
);
2446 qp_conf
.nb_descriptors
= 2048;
2448 retval
= rte_cryptodev_queue_pair_setup(cdev_id
, 0, &qp_conf
,
2449 socket_id
, session_pool_socket
[socket_id
]);
2451 printf("Failed to setup queue pair %u on cryptodev %u",
2456 retval
= rte_cryptodev_start(cdev_id
);
2458 printf("Failed to start device %u: error %d\n",
2464 return enabled_cdev_count
;
2468 initialize_ports(struct l2fwd_crypto_options
*options
)
2470 uint16_t last_portid
= 0, portid
;
2471 unsigned enabled_portcount
= 0;
2472 unsigned nb_ports
= rte_eth_dev_count_avail();
2474 if (nb_ports
== 0) {
2475 printf("No Ethernet ports - bye\n");
2479 /* Reset l2fwd_dst_ports */
2480 for (portid
= 0; portid
< RTE_MAX_ETHPORTS
; portid
++)
2481 l2fwd_dst_ports
[portid
] = 0;
2483 RTE_ETH_FOREACH_DEV(portid
) {
2485 struct rte_eth_dev_info dev_info
;
2486 struct rte_eth_rxconf rxq_conf
;
2487 struct rte_eth_txconf txq_conf
;
2488 struct rte_eth_conf local_port_conf
= port_conf
;
2490 /* Skip ports that are not enabled */
2491 if ((options
->portmask
& (1 << portid
)) == 0)
2495 printf("Initializing port %u... ", portid
);
2497 rte_eth_dev_info_get(portid
, &dev_info
);
2498 if (dev_info
.tx_offload_capa
& DEV_TX_OFFLOAD_MBUF_FAST_FREE
)
2499 local_port_conf
.txmode
.offloads
|=
2500 DEV_TX_OFFLOAD_MBUF_FAST_FREE
;
2501 retval
= rte_eth_dev_configure(portid
, 1, 1, &local_port_conf
);
2503 printf("Cannot configure device: err=%d, port=%u\n",
2508 retval
= rte_eth_dev_adjust_nb_rx_tx_desc(portid
, &nb_rxd
,
2511 printf("Cannot adjust number of descriptors: err=%d, port=%u\n",
2516 /* init one RX queue */
2518 rxq_conf
= dev_info
.default_rxconf
;
2519 rxq_conf
.offloads
= local_port_conf
.rxmode
.offloads
;
2520 retval
= rte_eth_rx_queue_setup(portid
, 0, nb_rxd
,
2521 rte_eth_dev_socket_id(portid
),
2522 &rxq_conf
, l2fwd_pktmbuf_pool
);
2524 printf("rte_eth_rx_queue_setup:err=%d, port=%u\n",
2529 /* init one TX queue on each port */
2531 txq_conf
= dev_info
.default_txconf
;
2532 txq_conf
.offloads
= local_port_conf
.txmode
.offloads
;
2533 retval
= rte_eth_tx_queue_setup(portid
, 0, nb_txd
,
2534 rte_eth_dev_socket_id(portid
),
2537 printf("rte_eth_tx_queue_setup:err=%d, port=%u\n",
2544 retval
= rte_eth_dev_start(portid
);
2546 printf("rte_eth_dev_start:err=%d, port=%u\n",
2551 rte_eth_promiscuous_enable(portid
);
2553 rte_eth_macaddr_get(portid
, &l2fwd_ports_eth_addr
[portid
]);
2555 printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
2557 l2fwd_ports_eth_addr
[portid
].addr_bytes
[0],
2558 l2fwd_ports_eth_addr
[portid
].addr_bytes
[1],
2559 l2fwd_ports_eth_addr
[portid
].addr_bytes
[2],
2560 l2fwd_ports_eth_addr
[portid
].addr_bytes
[3],
2561 l2fwd_ports_eth_addr
[portid
].addr_bytes
[4],
2562 l2fwd_ports_eth_addr
[portid
].addr_bytes
[5]);
2564 /* initialize port stats */
2565 memset(&port_statistics
, 0, sizeof(port_statistics
));
2567 /* Setup port forwarding table */
2568 if (enabled_portcount
% 2) {
2569 l2fwd_dst_ports
[portid
] = last_portid
;
2570 l2fwd_dst_ports
[last_portid
] = portid
;
2572 last_portid
= portid
;
2575 l2fwd_enabled_port_mask
|= (1 << portid
);
2576 enabled_portcount
++;
2579 if (enabled_portcount
== 1) {
2580 l2fwd_dst_ports
[last_portid
] = last_portid
;
2581 } else if (enabled_portcount
% 2) {
2582 printf("odd number of ports in portmask- bye\n");
2586 check_all_ports_link_status(l2fwd_enabled_port_mask
);
2588 return enabled_portcount
;
2592 reserve_key_memory(struct l2fwd_crypto_options
*options
)
2594 options
->cipher_xform
.cipher
.key
.data
= rte_malloc("crypto key",
2596 if (options
->cipher_xform
.cipher
.key
.data
== NULL
)
2597 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for cipher key");
2599 options
->auth_xform
.auth
.key
.data
= rte_malloc("auth key",
2601 if (options
->auth_xform
.auth
.key
.data
== NULL
)
2602 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for auth key");
2604 options
->aead_xform
.aead
.key
.data
= rte_malloc("aead key",
2606 if (options
->aead_xform
.aead
.key
.data
== NULL
)
2607 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for AEAD key");
2609 options
->cipher_iv
.data
= rte_malloc("cipher iv", MAX_KEY_SIZE
, 0);
2610 if (options
->cipher_iv
.data
== NULL
)
2611 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for cipher IV");
2613 options
->auth_iv
.data
= rte_malloc("auth iv", MAX_KEY_SIZE
, 0);
2614 if (options
->auth_iv
.data
== NULL
)
2615 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for auth IV");
2617 options
->aead_iv
.data
= rte_malloc("aead_iv", MAX_KEY_SIZE
, 0);
2618 if (options
->aead_iv
.data
== NULL
)
2619 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for AEAD iv");
2621 options
->aad
.data
= rte_malloc("aad", MAX_KEY_SIZE
, 0);
2622 if (options
->aad
.data
== NULL
)
2623 rte_exit(EXIT_FAILURE
, "Failed to allocate memory for AAD");
2624 options
->aad
.phys_addr
= rte_malloc_virt2iova(options
->aad
.data
);
2628 main(int argc
, char **argv
)
2630 struct lcore_queue_conf
*qconf
= NULL
;
2631 struct l2fwd_crypto_options options
;
2633 uint8_t nb_cryptodevs
, cdev_id
;
2635 unsigned lcore_id
, rx_lcore_id
= 0;
2636 int ret
, enabled_cdevcount
, enabled_portcount
;
2637 uint8_t enabled_cdevs
[RTE_CRYPTO_MAX_DEVS
] = {0};
2640 ret
= rte_eal_init(argc
, argv
);
2642 rte_exit(EXIT_FAILURE
, "Invalid EAL arguments\n");
2646 /* reserve memory for Cipher/Auth key and IV */
2647 reserve_key_memory(&options
);
2649 /* parse application arguments (after the EAL ones) */
2650 ret
= l2fwd_crypto_parse_args(&options
, argc
, argv
);
2652 rte_exit(EXIT_FAILURE
, "Invalid L2FWD-CRYPTO arguments\n");
2654 printf("MAC updating %s\n",
2655 options
.mac_updating
? "enabled" : "disabled");
2657 /* create the mbuf pool */
2658 l2fwd_pktmbuf_pool
= rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF
, 512,
2659 sizeof(struct rte_crypto_op
),
2660 RTE_MBUF_DEFAULT_BUF_SIZE
, rte_socket_id());
2661 if (l2fwd_pktmbuf_pool
== NULL
)
2662 rte_exit(EXIT_FAILURE
, "Cannot create mbuf pool\n");
2664 /* create crypto op pool */
2665 l2fwd_crypto_op_pool
= rte_crypto_op_pool_create("crypto_op_pool",
2666 RTE_CRYPTO_OP_TYPE_SYMMETRIC
, NB_MBUF
, 128, MAXIMUM_IV_LENGTH
,
2668 if (l2fwd_crypto_op_pool
== NULL
)
2669 rte_exit(EXIT_FAILURE
, "Cannot create crypto op pool\n");
2671 /* Enable Ethernet ports */
2672 enabled_portcount
= initialize_ports(&options
);
2673 if (enabled_portcount
< 1)
2674 rte_exit(EXIT_FAILURE
, "Failed to initial Ethernet ports\n");
2676 /* Initialize the port/queue configuration of each logical core */
2677 RTE_ETH_FOREACH_DEV(portid
) {
2679 /* skip ports that are not enabled */
2680 if ((options
.portmask
& (1 << portid
)) == 0)
2683 if (options
.single_lcore
&& qconf
== NULL
) {
2684 while (rte_lcore_is_enabled(rx_lcore_id
) == 0) {
2686 if (rx_lcore_id
>= RTE_MAX_LCORE
)
2687 rte_exit(EXIT_FAILURE
,
2688 "Not enough cores\n");
2690 } else if (!options
.single_lcore
) {
2691 /* get the lcore_id for this port */
2692 while (rte_lcore_is_enabled(rx_lcore_id
) == 0 ||
2693 lcore_queue_conf
[rx_lcore_id
].nb_rx_ports
==
2694 options
.nb_ports_per_lcore
) {
2696 if (rx_lcore_id
>= RTE_MAX_LCORE
)
2697 rte_exit(EXIT_FAILURE
,
2698 "Not enough cores\n");
2702 /* Assigned a new logical core in the loop above. */
2703 if (qconf
!= &lcore_queue_conf
[rx_lcore_id
])
2704 qconf
= &lcore_queue_conf
[rx_lcore_id
];
2706 qconf
->rx_port_list
[qconf
->nb_rx_ports
] = portid
;
2707 qconf
->nb_rx_ports
++;
2709 printf("Lcore %u: RX port %u\n", rx_lcore_id
, portid
);
2712 /* Enable Crypto devices */
2713 enabled_cdevcount
= initialize_cryptodevs(&options
, enabled_portcount
,
2715 if (enabled_cdevcount
< 0)
2716 rte_exit(EXIT_FAILURE
, "Failed to initialize crypto devices\n");
2718 if (enabled_cdevcount
< enabled_portcount
)
2719 rte_exit(EXIT_FAILURE
, "Number of capable crypto devices (%d) "
2720 "has to be more or equal to number of ports (%d)\n",
2721 enabled_cdevcount
, enabled_portcount
);
2723 nb_cryptodevs
= rte_cryptodev_count();
2725 /* Initialize the port/cryptodev configuration of each logical core */
2726 for (rx_lcore_id
= 0, qconf
= NULL
, cdev_id
= 0;
2727 cdev_id
< nb_cryptodevs
&& enabled_cdevcount
;
2729 /* Crypto op not supported by crypto device */
2730 if (!enabled_cdevs
[cdev_id
])
2733 if (options
.single_lcore
&& qconf
== NULL
) {
2734 while (rte_lcore_is_enabled(rx_lcore_id
) == 0) {
2736 if (rx_lcore_id
>= RTE_MAX_LCORE
)
2737 rte_exit(EXIT_FAILURE
,
2738 "Not enough cores\n");
2740 } else if (!options
.single_lcore
) {
2741 /* get the lcore_id for this port */
2742 while (rte_lcore_is_enabled(rx_lcore_id
) == 0 ||
2743 lcore_queue_conf
[rx_lcore_id
].nb_crypto_devs
==
2744 options
.nb_ports_per_lcore
) {
2746 if (rx_lcore_id
>= RTE_MAX_LCORE
)
2747 rte_exit(EXIT_FAILURE
,
2748 "Not enough cores\n");
2752 /* Assigned a new logical core in the loop above. */
2753 if (qconf
!= &lcore_queue_conf
[rx_lcore_id
])
2754 qconf
= &lcore_queue_conf
[rx_lcore_id
];
2756 qconf
->cryptodev_list
[qconf
->nb_crypto_devs
] = cdev_id
;
2757 qconf
->nb_crypto_devs
++;
2759 enabled_cdevcount
--;
2761 printf("Lcore %u: cryptodev %u\n", rx_lcore_id
,
2765 /* launch per-lcore init on every lcore */
2766 rte_eal_mp_remote_launch(l2fwd_launch_one_lcore
, (void *)&options
,
2768 RTE_LCORE_FOREACH_SLAVE(lcore_id
) {
2769 if (rte_eal_wait_lcore(lcore_id
) < 0)