1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2017 Intel Corporation
5 #include <netinet/in.h>
6 #include <netinet/ip.h>
8 #include <rte_branch_prediction.h>
10 #include <rte_crypto.h>
11 #include <rte_security.h>
12 #include <rte_cryptodev.h>
13 #include <rte_ethdev.h>
21 set_ipsec_conf(struct ipsec_sa
*sa
, struct rte_security_ipsec_xform
*ipsec
)
23 if (ipsec
->mode
== RTE_SECURITY_IPSEC_SA_MODE_TUNNEL
) {
24 struct rte_security_ipsec_tunnel_param
*tunnel
=
26 if (sa
->flags
== IP4_TUNNEL
) {
28 RTE_SECURITY_IPSEC_TUNNEL_IPV4
;
29 tunnel
->ipv4
.ttl
= IPDEFTTL
;
31 memcpy((uint8_t *)&tunnel
->ipv4
.src_ip
,
32 (uint8_t *)&sa
->src
.ip
.ip4
, 4);
34 memcpy((uint8_t *)&tunnel
->ipv4
.dst_ip
,
35 (uint8_t *)&sa
->dst
.ip
.ip4
, 4);
37 /* TODO support for Transport and IPV6 tunnel */
39 ipsec
->esn_soft_limit
= IPSEC_OFFLOAD_ESN_SOFTLIMIT
;
43 create_session(struct ipsec_ctx
*ipsec_ctx
, struct ipsec_sa
*sa
)
45 struct rte_cryptodev_info cdev_info
;
46 unsigned long cdev_id_qp
= 0;
48 struct cdev_key key
= { 0 };
50 key
.lcore_id
= (uint8_t)rte_lcore_id();
52 key
.cipher_algo
= (uint8_t)sa
->cipher_algo
;
53 key
.auth_algo
= (uint8_t)sa
->auth_algo
;
54 key
.aead_algo
= (uint8_t)sa
->aead_algo
;
56 if (sa
->type
== RTE_SECURITY_ACTION_TYPE_NONE
) {
57 ret
= rte_hash_lookup_data(ipsec_ctx
->cdev_map
, &key
,
58 (void **)&cdev_id_qp
);
61 "No cryptodev: core %u, cipher_algo %u, "
62 "auth_algo %u, aead_algo %u\n",
71 RTE_LOG_DP(DEBUG
, IPSEC
, "Create session for SA spi %u on cryptodev "
72 "%u qp %u\n", sa
->spi
,
73 ipsec_ctx
->tbl
[cdev_id_qp
].id
,
74 ipsec_ctx
->tbl
[cdev_id_qp
].qp
);
76 if (sa
->type
!= RTE_SECURITY_ACTION_TYPE_NONE
) {
77 struct rte_security_session_conf sess_conf
= {
78 .action_type
= sa
->type
,
79 .protocol
= RTE_SECURITY_PROTOCOL_IPSEC
,
84 .direction
= sa
->direction
,
85 .proto
= RTE_SECURITY_IPSEC_SA_PROTO_ESP
,
86 .mode
= (sa
->flags
== IP4_TUNNEL
||
87 sa
->flags
== IP6_TUNNEL
) ?
88 RTE_SECURITY_IPSEC_SA_MODE_TUNNEL
:
89 RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT
,
91 .crypto_xform
= sa
->xforms
,
96 if (sa
->type
== RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL
) {
97 struct rte_security_ctx
*ctx
= (struct rte_security_ctx
*)
98 rte_cryptodev_get_sec_ctx(
99 ipsec_ctx
->tbl
[cdev_id_qp
].id
);
101 /* Set IPsec parameters in conf */
102 set_ipsec_conf(sa
, &(sess_conf
.ipsec
));
104 sa
->sec_session
= rte_security_session_create(ctx
,
105 &sess_conf
, ipsec_ctx
->session_priv_pool
);
106 if (sa
->sec_session
== NULL
) {
108 "SEC Session init failed: err: %d\n", ret
);
111 } else if (sa
->type
== RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO
) {
112 struct rte_flow_error err
;
113 struct rte_security_ctx
*ctx
= (struct rte_security_ctx
*)
114 rte_eth_dev_get_sec_ctx(
116 const struct rte_security_capability
*sec_cap
;
119 sa
->sec_session
= rte_security_session_create(ctx
,
120 &sess_conf
, ipsec_ctx
->session_priv_pool
);
121 if (sa
->sec_session
== NULL
) {
123 "SEC Session init failed: err: %d\n", ret
);
127 sec_cap
= rte_security_capabilities_get(ctx
);
129 /* iterate until ESP tunnel*/
130 while (sec_cap
->action
!=
131 RTE_SECURITY_ACTION_TYPE_NONE
) {
133 if (sec_cap
->action
== sa
->type
&&
135 RTE_SECURITY_PROTOCOL_IPSEC
&&
136 sec_cap
->ipsec
.mode
==
137 RTE_SECURITY_IPSEC_SA_MODE_TUNNEL
&&
138 sec_cap
->ipsec
.direction
== sa
->direction
)
143 if (sec_cap
->action
== RTE_SECURITY_ACTION_TYPE_NONE
) {
145 "No suitable security capability found\n");
149 sa
->ol_flags
= sec_cap
->ol_flags
;
150 sa
->security_ctx
= ctx
;
151 sa
->pattern
[0].type
= RTE_FLOW_ITEM_TYPE_ETH
;
153 sa
->pattern
[1].type
= RTE_FLOW_ITEM_TYPE_IPV4
;
154 sa
->pattern
[1].mask
= &rte_flow_item_ipv4_mask
;
155 if (sa
->flags
& IP6_TUNNEL
) {
156 sa
->pattern
[1].spec
= &sa
->ipv6_spec
;
157 memcpy(sa
->ipv6_spec
.hdr
.dst_addr
,
158 sa
->dst
.ip
.ip6
.ip6_b
, 16);
159 memcpy(sa
->ipv6_spec
.hdr
.src_addr
,
160 sa
->src
.ip
.ip6
.ip6_b
, 16);
162 sa
->pattern
[1].spec
= &sa
->ipv4_spec
;
163 sa
->ipv4_spec
.hdr
.dst_addr
= sa
->dst
.ip
.ip4
;
164 sa
->ipv4_spec
.hdr
.src_addr
= sa
->src
.ip
.ip4
;
167 sa
->pattern
[2].type
= RTE_FLOW_ITEM_TYPE_ESP
;
168 sa
->pattern
[2].spec
= &sa
->esp_spec
;
169 sa
->pattern
[2].mask
= &rte_flow_item_esp_mask
;
170 sa
->esp_spec
.hdr
.spi
= rte_cpu_to_be_32(sa
->spi
);
172 sa
->pattern
[3].type
= RTE_FLOW_ITEM_TYPE_END
;
174 sa
->action
[0].type
= RTE_FLOW_ACTION_TYPE_SECURITY
;
175 sa
->action
[0].conf
= sa
->sec_session
;
177 sa
->action
[1].type
= RTE_FLOW_ACTION_TYPE_END
;
179 sa
->attr
.egress
= (sa
->direction
==
180 RTE_SECURITY_IPSEC_SA_DIR_EGRESS
);
181 sa
->attr
.ingress
= (sa
->direction
==
182 RTE_SECURITY_IPSEC_SA_DIR_INGRESS
);
183 if (sa
->attr
.ingress
) {
185 struct rte_eth_rss_conf rss_conf
= {
189 struct rte_eth_dev
*eth_dev
;
190 uint16_t queue
[RTE_MAX_QUEUES_PER_PORT
];
191 struct rte_flow_action_rss action_rss
;
195 sa
->action
[2].type
= RTE_FLOW_ACTION_TYPE_END
;
197 sa
->action
[1].type
= RTE_FLOW_ACTION_TYPE_RSS
;
198 sa
->action
[1].conf
= &action_rss
;
199 eth_dev
= ctx
->device
;
200 rte_eth_dev_rss_hash_conf_get(sa
->portid
,
203 i
< eth_dev
->data
->nb_rx_queues
; ++i
)
204 if (eth_dev
->data
->rx_queues
[i
])
206 action_rss
= (struct rte_flow_action_rss
){
207 .types
= rss_conf
.rss_hf
,
208 .key_len
= rss_conf
.rss_key_len
,
213 ret
= rte_flow_validate(sa
->portid
, &sa
->attr
,
214 sa
->pattern
, sa
->action
,
219 sa
->action
[1].type
= RTE_FLOW_ACTION_TYPE_QUEUE
;
221 &(struct rte_flow_action_queue
){
224 ret
= rte_flow_validate(sa
->portid
, &sa
->attr
,
225 sa
->pattern
, sa
->action
,
228 sa
->action
[1].type
= RTE_FLOW_ACTION_TYPE_END
;
229 sa
->action
[1].conf
= NULL
;
230 ret
= rte_flow_validate(sa
->portid
, &sa
->attr
,
231 sa
->pattern
, sa
->action
,
234 goto flow_create_failure
;
235 } else if (sa
->attr
.egress
&&
237 RTE_SECURITY_TX_HW_TRAILER_OFFLOAD
)) {
239 RTE_FLOW_ACTION_TYPE_PASSTHRU
;
241 RTE_FLOW_ACTION_TYPE_END
;
244 sa
->flow
= rte_flow_create(sa
->portid
,
245 &sa
->attr
, sa
->pattern
, sa
->action
, &err
);
246 if (sa
->flow
== NULL
) {
249 "Failed to create ipsec flow msg: %s\n",
253 } else if (sa
->type
==
254 RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL
) {
255 struct rte_security_ctx
*ctx
=
256 (struct rte_security_ctx
*)
257 rte_eth_dev_get_sec_ctx(sa
->portid
);
258 const struct rte_security_capability
*sec_cap
;
262 "Ethernet device doesn't have security features registered\n");
266 /* Set IPsec parameters in conf */
267 set_ipsec_conf(sa
, &(sess_conf
.ipsec
));
269 /* Save SA as userdata for the security session. When
270 * the packet is received, this userdata will be
271 * retrieved using the metadata from the packet.
273 * The PMD is expected to set similar metadata for other
274 * operations, like rte_eth_event, which are tied to
275 * security session. In such cases, the userdata could
276 * be obtained to uniquely identify the security
277 * parameters denoted.
280 sess_conf
.userdata
= (void *) sa
;
282 sa
->sec_session
= rte_security_session_create(ctx
,
283 &sess_conf
, ipsec_ctx
->session_pool
);
284 if (sa
->sec_session
== NULL
) {
286 "SEC Session init failed: err: %d\n", ret
);
290 sec_cap
= rte_security_capabilities_get(ctx
);
292 if (sec_cap
== NULL
) {
294 "No capabilities registered\n");
298 /* iterate until ESP tunnel*/
299 while (sec_cap
->action
!=
300 RTE_SECURITY_ACTION_TYPE_NONE
) {
302 if (sec_cap
->action
== sa
->type
&&
304 RTE_SECURITY_PROTOCOL_IPSEC
&&
305 sec_cap
->ipsec
.mode
==
306 RTE_SECURITY_IPSEC_SA_MODE_TUNNEL
&&
307 sec_cap
->ipsec
.direction
== sa
->direction
)
312 if (sec_cap
->action
== RTE_SECURITY_ACTION_TYPE_NONE
) {
314 "No suitable security capability found\n");
318 sa
->ol_flags
= sec_cap
->ol_flags
;
319 sa
->security_ctx
= ctx
;
322 sa
->crypto_session
= rte_cryptodev_sym_session_create(
323 ipsec_ctx
->session_pool
);
324 rte_cryptodev_sym_session_init(ipsec_ctx
->tbl
[cdev_id_qp
].id
,
325 sa
->crypto_session
, sa
->xforms
,
326 ipsec_ctx
->session_priv_pool
);
328 rte_cryptodev_info_get(ipsec_ctx
->tbl
[cdev_id_qp
].id
,
331 sa
->cdev_id_qp
= cdev_id_qp
;
337 * queue crypto-ops into PMD queue.
340 enqueue_cop_burst(struct cdev_qp
*cqp
)
342 uint32_t i
, len
, ret
;
345 ret
= rte_cryptodev_enqueue_burst(cqp
->id
, cqp
->qp
, cqp
->buf
, len
);
347 RTE_LOG_DP(DEBUG
, IPSEC
, "Cryptodev %u queue %u:"
348 " enqueued %u crypto ops out of %u\n",
349 cqp
->id
, cqp
->qp
, ret
, len
);
350 /* drop packets that we fail to enqueue */
351 for (i
= ret
; i
< len
; i
++)
352 rte_pktmbuf_free(cqp
->buf
[i
]->sym
->m_src
);
354 cqp
->in_flight
+= ret
;
359 enqueue_cop(struct cdev_qp
*cqp
, struct rte_crypto_op
*cop
)
361 cqp
->buf
[cqp
->len
++] = cop
;
363 if (cqp
->len
== MAX_PKT_BURST
)
364 enqueue_cop_burst(cqp
);
368 ipsec_enqueue(ipsec_xform_fn xform_func
, struct ipsec_ctx
*ipsec_ctx
,
369 struct rte_mbuf
*pkts
[], struct ipsec_sa
*sas
[],
373 struct ipsec_mbuf_metadata
*priv
;
374 struct rte_crypto_sym_op
*sym_cop
;
377 for (i
= 0; i
< nb_pkts
; i
++) {
378 if (unlikely(sas
[i
] == NULL
)) {
379 rte_pktmbuf_free(pkts
[i
]);
383 rte_prefetch0(sas
[i
]);
384 rte_prefetch0(pkts
[i
]);
386 priv
= get_priv(pkts
[i
]);
391 case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL
:
392 priv
->cop
.type
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
;
393 priv
->cop
.status
= RTE_CRYPTO_OP_STATUS_NOT_PROCESSED
;
395 rte_prefetch0(&priv
->sym_cop
);
397 if ((unlikely(sa
->sec_session
== NULL
)) &&
398 create_session(ipsec_ctx
, sa
)) {
399 rte_pktmbuf_free(pkts
[i
]);
403 sym_cop
= get_sym_cop(&priv
->cop
);
404 sym_cop
->m_src
= pkts
[i
];
406 rte_security_attach_session(&priv
->cop
,
409 case RTE_SECURITY_ACTION_TYPE_NONE
:
411 priv
->cop
.type
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
;
412 priv
->cop
.status
= RTE_CRYPTO_OP_STATUS_NOT_PROCESSED
;
414 rte_prefetch0(&priv
->sym_cop
);
416 if ((unlikely(sa
->crypto_session
== NULL
)) &&
417 create_session(ipsec_ctx
, sa
)) {
418 rte_pktmbuf_free(pkts
[i
]);
422 rte_crypto_op_attach_sym_session(&priv
->cop
,
425 ret
= xform_func(pkts
[i
], sa
, &priv
->cop
);
427 rte_pktmbuf_free(pkts
[i
]);
431 case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL
:
432 if ((unlikely(sa
->sec_session
== NULL
)) &&
433 create_session(ipsec_ctx
, sa
)) {
434 rte_pktmbuf_free(pkts
[i
]);
438 ipsec_ctx
->ol_pkts
[ipsec_ctx
->ol_pkts_cnt
++] = pkts
[i
];
439 if (sa
->ol_flags
& RTE_SECURITY_TX_OLOAD_NEED_MDATA
)
440 rte_security_set_pkt_metadata(
442 sa
->sec_session
, pkts
[i
], NULL
);
444 case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO
:
445 priv
->cop
.type
= RTE_CRYPTO_OP_TYPE_SYMMETRIC
;
446 priv
->cop
.status
= RTE_CRYPTO_OP_STATUS_NOT_PROCESSED
;
448 rte_prefetch0(&priv
->sym_cop
);
450 if ((unlikely(sa
->sec_session
== NULL
)) &&
451 create_session(ipsec_ctx
, sa
)) {
452 rte_pktmbuf_free(pkts
[i
]);
456 rte_security_attach_session(&priv
->cop
,
459 ret
= xform_func(pkts
[i
], sa
, &priv
->cop
);
461 rte_pktmbuf_free(pkts
[i
]);
465 ipsec_ctx
->ol_pkts
[ipsec_ctx
->ol_pkts_cnt
++] = pkts
[i
];
466 if (sa
->ol_flags
& RTE_SECURITY_TX_OLOAD_NEED_MDATA
)
467 rte_security_set_pkt_metadata(
469 sa
->sec_session
, pkts
[i
], NULL
);
473 RTE_ASSERT(sa
->cdev_id_qp
< ipsec_ctx
->nb_qps
);
474 enqueue_cop(&ipsec_ctx
->tbl
[sa
->cdev_id_qp
], &priv
->cop
);
478 static inline int32_t
479 ipsec_inline_dequeue(ipsec_xform_fn xform_func
, struct ipsec_ctx
*ipsec_ctx
,
480 struct rte_mbuf
*pkts
[], uint16_t max_pkts
)
482 int32_t nb_pkts
, ret
;
483 struct ipsec_mbuf_metadata
*priv
;
485 struct rte_mbuf
*pkt
;
488 while (ipsec_ctx
->ol_pkts_cnt
> 0 && nb_pkts
< max_pkts
) {
489 pkt
= ipsec_ctx
->ol_pkts
[--ipsec_ctx
->ol_pkts_cnt
];
491 priv
= get_priv(pkt
);
493 ret
= xform_func(pkt
, sa
, &priv
->cop
);
495 rte_pktmbuf_free(pkt
);
498 pkts
[nb_pkts
++] = pkt
;
505 ipsec_dequeue(ipsec_xform_fn xform_func
, struct ipsec_ctx
*ipsec_ctx
,
506 struct rte_mbuf
*pkts
[], uint16_t max_pkts
)
508 int32_t nb_pkts
= 0, ret
= 0, i
, j
, nb_cops
;
509 struct ipsec_mbuf_metadata
*priv
;
510 struct rte_crypto_op
*cops
[max_pkts
];
512 struct rte_mbuf
*pkt
;
514 for (i
= 0; i
< ipsec_ctx
->nb_qps
&& nb_pkts
< max_pkts
; i
++) {
517 cqp
= &ipsec_ctx
->tbl
[ipsec_ctx
->last_qp
++];
518 if (ipsec_ctx
->last_qp
== ipsec_ctx
->nb_qps
)
519 ipsec_ctx
->last_qp
%= ipsec_ctx
->nb_qps
;
521 if (cqp
->in_flight
== 0)
524 nb_cops
= rte_cryptodev_dequeue_burst(cqp
->id
, cqp
->qp
,
525 cops
, max_pkts
- nb_pkts
);
527 cqp
->in_flight
-= nb_cops
;
529 for (j
= 0; j
< nb_cops
; j
++) {
530 pkt
= cops
[j
]->sym
->m_src
;
533 priv
= get_priv(pkt
);
536 RTE_ASSERT(sa
!= NULL
);
538 if (sa
->type
== RTE_SECURITY_ACTION_TYPE_NONE
) {
539 ret
= xform_func(pkt
, sa
, cops
[j
]);
541 rte_pktmbuf_free(pkt
);
545 pkts
[nb_pkts
++] = pkt
;
554 ipsec_inbound(struct ipsec_ctx
*ctx
, struct rte_mbuf
*pkts
[],
555 uint16_t nb_pkts
, uint16_t len
)
557 struct ipsec_sa
*sas
[nb_pkts
];
559 inbound_sa_lookup(ctx
->sa_ctx
, pkts
, sas
, nb_pkts
);
561 ipsec_enqueue(esp_inbound
, ctx
, pkts
, sas
, nb_pkts
);
563 return ipsec_inline_dequeue(esp_inbound_post
, ctx
, pkts
, len
);
567 ipsec_inbound_cqp_dequeue(struct ipsec_ctx
*ctx
, struct rte_mbuf
*pkts
[],
570 return ipsec_dequeue(esp_inbound_post
, ctx
, pkts
, len
);
574 ipsec_outbound(struct ipsec_ctx
*ctx
, struct rte_mbuf
*pkts
[],
575 uint32_t sa_idx
[], uint16_t nb_pkts
, uint16_t len
)
577 struct ipsec_sa
*sas
[nb_pkts
];
579 outbound_sa_lookup(ctx
->sa_ctx
, sa_idx
, sas
, nb_pkts
);
581 ipsec_enqueue(esp_outbound
, ctx
, pkts
, sas
, nb_pkts
);
583 return ipsec_inline_dequeue(esp_outbound_post
, ctx
, pkts
, len
);
587 ipsec_outbound_cqp_dequeue(struct ipsec_ctx
*ctx
, struct rte_mbuf
*pkts
[],
590 return ipsec_dequeue(esp_outbound_post
, ctx
, pkts
, len
);