1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2018 Intel Corporation
5 #include <rte_common.h>
6 #include <rte_hexdump.h>
7 #include <rte_cryptodev.h>
8 #include <rte_cryptodev_pmd.h>
9 #include <rte_bus_vdev.h>
10 #include <rte_malloc.h>
11 #include <rte_cpuflags.h>
13 #include "rte_snow3g_pmd_private.h"
15 #define SNOW3G_IV_LENGTH 16
16 #define SNOW3G_MAX_BURST 8
19 static uint8_t cryptodev_driver_id
;
21 /** Get xform chain order. */
22 static enum snow3g_operation
23 snow3g_get_mode(const struct rte_crypto_sym_xform
*xform
)
26 return SNOW3G_OP_NOT_SUPPORTED
;
29 if (xform
->next
->next
!= NULL
)
30 return SNOW3G_OP_NOT_SUPPORTED
;
32 if (xform
->type
== RTE_CRYPTO_SYM_XFORM_AUTH
) {
33 if (xform
->next
== NULL
)
34 return SNOW3G_OP_ONLY_AUTH
;
35 else if (xform
->next
->type
== RTE_CRYPTO_SYM_XFORM_CIPHER
)
36 return SNOW3G_OP_AUTH_CIPHER
;
38 return SNOW3G_OP_NOT_SUPPORTED
;
41 if (xform
->type
== RTE_CRYPTO_SYM_XFORM_CIPHER
) {
42 if (xform
->next
== NULL
)
43 return SNOW3G_OP_ONLY_CIPHER
;
44 else if (xform
->next
->type
== RTE_CRYPTO_SYM_XFORM_AUTH
)
45 return SNOW3G_OP_CIPHER_AUTH
;
47 return SNOW3G_OP_NOT_SUPPORTED
;
50 return SNOW3G_OP_NOT_SUPPORTED
;
54 /** Parse crypto xform chain and set private session parameters. */
56 snow3g_set_session_parameters(struct snow3g_session
*sess
,
57 const struct rte_crypto_sym_xform
*xform
)
59 const struct rte_crypto_sym_xform
*auth_xform
= NULL
;
60 const struct rte_crypto_sym_xform
*cipher_xform
= NULL
;
61 enum snow3g_operation mode
;
63 /* Select Crypto operation - hash then cipher / cipher then hash */
64 mode
= snow3g_get_mode(xform
);
67 case SNOW3G_OP_CIPHER_AUTH
:
68 auth_xform
= xform
->next
;
71 case SNOW3G_OP_ONLY_CIPHER
:
74 case SNOW3G_OP_AUTH_CIPHER
:
75 cipher_xform
= xform
->next
;
77 case SNOW3G_OP_ONLY_AUTH
:
80 case SNOW3G_OP_NOT_SUPPORTED
:
82 SNOW3G_LOG(ERR
, "Unsupported operation chain order parameter");
87 /* Only SNOW 3G UEA2 supported */
88 if (cipher_xform
->cipher
.algo
!= RTE_CRYPTO_CIPHER_SNOW3G_UEA2
)
91 if (cipher_xform
->cipher
.iv
.length
!= SNOW3G_IV_LENGTH
) {
92 SNOW3G_LOG(ERR
, "Wrong IV length");
95 sess
->cipher_iv_offset
= cipher_xform
->cipher
.iv
.offset
;
98 sso_snow3g_init_key_sched(cipher_xform
->cipher
.key
.data
,
99 &sess
->pKeySched_cipher
);
103 /* Only SNOW 3G UIA2 supported */
104 if (auth_xform
->auth
.algo
!= RTE_CRYPTO_AUTH_SNOW3G_UIA2
)
107 if (auth_xform
->auth
.digest_length
!= SNOW3G_DIGEST_LENGTH
) {
108 SNOW3G_LOG(ERR
, "Wrong digest length");
112 sess
->auth_op
= auth_xform
->auth
.op
;
114 if (auth_xform
->auth
.iv
.length
!= SNOW3G_IV_LENGTH
) {
115 SNOW3G_LOG(ERR
, "Wrong IV length");
118 sess
->auth_iv_offset
= auth_xform
->auth
.iv
.offset
;
121 sso_snow3g_init_key_sched(auth_xform
->auth
.key
.data
,
122 &sess
->pKeySched_hash
);
131 /** Get SNOW 3G session. */
132 static struct snow3g_session
*
133 snow3g_get_session(struct snow3g_qp
*qp
, struct rte_crypto_op
*op
)
135 struct snow3g_session
*sess
= NULL
;
137 if (op
->sess_type
== RTE_CRYPTO_OP_WITH_SESSION
) {
138 if (likely(op
->sym
->session
!= NULL
))
139 sess
= (struct snow3g_session
*)
140 get_sym_session_private_data(
142 cryptodev_driver_id
);
145 void *_sess_private_data
= NULL
;
147 if (rte_mempool_get(qp
->sess_mp
, (void **)&_sess
))
150 if (rte_mempool_get(qp
->sess_mp_priv
,
151 (void **)&_sess_private_data
))
154 sess
= (struct snow3g_session
*)_sess_private_data
;
156 if (unlikely(snow3g_set_session_parameters(sess
,
157 op
->sym
->xform
) != 0)) {
158 rte_mempool_put(qp
->sess_mp
, _sess
);
159 rte_mempool_put(qp
->sess_mp_priv
, _sess_private_data
);
162 op
->sym
->session
= (struct rte_cryptodev_sym_session
*)_sess
;
163 set_sym_session_private_data(op
->sym
->session
,
164 cryptodev_driver_id
, _sess_private_data
);
167 if (unlikely(sess
== NULL
))
168 op
->status
= RTE_CRYPTO_OP_STATUS_INVALID_SESSION
;
174 /** Encrypt/decrypt mbufs with same cipher key. */
176 process_snow3g_cipher_op(struct rte_crypto_op
**ops
,
177 struct snow3g_session
*session
,
181 uint8_t processed_ops
= 0;
182 uint8_t *src
[SNOW3G_MAX_BURST
], *dst
[SNOW3G_MAX_BURST
];
183 uint8_t *iv
[SNOW3G_MAX_BURST
];
184 uint32_t num_bytes
[SNOW3G_MAX_BURST
];
186 for (i
= 0; i
< num_ops
; i
++) {
187 src
[i
] = rte_pktmbuf_mtod(ops
[i
]->sym
->m_src
, uint8_t *) +
188 (ops
[i
]->sym
->cipher
.data
.offset
>> 3);
189 dst
[i
] = ops
[i
]->sym
->m_dst
?
190 rte_pktmbuf_mtod(ops
[i
]->sym
->m_dst
, uint8_t *) +
191 (ops
[i
]->sym
->cipher
.data
.offset
>> 3) :
192 rte_pktmbuf_mtod(ops
[i
]->sym
->m_src
, uint8_t *) +
193 (ops
[i
]->sym
->cipher
.data
.offset
>> 3);
194 iv
[i
] = rte_crypto_op_ctod_offset(ops
[i
], uint8_t *,
195 session
->cipher_iv_offset
);
196 num_bytes
[i
] = ops
[i
]->sym
->cipher
.data
.length
>> 3;
201 sso_snow3g_f8_n_buffer(&session
->pKeySched_cipher
, iv
, src
, dst
,
202 num_bytes
, processed_ops
);
204 return processed_ops
;
207 /** Encrypt/decrypt mbuf (bit level function). */
209 process_snow3g_cipher_op_bit(struct rte_crypto_op
*op
,
210 struct snow3g_session
*session
)
214 uint32_t length_in_bits
, offset_in_bits
;
216 offset_in_bits
= op
->sym
->cipher
.data
.offset
;
217 src
= rte_pktmbuf_mtod(op
->sym
->m_src
, uint8_t *);
218 if (op
->sym
->m_dst
== NULL
) {
219 op
->status
= RTE_CRYPTO_OP_STATUS_INVALID_ARGS
;
220 SNOW3G_LOG(ERR
, "bit-level in-place not supported\n");
223 dst
= rte_pktmbuf_mtod(op
->sym
->m_dst
, uint8_t *);
224 iv
= rte_crypto_op_ctod_offset(op
, uint8_t *,
225 session
->cipher_iv_offset
);
226 length_in_bits
= op
->sym
->cipher
.data
.length
;
228 sso_snow3g_f8_1_buffer_bit(&session
->pKeySched_cipher
, iv
,
229 src
, dst
, length_in_bits
, offset_in_bits
);
234 /** Generate/verify hash from mbufs with same hash key. */
236 process_snow3g_hash_op(struct snow3g_qp
*qp
, struct rte_crypto_op
**ops
,
237 struct snow3g_session
*session
,
241 uint8_t processed_ops
= 0;
243 uint32_t length_in_bits
;
246 for (i
= 0; i
< num_ops
; i
++) {
247 /* Data must be byte aligned */
248 if ((ops
[i
]->sym
->auth
.data
.offset
% BYTE_LEN
) != 0) {
249 ops
[i
]->status
= RTE_CRYPTO_OP_STATUS_INVALID_ARGS
;
250 SNOW3G_LOG(ERR
, "Offset");
254 length_in_bits
= ops
[i
]->sym
->auth
.data
.length
;
256 src
= rte_pktmbuf_mtod(ops
[i
]->sym
->m_src
, uint8_t *) +
257 (ops
[i
]->sym
->auth
.data
.offset
>> 3);
258 iv
= rte_crypto_op_ctod_offset(ops
[i
], uint8_t *,
259 session
->auth_iv_offset
);
261 if (session
->auth_op
== RTE_CRYPTO_AUTH_OP_VERIFY
) {
262 dst
= qp
->temp_digest
;
264 sso_snow3g_f9_1_buffer(&session
->pKeySched_hash
,
266 length_in_bits
, dst
);
268 if (memcmp(dst
, ops
[i
]->sym
->auth
.digest
.data
,
269 SNOW3G_DIGEST_LENGTH
) != 0)
270 ops
[i
]->status
= RTE_CRYPTO_OP_STATUS_AUTH_FAILED
;
272 dst
= ops
[i
]->sym
->auth
.digest
.data
;
274 sso_snow3g_f9_1_buffer(&session
->pKeySched_hash
,
276 length_in_bits
, dst
);
281 return processed_ops
;
284 /** Process a batch of crypto ops which shares the same session. */
286 process_ops(struct rte_crypto_op
**ops
, struct snow3g_session
*session
,
287 struct snow3g_qp
*qp
, uint8_t num_ops
,
288 uint16_t *accumulated_enqueued_ops
)
291 unsigned enqueued_ops
, processed_ops
;
293 #ifdef RTE_LIBRTE_PMD_SNOW3G_DEBUG
294 for (i
= 0; i
< num_ops
; i
++) {
295 if (!rte_pktmbuf_is_contiguous(ops
[i
]->sym
->m_src
) ||
296 (ops
[i
]->sym
->m_dst
!= NULL
&&
297 !rte_pktmbuf_is_contiguous(
298 ops
[i
]->sym
->m_dst
))) {
299 SNOW3G_LOG(ERR
, "PMD supports only contiguous mbufs, "
300 "op (%p) provides noncontiguous mbuf as "
301 "source/destination buffer.\n", ops
[i
]);
302 ops
[i
]->status
= RTE_CRYPTO_OP_STATUS_INVALID_ARGS
;
308 switch (session
->op
) {
309 case SNOW3G_OP_ONLY_CIPHER
:
310 processed_ops
= process_snow3g_cipher_op(ops
,
313 case SNOW3G_OP_ONLY_AUTH
:
314 processed_ops
= process_snow3g_hash_op(qp
, ops
, session
,
317 case SNOW3G_OP_CIPHER_AUTH
:
318 processed_ops
= process_snow3g_cipher_op(ops
, session
,
320 process_snow3g_hash_op(qp
, ops
, session
, processed_ops
);
322 case SNOW3G_OP_AUTH_CIPHER
:
323 processed_ops
= process_snow3g_hash_op(qp
, ops
, session
,
325 process_snow3g_cipher_op(ops
, session
, processed_ops
);
328 /* Operation not supported. */
332 for (i
= 0; i
< num_ops
; i
++) {
334 * If there was no error/authentication failure,
335 * change status to successful.
337 if (ops
[i
]->status
== RTE_CRYPTO_OP_STATUS_NOT_PROCESSED
)
338 ops
[i
]->status
= RTE_CRYPTO_OP_STATUS_SUCCESS
;
339 /* Free session if a session-less crypto op. */
340 if (ops
[i
]->sess_type
== RTE_CRYPTO_OP_SESSIONLESS
) {
341 memset(session
, 0, sizeof(struct snow3g_session
));
342 memset(ops
[i
]->sym
->session
, 0,
343 rte_cryptodev_sym_get_existing_header_session_size(
344 ops
[i
]->sym
->session
));
345 rte_mempool_put(qp
->sess_mp_priv
, session
);
346 rte_mempool_put(qp
->sess_mp
, ops
[i
]->sym
->session
);
347 ops
[i
]->sym
->session
= NULL
;
351 enqueued_ops
= rte_ring_enqueue_burst(qp
->processed_ops
,
352 (void **)ops
, processed_ops
, NULL
);
353 qp
->qp_stats
.enqueued_count
+= enqueued_ops
;
354 *accumulated_enqueued_ops
+= enqueued_ops
;
359 /** Process a crypto op with length/offset in bits. */
361 process_op_bit(struct rte_crypto_op
*op
, struct snow3g_session
*session
,
362 struct snow3g_qp
*qp
, uint16_t *accumulated_enqueued_ops
)
364 unsigned enqueued_op
, processed_op
;
366 switch (session
->op
) {
367 case SNOW3G_OP_ONLY_CIPHER
:
368 processed_op
= process_snow3g_cipher_op_bit(op
,
371 case SNOW3G_OP_ONLY_AUTH
:
372 processed_op
= process_snow3g_hash_op(qp
, &op
, session
, 1);
374 case SNOW3G_OP_CIPHER_AUTH
:
375 processed_op
= process_snow3g_cipher_op_bit(op
, session
);
376 if (processed_op
== 1)
377 process_snow3g_hash_op(qp
, &op
, session
, 1);
379 case SNOW3G_OP_AUTH_CIPHER
:
380 processed_op
= process_snow3g_hash_op(qp
, &op
, session
, 1);
381 if (processed_op
== 1)
382 process_snow3g_cipher_op_bit(op
, session
);
385 /* Operation not supported. */
390 * If there was no error/authentication failure,
391 * change status to successful.
393 if (op
->status
== RTE_CRYPTO_OP_STATUS_NOT_PROCESSED
)
394 op
->status
= RTE_CRYPTO_OP_STATUS_SUCCESS
;
396 /* Free session if a session-less crypto op. */
397 if (op
->sess_type
== RTE_CRYPTO_OP_SESSIONLESS
) {
398 memset(op
->sym
->session
, 0, sizeof(struct snow3g_session
));
399 rte_cryptodev_sym_session_free(op
->sym
->session
);
400 op
->sym
->session
= NULL
;
403 enqueued_op
= rte_ring_enqueue_burst(qp
->processed_ops
,
404 (void **)&op
, processed_op
, NULL
);
405 qp
->qp_stats
.enqueued_count
+= enqueued_op
;
406 *accumulated_enqueued_ops
+= enqueued_op
;
412 snow3g_pmd_enqueue_burst(void *queue_pair
, struct rte_crypto_op
**ops
,
415 struct rte_crypto_op
*c_ops
[SNOW3G_MAX_BURST
];
416 struct rte_crypto_op
*curr_c_op
;
418 struct snow3g_session
*prev_sess
= NULL
, *curr_sess
= NULL
;
419 struct snow3g_qp
*qp
= queue_pair
;
421 uint8_t burst_size
= 0;
422 uint16_t enqueued_ops
= 0;
423 uint8_t processed_ops
;
425 for (i
= 0; i
< nb_ops
; i
++) {
428 /* Set status as enqueued (not processed yet) by default. */
429 curr_c_op
->status
= RTE_CRYPTO_OP_STATUS_NOT_PROCESSED
;
431 curr_sess
= snow3g_get_session(qp
, curr_c_op
);
432 if (unlikely(curr_sess
== NULL
||
433 curr_sess
->op
== SNOW3G_OP_NOT_SUPPORTED
)) {
435 RTE_CRYPTO_OP_STATUS_INVALID_SESSION
;
439 /* If length/offset is at bit-level, process this buffer alone. */
440 if (((curr_c_op
->sym
->cipher
.data
.length
% BYTE_LEN
) != 0)
441 || ((curr_c_op
->sym
->cipher
.data
.offset
443 /* Process the ops of the previous session. */
444 if (prev_sess
!= NULL
) {
445 processed_ops
= process_ops(c_ops
, prev_sess
,
446 qp
, burst_size
, &enqueued_ops
);
447 if (processed_ops
< burst_size
) {
456 processed_ops
= process_op_bit(curr_c_op
, curr_sess
,
458 if (processed_ops
!= 1)
464 /* Batch ops that share the same session. */
465 if (prev_sess
== NULL
) {
466 prev_sess
= curr_sess
;
467 c_ops
[burst_size
++] = curr_c_op
;
468 } else if (curr_sess
== prev_sess
) {
469 c_ops
[burst_size
++] = curr_c_op
;
471 * When there are enough ops to process in a batch,
472 * process them, and start a new batch.
474 if (burst_size
== SNOW3G_MAX_BURST
) {
475 processed_ops
= process_ops(c_ops
, prev_sess
,
476 qp
, burst_size
, &enqueued_ops
);
477 if (processed_ops
< burst_size
) {
487 * Different session, process the ops
488 * of the previous session.
490 processed_ops
= process_ops(c_ops
, prev_sess
,
491 qp
, burst_size
, &enqueued_ops
);
492 if (processed_ops
< burst_size
) {
498 prev_sess
= curr_sess
;
500 c_ops
[burst_size
++] = curr_c_op
;
504 if (burst_size
!= 0) {
505 /* Process the crypto ops of the last session. */
506 processed_ops
= process_ops(c_ops
, prev_sess
,
507 qp
, burst_size
, &enqueued_ops
);
510 qp
->qp_stats
.enqueue_err_count
+= nb_ops
- enqueued_ops
;
515 snow3g_pmd_dequeue_burst(void *queue_pair
,
516 struct rte_crypto_op
**c_ops
, uint16_t nb_ops
)
518 struct snow3g_qp
*qp
= queue_pair
;
520 unsigned nb_dequeued
;
522 nb_dequeued
= rte_ring_dequeue_burst(qp
->processed_ops
,
523 (void **)c_ops
, nb_ops
, NULL
);
524 qp
->qp_stats
.dequeued_count
+= nb_dequeued
;
529 static int cryptodev_snow3g_remove(struct rte_vdev_device
*vdev
);
532 cryptodev_snow3g_create(const char *name
,
533 struct rte_vdev_device
*vdev
,
534 struct rte_cryptodev_pmd_init_params
*init_params
)
536 struct rte_cryptodev
*dev
;
537 struct snow3g_private
*internals
;
538 uint64_t cpu_flags
= RTE_CRYPTODEV_FF_CPU_SSE
;
540 dev
= rte_cryptodev_pmd_create(name
, &vdev
->device
, init_params
);
542 SNOW3G_LOG(ERR
, "failed to create cryptodev vdev");
546 dev
->driver_id
= cryptodev_driver_id
;
547 dev
->dev_ops
= rte_snow3g_pmd_ops
;
549 /* Register RX/TX burst functions for data path. */
550 dev
->dequeue_burst
= snow3g_pmd_dequeue_burst
;
551 dev
->enqueue_burst
= snow3g_pmd_enqueue_burst
;
553 dev
->feature_flags
= RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO
|
554 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING
|
557 internals
= dev
->data
->dev_private
;
559 internals
->max_nb_queue_pairs
= init_params
->max_nb_queue_pairs
;
563 SNOW3G_LOG(ERR
, "driver %s: cryptodev_snow3g_create failed",
566 cryptodev_snow3g_remove(vdev
);
571 cryptodev_snow3g_probe(struct rte_vdev_device
*vdev
)
573 struct rte_cryptodev_pmd_init_params init_params
= {
575 sizeof(struct snow3g_private
),
577 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
580 const char *input_args
;
582 name
= rte_vdev_device_name(vdev
);
585 input_args
= rte_vdev_device_args(vdev
);
587 rte_cryptodev_pmd_parse_input_args(&init_params
, input_args
);
589 return cryptodev_snow3g_create(name
, vdev
, &init_params
);
593 cryptodev_snow3g_remove(struct rte_vdev_device
*vdev
)
595 struct rte_cryptodev
*cryptodev
;
598 name
= rte_vdev_device_name(vdev
);
602 cryptodev
= rte_cryptodev_pmd_get_named_dev(name
);
603 if (cryptodev
== NULL
)
606 return rte_cryptodev_pmd_destroy(cryptodev
);
609 static struct rte_vdev_driver cryptodev_snow3g_pmd_drv
= {
610 .probe
= cryptodev_snow3g_probe
,
611 .remove
= cryptodev_snow3g_remove
614 static struct cryptodev_driver snow3g_crypto_drv
;
616 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SNOW3G_PMD
, cryptodev_snow3g_pmd_drv
);
617 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_SNOW3G_PMD
, cryptodev_snow3g_pmd
);
618 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SNOW3G_PMD
,
619 "max_nb_queue_pairs=<int> "
621 RTE_PMD_REGISTER_CRYPTO_DRIVER(snow3g_crypto_drv
,
622 cryptodev_snow3g_pmd_drv
.driver
, cryptodev_driver_id
);
624 RTE_INIT(snow3g_init_log
)
626 snow3g_logtype_driver
= rte_log_register("pmd.crypto.snow3g");