]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2016-2018 Intel Corporation | |
7c673cae FG |
3 | */ |
4 | ||
5 | #include <rte_common.h> | |
7c673cae FG |
6 | #include <rte_hexdump.h> |
7 | #include <rte_cryptodev.h> | |
8 | #include <rte_cryptodev_pmd.h> | |
9f95a23c | 9 | #include <rte_bus_vdev.h> |
7c673cae FG |
10 | #include <rte_malloc.h> |
11 | #include <rte_cpuflags.h> | |
12 | ||
13 | #include "rte_snow3g_pmd_private.h" | |
14 | ||
15 | #define SNOW3G_IV_LENGTH 16 | |
7c673cae FG |
16 | #define SNOW3G_MAX_BURST 8 |
17 | #define BYTE_LEN 8 | |
18 | ||
9f95a23c TL |
19 | static uint8_t cryptodev_driver_id; |
20 | ||
7c673cae FG |
21 | /** Get xform chain order. */ |
22 | static enum snow3g_operation | |
23 | snow3g_get_mode(const struct rte_crypto_sym_xform *xform) | |
24 | { | |
25 | if (xform == NULL) | |
26 | return SNOW3G_OP_NOT_SUPPORTED; | |
27 | ||
28 | if (xform->next) | |
29 | if (xform->next->next != NULL) | |
30 | return SNOW3G_OP_NOT_SUPPORTED; | |
31 | ||
32 | if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { | |
33 | if (xform->next == NULL) | |
34 | return SNOW3G_OP_ONLY_AUTH; | |
35 | else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) | |
36 | return SNOW3G_OP_AUTH_CIPHER; | |
37 | else | |
38 | return SNOW3G_OP_NOT_SUPPORTED; | |
39 | } | |
40 | ||
41 | if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { | |
42 | if (xform->next == NULL) | |
43 | return SNOW3G_OP_ONLY_CIPHER; | |
44 | else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) | |
45 | return SNOW3G_OP_CIPHER_AUTH; | |
46 | else | |
47 | return SNOW3G_OP_NOT_SUPPORTED; | |
48 | } | |
49 | ||
50 | return SNOW3G_OP_NOT_SUPPORTED; | |
51 | } | |
52 | ||
53 | ||
54 | /** Parse crypto xform chain and set private session parameters. */ | |
55 | int | |
56 | snow3g_set_session_parameters(struct snow3g_session *sess, | |
57 | const struct rte_crypto_sym_xform *xform) | |
58 | { | |
59 | const struct rte_crypto_sym_xform *auth_xform = NULL; | |
60 | const struct rte_crypto_sym_xform *cipher_xform = NULL; | |
61 | enum snow3g_operation mode; | |
62 | ||
63 | /* Select Crypto operation - hash then cipher / cipher then hash */ | |
64 | mode = snow3g_get_mode(xform); | |
65 | ||
66 | switch (mode) { | |
67 | case SNOW3G_OP_CIPHER_AUTH: | |
68 | auth_xform = xform->next; | |
69 | ||
70 | /* Fall-through */ | |
71 | case SNOW3G_OP_ONLY_CIPHER: | |
72 | cipher_xform = xform; | |
73 | break; | |
74 | case SNOW3G_OP_AUTH_CIPHER: | |
75 | cipher_xform = xform->next; | |
76 | /* Fall-through */ | |
77 | case SNOW3G_OP_ONLY_AUTH: | |
78 | auth_xform = xform; | |
79 | break; | |
80 | case SNOW3G_OP_NOT_SUPPORTED: | |
81 | default: | |
9f95a23c TL |
82 | SNOW3G_LOG(ERR, "Unsupported operation chain order parameter"); |
83 | return -ENOTSUP; | |
7c673cae FG |
84 | } |
85 | ||
86 | if (cipher_xform) { | |
87 | /* Only SNOW 3G UEA2 supported */ | |
88 | if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_SNOW3G_UEA2) | |
9f95a23c TL |
89 | return -ENOTSUP; |
90 | ||
91 | if (cipher_xform->cipher.iv.length != SNOW3G_IV_LENGTH) { | |
92 | SNOW3G_LOG(ERR, "Wrong IV length"); | |
7c673cae | 93 | return -EINVAL; |
9f95a23c TL |
94 | } |
95 | sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; | |
96 | ||
7c673cae | 97 | /* Initialize key */ |
11fdf7f2 | 98 | sso_snow3g_init_key_sched(cipher_xform->cipher.key.data, |
7c673cae FG |
99 | &sess->pKeySched_cipher); |
100 | } | |
101 | ||
102 | if (auth_xform) { | |
103 | /* Only SNOW 3G UIA2 supported */ | |
104 | if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_SNOW3G_UIA2) | |
9f95a23c TL |
105 | return -ENOTSUP; |
106 | ||
107 | if (auth_xform->auth.digest_length != SNOW3G_DIGEST_LENGTH) { | |
108 | SNOW3G_LOG(ERR, "Wrong digest length"); | |
7c673cae | 109 | return -EINVAL; |
9f95a23c TL |
110 | } |
111 | ||
7c673cae | 112 | sess->auth_op = auth_xform->auth.op; |
9f95a23c TL |
113 | |
114 | if (auth_xform->auth.iv.length != SNOW3G_IV_LENGTH) { | |
115 | SNOW3G_LOG(ERR, "Wrong IV length"); | |
116 | return -EINVAL; | |
117 | } | |
118 | sess->auth_iv_offset = auth_xform->auth.iv.offset; | |
119 | ||
7c673cae | 120 | /* Initialize key */ |
11fdf7f2 | 121 | sso_snow3g_init_key_sched(auth_xform->auth.key.data, |
7c673cae FG |
122 | &sess->pKeySched_hash); |
123 | } | |
124 | ||
125 | ||
126 | sess->op = mode; | |
127 | ||
128 | return 0; | |
129 | } | |
130 | ||
131 | /** Get SNOW 3G session. */ | |
132 | static struct snow3g_session * | |
133 | snow3g_get_session(struct snow3g_qp *qp, struct rte_crypto_op *op) | |
134 | { | |
9f95a23c TL |
135 | struct snow3g_session *sess = NULL; |
136 | ||
137 | if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { | |
138 | if (likely(op->sym->session != NULL)) | |
139 | sess = (struct snow3g_session *) | |
140 | get_sym_session_private_data( | |
141 | op->sym->session, | |
142 | cryptodev_driver_id); | |
143 | } else { | |
144 | void *_sess = NULL; | |
145 | void *_sess_private_data = NULL; | |
146 | ||
147 | if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) | |
7c673cae FG |
148 | return NULL; |
149 | ||
9f95a23c TL |
150 | if (rte_mempool_get(qp->sess_mp_priv, |
151 | (void **)&_sess_private_data)) | |
7c673cae FG |
152 | return NULL; |
153 | ||
9f95a23c | 154 | sess = (struct snow3g_session *)_sess_private_data; |
7c673cae FG |
155 | |
156 | if (unlikely(snow3g_set_session_parameters(sess, | |
9f95a23c TL |
157 | op->sym->xform) != 0)) { |
158 | rte_mempool_put(qp->sess_mp, _sess); | |
159 | rte_mempool_put(qp->sess_mp_priv, _sess_private_data); | |
160 | sess = NULL; | |
161 | } | |
162 | op->sym->session = (struct rte_cryptodev_sym_session *)_sess; | |
163 | set_sym_session_private_data(op->sym->session, | |
164 | cryptodev_driver_id, _sess_private_data); | |
7c673cae FG |
165 | } |
166 | ||
9f95a23c TL |
167 | if (unlikely(sess == NULL)) |
168 | op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; | |
169 | ||
170 | ||
7c673cae FG |
171 | return sess; |
172 | } | |
173 | ||
174 | /** Encrypt/decrypt mbufs with same cipher key. */ | |
175 | static uint8_t | |
176 | process_snow3g_cipher_op(struct rte_crypto_op **ops, | |
177 | struct snow3g_session *session, | |
178 | uint8_t num_ops) | |
179 | { | |
180 | unsigned i; | |
181 | uint8_t processed_ops = 0; | |
182 | uint8_t *src[SNOW3G_MAX_BURST], *dst[SNOW3G_MAX_BURST]; | |
9f95a23c | 183 | uint8_t *iv[SNOW3G_MAX_BURST]; |
7c673cae FG |
184 | uint32_t num_bytes[SNOW3G_MAX_BURST]; |
185 | ||
186 | for (i = 0; i < num_ops; i++) { | |
7c673cae FG |
187 | src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + |
188 | (ops[i]->sym->cipher.data.offset >> 3); | |
189 | dst[i] = ops[i]->sym->m_dst ? | |
190 | rte_pktmbuf_mtod(ops[i]->sym->m_dst, uint8_t *) + | |
191 | (ops[i]->sym->cipher.data.offset >> 3) : | |
192 | rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + | |
193 | (ops[i]->sym->cipher.data.offset >> 3); | |
9f95a23c TL |
194 | iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *, |
195 | session->cipher_iv_offset); | |
7c673cae FG |
196 | num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; |
197 | ||
198 | processed_ops++; | |
199 | } | |
200 | ||
9f95a23c | 201 | sso_snow3g_f8_n_buffer(&session->pKeySched_cipher, iv, src, dst, |
7c673cae FG |
202 | num_bytes, processed_ops); |
203 | ||
204 | return processed_ops; | |
205 | } | |
206 | ||
207 | /** Encrypt/decrypt mbuf (bit level function). */ | |
208 | static uint8_t | |
209 | process_snow3g_cipher_op_bit(struct rte_crypto_op *op, | |
210 | struct snow3g_session *session) | |
211 | { | |
212 | uint8_t *src, *dst; | |
9f95a23c | 213 | uint8_t *iv; |
7c673cae FG |
214 | uint32_t length_in_bits, offset_in_bits; |
215 | ||
7c673cae FG |
216 | offset_in_bits = op->sym->cipher.data.offset; |
217 | src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *); | |
218 | if (op->sym->m_dst == NULL) { | |
219 | op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
9f95a23c | 220 | SNOW3G_LOG(ERR, "bit-level in-place not supported\n"); |
7c673cae FG |
221 | return 0; |
222 | } | |
223 | dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *); | |
9f95a23c TL |
224 | iv = rte_crypto_op_ctod_offset(op, uint8_t *, |
225 | session->cipher_iv_offset); | |
7c673cae FG |
226 | length_in_bits = op->sym->cipher.data.length; |
227 | ||
9f95a23c | 228 | sso_snow3g_f8_1_buffer_bit(&session->pKeySched_cipher, iv, |
7c673cae FG |
229 | src, dst, length_in_bits, offset_in_bits); |
230 | ||
231 | return 1; | |
232 | } | |
233 | ||
234 | /** Generate/verify hash from mbufs with same hash key. */ | |
235 | static int | |
9f95a23c | 236 | process_snow3g_hash_op(struct snow3g_qp *qp, struct rte_crypto_op **ops, |
7c673cae FG |
237 | struct snow3g_session *session, |
238 | uint8_t num_ops) | |
239 | { | |
240 | unsigned i; | |
241 | uint8_t processed_ops = 0; | |
242 | uint8_t *src, *dst; | |
243 | uint32_t length_in_bits; | |
9f95a23c | 244 | uint8_t *iv; |
7c673cae FG |
245 | |
246 | for (i = 0; i < num_ops; i++) { | |
7c673cae FG |
247 | /* Data must be byte aligned */ |
248 | if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { | |
249 | ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
9f95a23c | 250 | SNOW3G_LOG(ERR, "Offset"); |
7c673cae FG |
251 | break; |
252 | } | |
253 | ||
254 | length_in_bits = ops[i]->sym->auth.data.length; | |
255 | ||
256 | src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + | |
257 | (ops[i]->sym->auth.data.offset >> 3); | |
9f95a23c TL |
258 | iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *, |
259 | session->auth_iv_offset); | |
7c673cae FG |
260 | |
261 | if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { | |
9f95a23c | 262 | dst = qp->temp_digest; |
7c673cae FG |
263 | |
264 | sso_snow3g_f9_1_buffer(&session->pKeySched_hash, | |
9f95a23c | 265 | iv, src, |
7c673cae FG |
266 | length_in_bits, dst); |
267 | /* Verify digest. */ | |
268 | if (memcmp(dst, ops[i]->sym->auth.digest.data, | |
9f95a23c | 269 | SNOW3G_DIGEST_LENGTH) != 0) |
7c673cae | 270 | ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; |
7c673cae FG |
271 | } else { |
272 | dst = ops[i]->sym->auth.digest.data; | |
273 | ||
274 | sso_snow3g_f9_1_buffer(&session->pKeySched_hash, | |
9f95a23c | 275 | iv, src, |
7c673cae FG |
276 | length_in_bits, dst); |
277 | } | |
278 | processed_ops++; | |
279 | } | |
280 | ||
281 | return processed_ops; | |
282 | } | |
283 | ||
284 | /** Process a batch of crypto ops which shares the same session. */ | |
285 | static int | |
286 | process_ops(struct rte_crypto_op **ops, struct snow3g_session *session, | |
287 | struct snow3g_qp *qp, uint8_t num_ops, | |
288 | uint16_t *accumulated_enqueued_ops) | |
289 | { | |
290 | unsigned i; | |
291 | unsigned enqueued_ops, processed_ops; | |
292 | ||
11fdf7f2 TL |
293 | #ifdef RTE_LIBRTE_PMD_SNOW3G_DEBUG |
294 | for (i = 0; i < num_ops; i++) { | |
295 | if (!rte_pktmbuf_is_contiguous(ops[i]->sym->m_src) || | |
296 | (ops[i]->sym->m_dst != NULL && | |
297 | !rte_pktmbuf_is_contiguous( | |
298 | ops[i]->sym->m_dst))) { | |
9f95a23c | 299 | SNOW3G_LOG(ERR, "PMD supports only contiguous mbufs, " |
11fdf7f2 TL |
300 | "op (%p) provides noncontiguous mbuf as " |
301 | "source/destination buffer.\n", ops[i]); | |
302 | ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
303 | return 0; | |
304 | } | |
305 | } | |
306 | #endif | |
307 | ||
7c673cae FG |
308 | switch (session->op) { |
309 | case SNOW3G_OP_ONLY_CIPHER: | |
310 | processed_ops = process_snow3g_cipher_op(ops, | |
311 | session, num_ops); | |
312 | break; | |
313 | case SNOW3G_OP_ONLY_AUTH: | |
9f95a23c | 314 | processed_ops = process_snow3g_hash_op(qp, ops, session, |
7c673cae FG |
315 | num_ops); |
316 | break; | |
317 | case SNOW3G_OP_CIPHER_AUTH: | |
318 | processed_ops = process_snow3g_cipher_op(ops, session, | |
319 | num_ops); | |
9f95a23c | 320 | process_snow3g_hash_op(qp, ops, session, processed_ops); |
7c673cae FG |
321 | break; |
322 | case SNOW3G_OP_AUTH_CIPHER: | |
9f95a23c | 323 | processed_ops = process_snow3g_hash_op(qp, ops, session, |
7c673cae FG |
324 | num_ops); |
325 | process_snow3g_cipher_op(ops, session, processed_ops); | |
326 | break; | |
327 | default: | |
328 | /* Operation not supported. */ | |
329 | processed_ops = 0; | |
330 | } | |
331 | ||
332 | for (i = 0; i < num_ops; i++) { | |
333 | /* | |
334 | * If there was no error/authentication failure, | |
335 | * change status to successful. | |
336 | */ | |
337 | if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) | |
338 | ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; | |
339 | /* Free session if a session-less crypto op. */ | |
9f95a23c TL |
340 | if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { |
341 | memset(session, 0, sizeof(struct snow3g_session)); | |
342 | memset(ops[i]->sym->session, 0, | |
343 | rte_cryptodev_sym_get_existing_header_session_size( | |
344 | ops[i]->sym->session)); | |
345 | rte_mempool_put(qp->sess_mp_priv, session); | |
7c673cae FG |
346 | rte_mempool_put(qp->sess_mp, ops[i]->sym->session); |
347 | ops[i]->sym->session = NULL; | |
348 | } | |
349 | } | |
350 | ||
351 | enqueued_ops = rte_ring_enqueue_burst(qp->processed_ops, | |
11fdf7f2 | 352 | (void **)ops, processed_ops, NULL); |
7c673cae FG |
353 | qp->qp_stats.enqueued_count += enqueued_ops; |
354 | *accumulated_enqueued_ops += enqueued_ops; | |
355 | ||
356 | return enqueued_ops; | |
357 | } | |
358 | ||
359 | /** Process a crypto op with length/offset in bits. */ | |
360 | static int | |
361 | process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, | |
362 | struct snow3g_qp *qp, uint16_t *accumulated_enqueued_ops) | |
363 | { | |
364 | unsigned enqueued_op, processed_op; | |
365 | ||
366 | switch (session->op) { | |
367 | case SNOW3G_OP_ONLY_CIPHER: | |
368 | processed_op = process_snow3g_cipher_op_bit(op, | |
369 | session); | |
370 | break; | |
371 | case SNOW3G_OP_ONLY_AUTH: | |
9f95a23c | 372 | processed_op = process_snow3g_hash_op(qp, &op, session, 1); |
7c673cae FG |
373 | break; |
374 | case SNOW3G_OP_CIPHER_AUTH: | |
375 | processed_op = process_snow3g_cipher_op_bit(op, session); | |
376 | if (processed_op == 1) | |
9f95a23c | 377 | process_snow3g_hash_op(qp, &op, session, 1); |
7c673cae FG |
378 | break; |
379 | case SNOW3G_OP_AUTH_CIPHER: | |
9f95a23c | 380 | processed_op = process_snow3g_hash_op(qp, &op, session, 1); |
7c673cae FG |
381 | if (processed_op == 1) |
382 | process_snow3g_cipher_op_bit(op, session); | |
383 | break; | |
384 | default: | |
385 | /* Operation not supported. */ | |
386 | processed_op = 0; | |
387 | } | |
388 | ||
389 | /* | |
390 | * If there was no error/authentication failure, | |
391 | * change status to successful. | |
392 | */ | |
393 | if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) | |
394 | op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; | |
395 | ||
396 | /* Free session if a session-less crypto op. */ | |
9f95a23c TL |
397 | if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { |
398 | memset(op->sym->session, 0, sizeof(struct snow3g_session)); | |
399 | rte_cryptodev_sym_session_free(op->sym->session); | |
7c673cae FG |
400 | op->sym->session = NULL; |
401 | } | |
402 | ||
403 | enqueued_op = rte_ring_enqueue_burst(qp->processed_ops, | |
11fdf7f2 | 404 | (void **)&op, processed_op, NULL); |
7c673cae FG |
405 | qp->qp_stats.enqueued_count += enqueued_op; |
406 | *accumulated_enqueued_ops += enqueued_op; | |
407 | ||
408 | return enqueued_op; | |
409 | } | |
410 | ||
411 | static uint16_t | |
412 | snow3g_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, | |
413 | uint16_t nb_ops) | |
414 | { | |
415 | struct rte_crypto_op *c_ops[SNOW3G_MAX_BURST]; | |
416 | struct rte_crypto_op *curr_c_op; | |
417 | ||
418 | struct snow3g_session *prev_sess = NULL, *curr_sess = NULL; | |
419 | struct snow3g_qp *qp = queue_pair; | |
420 | unsigned i; | |
421 | uint8_t burst_size = 0; | |
422 | uint16_t enqueued_ops = 0; | |
423 | uint8_t processed_ops; | |
424 | ||
425 | for (i = 0; i < nb_ops; i++) { | |
426 | curr_c_op = ops[i]; | |
427 | ||
428 | /* Set status as enqueued (not processed yet) by default. */ | |
429 | curr_c_op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; | |
430 | ||
431 | curr_sess = snow3g_get_session(qp, curr_c_op); | |
432 | if (unlikely(curr_sess == NULL || | |
433 | curr_sess->op == SNOW3G_OP_NOT_SUPPORTED)) { | |
434 | curr_c_op->status = | |
435 | RTE_CRYPTO_OP_STATUS_INVALID_SESSION; | |
436 | break; | |
437 | } | |
438 | ||
439 | /* If length/offset is at bit-level, process this buffer alone. */ | |
440 | if (((curr_c_op->sym->cipher.data.length % BYTE_LEN) != 0) | |
441 | || ((curr_c_op->sym->cipher.data.offset | |
442 | % BYTE_LEN) != 0)) { | |
443 | /* Process the ops of the previous session. */ | |
444 | if (prev_sess != NULL) { | |
445 | processed_ops = process_ops(c_ops, prev_sess, | |
446 | qp, burst_size, &enqueued_ops); | |
447 | if (processed_ops < burst_size) { | |
448 | burst_size = 0; | |
449 | break; | |
450 | } | |
451 | ||
452 | burst_size = 0; | |
453 | prev_sess = NULL; | |
454 | } | |
455 | ||
456 | processed_ops = process_op_bit(curr_c_op, curr_sess, | |
457 | qp, &enqueued_ops); | |
458 | if (processed_ops != 1) | |
459 | break; | |
460 | ||
461 | continue; | |
462 | } | |
463 | ||
464 | /* Batch ops that share the same session. */ | |
465 | if (prev_sess == NULL) { | |
466 | prev_sess = curr_sess; | |
467 | c_ops[burst_size++] = curr_c_op; | |
468 | } else if (curr_sess == prev_sess) { | |
469 | c_ops[burst_size++] = curr_c_op; | |
470 | /* | |
471 | * When there are enough ops to process in a batch, | |
472 | * process them, and start a new batch. | |
473 | */ | |
474 | if (burst_size == SNOW3G_MAX_BURST) { | |
475 | processed_ops = process_ops(c_ops, prev_sess, | |
476 | qp, burst_size, &enqueued_ops); | |
477 | if (processed_ops < burst_size) { | |
478 | burst_size = 0; | |
479 | break; | |
480 | } | |
481 | ||
482 | burst_size = 0; | |
483 | prev_sess = NULL; | |
484 | } | |
485 | } else { | |
486 | /* | |
487 | * Different session, process the ops | |
488 | * of the previous session. | |
489 | */ | |
490 | processed_ops = process_ops(c_ops, prev_sess, | |
491 | qp, burst_size, &enqueued_ops); | |
492 | if (processed_ops < burst_size) { | |
493 | burst_size = 0; | |
494 | break; | |
495 | } | |
496 | ||
497 | burst_size = 0; | |
498 | prev_sess = curr_sess; | |
499 | ||
500 | c_ops[burst_size++] = curr_c_op; | |
501 | } | |
502 | } | |
503 | ||
504 | if (burst_size != 0) { | |
505 | /* Process the crypto ops of the last session. */ | |
506 | processed_ops = process_ops(c_ops, prev_sess, | |
507 | qp, burst_size, &enqueued_ops); | |
508 | } | |
509 | ||
510 | qp->qp_stats.enqueue_err_count += nb_ops - enqueued_ops; | |
511 | return enqueued_ops; | |
512 | } | |
513 | ||
514 | static uint16_t | |
515 | snow3g_pmd_dequeue_burst(void *queue_pair, | |
516 | struct rte_crypto_op **c_ops, uint16_t nb_ops) | |
517 | { | |
518 | struct snow3g_qp *qp = queue_pair; | |
519 | ||
520 | unsigned nb_dequeued; | |
521 | ||
522 | nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, | |
11fdf7f2 | 523 | (void **)c_ops, nb_ops, NULL); |
7c673cae FG |
524 | qp->qp_stats.dequeued_count += nb_dequeued; |
525 | ||
526 | return nb_dequeued; | |
527 | } | |
528 | ||
11fdf7f2 | 529 | static int cryptodev_snow3g_remove(struct rte_vdev_device *vdev); |
7c673cae FG |
530 | |
531 | static int | |
532 | cryptodev_snow3g_create(const char *name, | |
11fdf7f2 | 533 | struct rte_vdev_device *vdev, |
9f95a23c | 534 | struct rte_cryptodev_pmd_init_params *init_params) |
7c673cae FG |
535 | { |
536 | struct rte_cryptodev *dev; | |
7c673cae | 537 | struct snow3g_private *internals; |
9f95a23c | 538 | uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; |
7c673cae | 539 | |
9f95a23c | 540 | dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); |
7c673cae | 541 | if (dev == NULL) { |
9f95a23c | 542 | SNOW3G_LOG(ERR, "failed to create cryptodev vdev"); |
7c673cae FG |
543 | goto init_error; |
544 | } | |
545 | ||
9f95a23c | 546 | dev->driver_id = cryptodev_driver_id; |
7c673cae FG |
547 | dev->dev_ops = rte_snow3g_pmd_ops; |
548 | ||
549 | /* Register RX/TX burst functions for data path. */ | |
550 | dev->dequeue_burst = snow3g_pmd_dequeue_burst; | |
551 | dev->enqueue_burst = snow3g_pmd_enqueue_burst; | |
552 | ||
553 | dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | | |
554 | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | | |
555 | cpu_flags; | |
556 | ||
557 | internals = dev->data->dev_private; | |
558 | ||
559 | internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; | |
7c673cae FG |
560 | |
561 | return 0; | |
562 | init_error: | |
9f95a23c | 563 | SNOW3G_LOG(ERR, "driver %s: cryptodev_snow3g_create failed", |
11fdf7f2 | 564 | init_params->name); |
7c673cae | 565 | |
11fdf7f2 | 566 | cryptodev_snow3g_remove(vdev); |
7c673cae FG |
567 | return -EFAULT; |
568 | } | |
569 | ||
570 | static int | |
11fdf7f2 | 571 | cryptodev_snow3g_probe(struct rte_vdev_device *vdev) |
7c673cae | 572 | { |
9f95a23c TL |
573 | struct rte_cryptodev_pmd_init_params init_params = { |
574 | "", | |
575 | sizeof(struct snow3g_private), | |
11fdf7f2 | 576 | rte_socket_id(), |
9f95a23c | 577 | RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS |
7c673cae | 578 | }; |
11fdf7f2 TL |
579 | const char *name; |
580 | const char *input_args; | |
581 | ||
582 | name = rte_vdev_device_name(vdev); | |
583 | if (name == NULL) | |
584 | return -EINVAL; | |
585 | input_args = rte_vdev_device_args(vdev); | |
7c673cae | 586 | |
9f95a23c | 587 | rte_cryptodev_pmd_parse_input_args(&init_params, input_args); |
7c673cae | 588 | |
11fdf7f2 | 589 | return cryptodev_snow3g_create(name, vdev, &init_params); |
7c673cae FG |
590 | } |
591 | ||
592 | static int | |
11fdf7f2 | 593 | cryptodev_snow3g_remove(struct rte_vdev_device *vdev) |
7c673cae | 594 | { |
9f95a23c | 595 | struct rte_cryptodev *cryptodev; |
11fdf7f2 TL |
596 | const char *name; |
597 | ||
598 | name = rte_vdev_device_name(vdev); | |
7c673cae FG |
599 | if (name == NULL) |
600 | return -EINVAL; | |
601 | ||
9f95a23c TL |
602 | cryptodev = rte_cryptodev_pmd_get_named_dev(name); |
603 | if (cryptodev == NULL) | |
604 | return -ENODEV; | |
7c673cae | 605 | |
9f95a23c | 606 | return rte_cryptodev_pmd_destroy(cryptodev); |
7c673cae FG |
607 | } |
608 | ||
609 | static struct rte_vdev_driver cryptodev_snow3g_pmd_drv = { | |
610 | .probe = cryptodev_snow3g_probe, | |
611 | .remove = cryptodev_snow3g_remove | |
612 | }; | |
613 | ||
9f95a23c TL |
614 | static struct cryptodev_driver snow3g_crypto_drv; |
615 | ||
7c673cae FG |
616 | RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd_drv); |
617 | RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd); | |
618 | RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SNOW3G_PMD, | |
619 | "max_nb_queue_pairs=<int> " | |
7c673cae | 620 | "socket_id=<int>"); |
9f95a23c TL |
621 | RTE_PMD_REGISTER_CRYPTO_DRIVER(snow3g_crypto_drv, |
622 | cryptodev_snow3g_pmd_drv.driver, cryptodev_driver_id); | |
623 | ||
624 | RTE_INIT(snow3g_init_log) | |
625 | { | |
626 | snow3g_logtype_driver = rte_log_register("pmd.crypto.snow3g"); | |
627 | } |