]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright(c) 2016 Intel Corporation. All rights reserved. | |
5 | * | |
6 | * Redistribution and use in source and binary forms, with or without | |
7 | * modification, are permitted provided that the following conditions | |
8 | * are met: | |
9 | * | |
10 | * * Redistributions of source code must retain the above copyright | |
11 | * notice, this list of conditions and the following disclaimer. | |
12 | * * Redistributions in binary form must reproduce the above copyright | |
13 | * notice, this list of conditions and the following disclaimer in | |
14 | * the documentation and/or other materials provided with the | |
15 | * distribution. | |
16 | * * Neither the name of Intel Corporation nor the names of its | |
17 | * contributors may be used to endorse or promote products derived | |
18 | * from this software without specific prior written permission. | |
19 | * | |
20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
31 | */ | |
32 | ||
33 | #include <rte_common.h> | |
34 | #include <rte_config.h> | |
35 | #include <rte_hexdump.h> | |
36 | #include <rte_cryptodev.h> | |
37 | #include <rte_cryptodev_pmd.h> | |
38 | #include <rte_vdev.h> | |
39 | #include <rte_malloc.h> | |
40 | #include <rte_cpuflags.h> | |
41 | ||
42 | #include "rte_snow3g_pmd_private.h" | |
43 | ||
44 | #define SNOW3G_IV_LENGTH 16 | |
45 | #define SNOW3G_DIGEST_LENGTH 4 | |
46 | #define SNOW3G_MAX_BURST 8 | |
47 | #define BYTE_LEN 8 | |
48 | ||
49 | /** | |
50 | * Global static parameter used to create a unique name for each SNOW 3G | |
51 | * crypto device. | |
52 | */ | |
53 | static unsigned unique_name_id; | |
54 | ||
55 | static inline int | |
56 | create_unique_device_name(char *name, size_t size) | |
57 | { | |
58 | int ret; | |
59 | ||
60 | if (name == NULL) | |
61 | return -EINVAL; | |
62 | ||
63 | ret = snprintf(name, size, "%s_%u", RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD), | |
64 | unique_name_id++); | |
65 | if (ret < 0) | |
66 | return ret; | |
67 | return 0; | |
68 | } | |
69 | ||
70 | /** Get xform chain order. */ | |
71 | static enum snow3g_operation | |
72 | snow3g_get_mode(const struct rte_crypto_sym_xform *xform) | |
73 | { | |
74 | if (xform == NULL) | |
75 | return SNOW3G_OP_NOT_SUPPORTED; | |
76 | ||
77 | if (xform->next) | |
78 | if (xform->next->next != NULL) | |
79 | return SNOW3G_OP_NOT_SUPPORTED; | |
80 | ||
81 | if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { | |
82 | if (xform->next == NULL) | |
83 | return SNOW3G_OP_ONLY_AUTH; | |
84 | else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) | |
85 | return SNOW3G_OP_AUTH_CIPHER; | |
86 | else | |
87 | return SNOW3G_OP_NOT_SUPPORTED; | |
88 | } | |
89 | ||
90 | if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { | |
91 | if (xform->next == NULL) | |
92 | return SNOW3G_OP_ONLY_CIPHER; | |
93 | else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) | |
94 | return SNOW3G_OP_CIPHER_AUTH; | |
95 | else | |
96 | return SNOW3G_OP_NOT_SUPPORTED; | |
97 | } | |
98 | ||
99 | return SNOW3G_OP_NOT_SUPPORTED; | |
100 | } | |
101 | ||
102 | ||
103 | /** Parse crypto xform chain and set private session parameters. */ | |
104 | int | |
105 | snow3g_set_session_parameters(struct snow3g_session *sess, | |
106 | const struct rte_crypto_sym_xform *xform) | |
107 | { | |
108 | const struct rte_crypto_sym_xform *auth_xform = NULL; | |
109 | const struct rte_crypto_sym_xform *cipher_xform = NULL; | |
110 | enum snow3g_operation mode; | |
111 | ||
112 | /* Select Crypto operation - hash then cipher / cipher then hash */ | |
113 | mode = snow3g_get_mode(xform); | |
114 | ||
115 | switch (mode) { | |
116 | case SNOW3G_OP_CIPHER_AUTH: | |
117 | auth_xform = xform->next; | |
118 | ||
119 | /* Fall-through */ | |
120 | case SNOW3G_OP_ONLY_CIPHER: | |
121 | cipher_xform = xform; | |
122 | break; | |
123 | case SNOW3G_OP_AUTH_CIPHER: | |
124 | cipher_xform = xform->next; | |
125 | /* Fall-through */ | |
126 | case SNOW3G_OP_ONLY_AUTH: | |
127 | auth_xform = xform; | |
128 | break; | |
129 | case SNOW3G_OP_NOT_SUPPORTED: | |
130 | default: | |
131 | SNOW3G_LOG_ERR("Unsupported operation chain order parameter"); | |
132 | return -EINVAL; | |
133 | } | |
134 | ||
135 | if (cipher_xform) { | |
136 | /* Only SNOW 3G UEA2 supported */ | |
137 | if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_SNOW3G_UEA2) | |
138 | return -EINVAL; | |
139 | /* Initialize key */ | |
140 | sso_snow3g_init_key_sched(xform->cipher.key.data, | |
141 | &sess->pKeySched_cipher); | |
142 | } | |
143 | ||
144 | if (auth_xform) { | |
145 | /* Only SNOW 3G UIA2 supported */ | |
146 | if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_SNOW3G_UIA2) | |
147 | return -EINVAL; | |
148 | sess->auth_op = auth_xform->auth.op; | |
149 | /* Initialize key */ | |
150 | sso_snow3g_init_key_sched(xform->auth.key.data, | |
151 | &sess->pKeySched_hash); | |
152 | } | |
153 | ||
154 | ||
155 | sess->op = mode; | |
156 | ||
157 | return 0; | |
158 | } | |
159 | ||
160 | /** Get SNOW 3G session. */ | |
161 | static struct snow3g_session * | |
162 | snow3g_get_session(struct snow3g_qp *qp, struct rte_crypto_op *op) | |
163 | { | |
164 | struct snow3g_session *sess; | |
165 | ||
166 | if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { | |
167 | if (unlikely(op->sym->session->dev_type != | |
168 | RTE_CRYPTODEV_SNOW3G_PMD)) | |
169 | return NULL; | |
170 | ||
171 | sess = (struct snow3g_session *)op->sym->session->_private; | |
172 | } else { | |
173 | struct rte_cryptodev_session *c_sess = NULL; | |
174 | ||
175 | if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) | |
176 | return NULL; | |
177 | ||
178 | sess = (struct snow3g_session *)c_sess->_private; | |
179 | ||
180 | if (unlikely(snow3g_set_session_parameters(sess, | |
181 | op->sym->xform) != 0)) | |
182 | return NULL; | |
183 | } | |
184 | ||
185 | return sess; | |
186 | } | |
187 | ||
188 | /** Encrypt/decrypt mbufs with same cipher key. */ | |
189 | static uint8_t | |
190 | process_snow3g_cipher_op(struct rte_crypto_op **ops, | |
191 | struct snow3g_session *session, | |
192 | uint8_t num_ops) | |
193 | { | |
194 | unsigned i; | |
195 | uint8_t processed_ops = 0; | |
196 | uint8_t *src[SNOW3G_MAX_BURST], *dst[SNOW3G_MAX_BURST]; | |
197 | uint8_t *IV[SNOW3G_MAX_BURST]; | |
198 | uint32_t num_bytes[SNOW3G_MAX_BURST]; | |
199 | ||
200 | for (i = 0; i < num_ops; i++) { | |
201 | /* Sanity checks. */ | |
202 | if (unlikely(ops[i]->sym->cipher.iv.length != SNOW3G_IV_LENGTH)) { | |
203 | ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
204 | SNOW3G_LOG_ERR("iv"); | |
205 | break; | |
206 | } | |
207 | ||
208 | src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + | |
209 | (ops[i]->sym->cipher.data.offset >> 3); | |
210 | dst[i] = ops[i]->sym->m_dst ? | |
211 | rte_pktmbuf_mtod(ops[i]->sym->m_dst, uint8_t *) + | |
212 | (ops[i]->sym->cipher.data.offset >> 3) : | |
213 | rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + | |
214 | (ops[i]->sym->cipher.data.offset >> 3); | |
215 | IV[i] = ops[i]->sym->cipher.iv.data; | |
216 | num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; | |
217 | ||
218 | processed_ops++; | |
219 | } | |
220 | ||
221 | sso_snow3g_f8_n_buffer(&session->pKeySched_cipher, IV, src, dst, | |
222 | num_bytes, processed_ops); | |
223 | ||
224 | return processed_ops; | |
225 | } | |
226 | ||
227 | /** Encrypt/decrypt mbuf (bit level function). */ | |
228 | static uint8_t | |
229 | process_snow3g_cipher_op_bit(struct rte_crypto_op *op, | |
230 | struct snow3g_session *session) | |
231 | { | |
232 | uint8_t *src, *dst; | |
233 | uint8_t *IV; | |
234 | uint32_t length_in_bits, offset_in_bits; | |
235 | ||
236 | /* Sanity checks. */ | |
237 | if (unlikely(op->sym->cipher.iv.length != SNOW3G_IV_LENGTH)) { | |
238 | op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
239 | SNOW3G_LOG_ERR("iv"); | |
240 | return 0; | |
241 | } | |
242 | ||
243 | offset_in_bits = op->sym->cipher.data.offset; | |
244 | src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *); | |
245 | if (op->sym->m_dst == NULL) { | |
246 | op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
247 | SNOW3G_LOG_ERR("bit-level in-place not supported\n"); | |
248 | return 0; | |
249 | } | |
250 | dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *); | |
251 | IV = op->sym->cipher.iv.data; | |
252 | length_in_bits = op->sym->cipher.data.length; | |
253 | ||
254 | sso_snow3g_f8_1_buffer_bit(&session->pKeySched_cipher, IV, | |
255 | src, dst, length_in_bits, offset_in_bits); | |
256 | ||
257 | return 1; | |
258 | } | |
259 | ||
260 | /** Generate/verify hash from mbufs with same hash key. */ | |
261 | static int | |
262 | process_snow3g_hash_op(struct rte_crypto_op **ops, | |
263 | struct snow3g_session *session, | |
264 | uint8_t num_ops) | |
265 | { | |
266 | unsigned i; | |
267 | uint8_t processed_ops = 0; | |
268 | uint8_t *src, *dst; | |
269 | uint32_t length_in_bits; | |
270 | ||
271 | for (i = 0; i < num_ops; i++) { | |
272 | if (unlikely(ops[i]->sym->auth.aad.length != SNOW3G_IV_LENGTH)) { | |
273 | ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
274 | SNOW3G_LOG_ERR("aad"); | |
275 | break; | |
276 | } | |
277 | ||
278 | if (unlikely(ops[i]->sym->auth.digest.length != SNOW3G_DIGEST_LENGTH)) { | |
279 | ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
280 | SNOW3G_LOG_ERR("digest"); | |
281 | break; | |
282 | } | |
283 | ||
284 | /* Data must be byte aligned */ | |
285 | if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { | |
286 | ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
287 | SNOW3G_LOG_ERR("Offset"); | |
288 | break; | |
289 | } | |
290 | ||
291 | length_in_bits = ops[i]->sym->auth.data.length; | |
292 | ||
293 | src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + | |
294 | (ops[i]->sym->auth.data.offset >> 3); | |
295 | ||
296 | if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { | |
297 | dst = (uint8_t *)rte_pktmbuf_append(ops[i]->sym->m_src, | |
298 | ops[i]->sym->auth.digest.length); | |
299 | ||
300 | sso_snow3g_f9_1_buffer(&session->pKeySched_hash, | |
301 | ops[i]->sym->auth.aad.data, src, | |
302 | length_in_bits, dst); | |
303 | /* Verify digest. */ | |
304 | if (memcmp(dst, ops[i]->sym->auth.digest.data, | |
305 | ops[i]->sym->auth.digest.length) != 0) | |
306 | ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; | |
307 | ||
308 | /* Trim area used for digest from mbuf. */ | |
309 | rte_pktmbuf_trim(ops[i]->sym->m_src, | |
310 | ops[i]->sym->auth.digest.length); | |
311 | } else { | |
312 | dst = ops[i]->sym->auth.digest.data; | |
313 | ||
314 | sso_snow3g_f9_1_buffer(&session->pKeySched_hash, | |
315 | ops[i]->sym->auth.aad.data, src, | |
316 | length_in_bits, dst); | |
317 | } | |
318 | processed_ops++; | |
319 | } | |
320 | ||
321 | return processed_ops; | |
322 | } | |
323 | ||
324 | /** Process a batch of crypto ops which shares the same session. */ | |
325 | static int | |
326 | process_ops(struct rte_crypto_op **ops, struct snow3g_session *session, | |
327 | struct snow3g_qp *qp, uint8_t num_ops, | |
328 | uint16_t *accumulated_enqueued_ops) | |
329 | { | |
330 | unsigned i; | |
331 | unsigned enqueued_ops, processed_ops; | |
332 | ||
333 | switch (session->op) { | |
334 | case SNOW3G_OP_ONLY_CIPHER: | |
335 | processed_ops = process_snow3g_cipher_op(ops, | |
336 | session, num_ops); | |
337 | break; | |
338 | case SNOW3G_OP_ONLY_AUTH: | |
339 | processed_ops = process_snow3g_hash_op(ops, session, | |
340 | num_ops); | |
341 | break; | |
342 | case SNOW3G_OP_CIPHER_AUTH: | |
343 | processed_ops = process_snow3g_cipher_op(ops, session, | |
344 | num_ops); | |
345 | process_snow3g_hash_op(ops, session, processed_ops); | |
346 | break; | |
347 | case SNOW3G_OP_AUTH_CIPHER: | |
348 | processed_ops = process_snow3g_hash_op(ops, session, | |
349 | num_ops); | |
350 | process_snow3g_cipher_op(ops, session, processed_ops); | |
351 | break; | |
352 | default: | |
353 | /* Operation not supported. */ | |
354 | processed_ops = 0; | |
355 | } | |
356 | ||
357 | for (i = 0; i < num_ops; i++) { | |
358 | /* | |
359 | * If there was no error/authentication failure, | |
360 | * change status to successful. | |
361 | */ | |
362 | if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) | |
363 | ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; | |
364 | /* Free session if a session-less crypto op. */ | |
365 | if (ops[i]->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { | |
366 | rte_mempool_put(qp->sess_mp, ops[i]->sym->session); | |
367 | ops[i]->sym->session = NULL; | |
368 | } | |
369 | } | |
370 | ||
371 | enqueued_ops = rte_ring_enqueue_burst(qp->processed_ops, | |
372 | (void **)ops, processed_ops); | |
373 | qp->qp_stats.enqueued_count += enqueued_ops; | |
374 | *accumulated_enqueued_ops += enqueued_ops; | |
375 | ||
376 | return enqueued_ops; | |
377 | } | |
378 | ||
379 | /** Process a crypto op with length/offset in bits. */ | |
380 | static int | |
381 | process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, | |
382 | struct snow3g_qp *qp, uint16_t *accumulated_enqueued_ops) | |
383 | { | |
384 | unsigned enqueued_op, processed_op; | |
385 | ||
386 | switch (session->op) { | |
387 | case SNOW3G_OP_ONLY_CIPHER: | |
388 | processed_op = process_snow3g_cipher_op_bit(op, | |
389 | session); | |
390 | break; | |
391 | case SNOW3G_OP_ONLY_AUTH: | |
392 | processed_op = process_snow3g_hash_op(&op, session, 1); | |
393 | break; | |
394 | case SNOW3G_OP_CIPHER_AUTH: | |
395 | processed_op = process_snow3g_cipher_op_bit(op, session); | |
396 | if (processed_op == 1) | |
397 | process_snow3g_hash_op(&op, session, 1); | |
398 | break; | |
399 | case SNOW3G_OP_AUTH_CIPHER: | |
400 | processed_op = process_snow3g_hash_op(&op, session, 1); | |
401 | if (processed_op == 1) | |
402 | process_snow3g_cipher_op_bit(op, session); | |
403 | break; | |
404 | default: | |
405 | /* Operation not supported. */ | |
406 | processed_op = 0; | |
407 | } | |
408 | ||
409 | /* | |
410 | * If there was no error/authentication failure, | |
411 | * change status to successful. | |
412 | */ | |
413 | if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) | |
414 | op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; | |
415 | ||
416 | /* Free session if a session-less crypto op. */ | |
417 | if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { | |
418 | rte_mempool_put(qp->sess_mp, op->sym->session); | |
419 | op->sym->session = NULL; | |
420 | } | |
421 | ||
422 | enqueued_op = rte_ring_enqueue_burst(qp->processed_ops, | |
423 | (void **)&op, processed_op); | |
424 | qp->qp_stats.enqueued_count += enqueued_op; | |
425 | *accumulated_enqueued_ops += enqueued_op; | |
426 | ||
427 | return enqueued_op; | |
428 | } | |
429 | ||
430 | static uint16_t | |
431 | snow3g_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, | |
432 | uint16_t nb_ops) | |
433 | { | |
434 | struct rte_crypto_op *c_ops[SNOW3G_MAX_BURST]; | |
435 | struct rte_crypto_op *curr_c_op; | |
436 | ||
437 | struct snow3g_session *prev_sess = NULL, *curr_sess = NULL; | |
438 | struct snow3g_qp *qp = queue_pair; | |
439 | unsigned i; | |
440 | uint8_t burst_size = 0; | |
441 | uint16_t enqueued_ops = 0; | |
442 | uint8_t processed_ops; | |
443 | ||
444 | for (i = 0; i < nb_ops; i++) { | |
445 | curr_c_op = ops[i]; | |
446 | ||
447 | /* Set status as enqueued (not processed yet) by default. */ | |
448 | curr_c_op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; | |
449 | ||
450 | curr_sess = snow3g_get_session(qp, curr_c_op); | |
451 | if (unlikely(curr_sess == NULL || | |
452 | curr_sess->op == SNOW3G_OP_NOT_SUPPORTED)) { | |
453 | curr_c_op->status = | |
454 | RTE_CRYPTO_OP_STATUS_INVALID_SESSION; | |
455 | break; | |
456 | } | |
457 | ||
458 | /* If length/offset is at bit-level, process this buffer alone. */ | |
459 | if (((curr_c_op->sym->cipher.data.length % BYTE_LEN) != 0) | |
460 | || ((curr_c_op->sym->cipher.data.offset | |
461 | % BYTE_LEN) != 0)) { | |
462 | /* Process the ops of the previous session. */ | |
463 | if (prev_sess != NULL) { | |
464 | processed_ops = process_ops(c_ops, prev_sess, | |
465 | qp, burst_size, &enqueued_ops); | |
466 | if (processed_ops < burst_size) { | |
467 | burst_size = 0; | |
468 | break; | |
469 | } | |
470 | ||
471 | burst_size = 0; | |
472 | prev_sess = NULL; | |
473 | } | |
474 | ||
475 | processed_ops = process_op_bit(curr_c_op, curr_sess, | |
476 | qp, &enqueued_ops); | |
477 | if (processed_ops != 1) | |
478 | break; | |
479 | ||
480 | continue; | |
481 | } | |
482 | ||
483 | /* Batch ops that share the same session. */ | |
484 | if (prev_sess == NULL) { | |
485 | prev_sess = curr_sess; | |
486 | c_ops[burst_size++] = curr_c_op; | |
487 | } else if (curr_sess == prev_sess) { | |
488 | c_ops[burst_size++] = curr_c_op; | |
489 | /* | |
490 | * When there are enough ops to process in a batch, | |
491 | * process them, and start a new batch. | |
492 | */ | |
493 | if (burst_size == SNOW3G_MAX_BURST) { | |
494 | processed_ops = process_ops(c_ops, prev_sess, | |
495 | qp, burst_size, &enqueued_ops); | |
496 | if (processed_ops < burst_size) { | |
497 | burst_size = 0; | |
498 | break; | |
499 | } | |
500 | ||
501 | burst_size = 0; | |
502 | prev_sess = NULL; | |
503 | } | |
504 | } else { | |
505 | /* | |
506 | * Different session, process the ops | |
507 | * of the previous session. | |
508 | */ | |
509 | processed_ops = process_ops(c_ops, prev_sess, | |
510 | qp, burst_size, &enqueued_ops); | |
511 | if (processed_ops < burst_size) { | |
512 | burst_size = 0; | |
513 | break; | |
514 | } | |
515 | ||
516 | burst_size = 0; | |
517 | prev_sess = curr_sess; | |
518 | ||
519 | c_ops[burst_size++] = curr_c_op; | |
520 | } | |
521 | } | |
522 | ||
523 | if (burst_size != 0) { | |
524 | /* Process the crypto ops of the last session. */ | |
525 | processed_ops = process_ops(c_ops, prev_sess, | |
526 | qp, burst_size, &enqueued_ops); | |
527 | } | |
528 | ||
529 | qp->qp_stats.enqueue_err_count += nb_ops - enqueued_ops; | |
530 | return enqueued_ops; | |
531 | } | |
532 | ||
533 | static uint16_t | |
534 | snow3g_pmd_dequeue_burst(void *queue_pair, | |
535 | struct rte_crypto_op **c_ops, uint16_t nb_ops) | |
536 | { | |
537 | struct snow3g_qp *qp = queue_pair; | |
538 | ||
539 | unsigned nb_dequeued; | |
540 | ||
541 | nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, | |
542 | (void **)c_ops, nb_ops); | |
543 | qp->qp_stats.dequeued_count += nb_dequeued; | |
544 | ||
545 | return nb_dequeued; | |
546 | } | |
547 | ||
548 | static int cryptodev_snow3g_remove(const char *name); | |
549 | ||
550 | static int | |
551 | cryptodev_snow3g_create(const char *name, | |
552 | struct rte_crypto_vdev_init_params *init_params) | |
553 | { | |
554 | struct rte_cryptodev *dev; | |
555 | char crypto_dev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; | |
556 | struct snow3g_private *internals; | |
557 | uint64_t cpu_flags = 0; | |
558 | ||
559 | /* Check CPU for supported vector instruction set */ | |
560 | if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) | |
561 | cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; | |
562 | else { | |
563 | SNOW3G_LOG_ERR("Vector instructions are not supported by CPU"); | |
564 | return -EFAULT; | |
565 | } | |
566 | ||
567 | ||
568 | /* Create a unique device name. */ | |
569 | if (create_unique_device_name(crypto_dev_name, | |
570 | RTE_CRYPTODEV_NAME_MAX_LEN) != 0) { | |
571 | SNOW3G_LOG_ERR("failed to create unique cryptodev name"); | |
572 | return -EINVAL; | |
573 | } | |
574 | ||
575 | dev = rte_cryptodev_pmd_virtual_dev_init(crypto_dev_name, | |
576 | sizeof(struct snow3g_private), init_params->socket_id); | |
577 | if (dev == NULL) { | |
578 | SNOW3G_LOG_ERR("failed to create cryptodev vdev"); | |
579 | goto init_error; | |
580 | } | |
581 | ||
582 | dev->dev_type = RTE_CRYPTODEV_SNOW3G_PMD; | |
583 | dev->dev_ops = rte_snow3g_pmd_ops; | |
584 | ||
585 | /* Register RX/TX burst functions for data path. */ | |
586 | dev->dequeue_burst = snow3g_pmd_dequeue_burst; | |
587 | dev->enqueue_burst = snow3g_pmd_enqueue_burst; | |
588 | ||
589 | dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | | |
590 | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | | |
591 | cpu_flags; | |
592 | ||
593 | internals = dev->data->dev_private; | |
594 | ||
595 | internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; | |
596 | internals->max_nb_sessions = init_params->max_nb_sessions; | |
597 | ||
598 | return 0; | |
599 | init_error: | |
600 | SNOW3G_LOG_ERR("driver %s: cryptodev_snow3g_create failed", name); | |
601 | ||
602 | cryptodev_snow3g_remove(crypto_dev_name); | |
603 | return -EFAULT; | |
604 | } | |
605 | ||
606 | static int | |
607 | cryptodev_snow3g_probe(const char *name, | |
608 | const char *input_args) | |
609 | { | |
610 | struct rte_crypto_vdev_init_params init_params = { | |
611 | RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, | |
612 | RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, | |
613 | rte_socket_id() | |
614 | }; | |
615 | ||
616 | rte_cryptodev_parse_vdev_init_params(&init_params, input_args); | |
617 | ||
618 | RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, | |
619 | init_params.socket_id); | |
620 | RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", | |
621 | init_params.max_nb_queue_pairs); | |
622 | RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", | |
623 | init_params.max_nb_sessions); | |
624 | ||
625 | return cryptodev_snow3g_create(name, &init_params); | |
626 | } | |
627 | ||
628 | static int | |
629 | cryptodev_snow3g_remove(const char *name) | |
630 | { | |
631 | if (name == NULL) | |
632 | return -EINVAL; | |
633 | ||
634 | RTE_LOG(INFO, PMD, "Closing SNOW 3G crypto device %s" | |
635 | " on numa socket %u\n", | |
636 | name, rte_socket_id()); | |
637 | ||
638 | return 0; | |
639 | } | |
640 | ||
641 | static struct rte_vdev_driver cryptodev_snow3g_pmd_drv = { | |
642 | .probe = cryptodev_snow3g_probe, | |
643 | .remove = cryptodev_snow3g_remove | |
644 | }; | |
645 | ||
646 | RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd_drv); | |
647 | RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd); | |
648 | RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SNOW3G_PMD, | |
649 | "max_nb_queue_pairs=<int> " | |
650 | "max_nb_sessions=<int> " | |
651 | "socket_id=<int>"); |