]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright(c) 2016 Intel Corporation. All rights reserved. | |
5 | * | |
6 | * Redistribution and use in source and binary forms, with or without | |
7 | * modification, are permitted provided that the following conditions | |
8 | * are met: | |
9 | * | |
10 | * * Redistributions of source code must retain the above copyright | |
11 | * notice, this list of conditions and the following disclaimer. | |
12 | * * Redistributions in binary form must reproduce the above copyright | |
13 | * notice, this list of conditions and the following disclaimer in | |
14 | * the documentation and/or other materials provided with the | |
15 | * distribution. | |
16 | * * Neither the name of Intel Corporation nor the names of its | |
17 | * contributors may be used to endorse or promote products derived | |
18 | * from this software without specific prior written permission. | |
19 | * | |
20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
31 | */ | |
32 | ||
33 | #include <rte_common.h> | |
34 | #include <rte_hexdump.h> | |
35 | #include <rte_cryptodev.h> | |
36 | #include <rte_cryptodev_pmd.h> | |
37 | #include <rte_vdev.h> | |
38 | #include <rte_malloc.h> | |
39 | #include <rte_cpuflags.h> | |
40 | ||
41 | #include <openssl/evp.h> | |
42 | ||
43 | #include "rte_openssl_pmd_private.h" | |
44 | ||
45 | static int cryptodev_openssl_remove(const char *name); | |
46 | ||
47 | /*----------------------------------------------------------------------------*/ | |
48 | ||
49 | /** | |
50 | * Global static parameter used to create a unique name for each | |
51 | * OPENSSL crypto device. | |
52 | */ | |
53 | static unsigned int unique_name_id; | |
54 | ||
55 | static inline int | |
56 | create_unique_device_name(char *name, size_t size) | |
57 | { | |
58 | int ret; | |
59 | ||
60 | if (name == NULL) | |
61 | return -EINVAL; | |
62 | ||
63 | ret = snprintf(name, size, "%s_%u", | |
64 | RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD), | |
65 | unique_name_id++); | |
66 | if (ret < 0) | |
67 | return ret; | |
68 | return 0; | |
69 | } | |
70 | ||
71 | /** | |
72 | * Increment counter by 1 | |
73 | * Counter is 64 bit array, big-endian | |
74 | */ | |
75 | static void | |
76 | ctr_inc(uint8_t *ctr) | |
77 | { | |
78 | uint64_t *ctr64 = (uint64_t *)ctr; | |
79 | ||
80 | *ctr64 = __builtin_bswap64(*ctr64); | |
81 | (*ctr64)++; | |
82 | *ctr64 = __builtin_bswap64(*ctr64); | |
83 | } | |
84 | ||
85 | /* | |
86 | *------------------------------------------------------------------------------ | |
87 | * Session Prepare | |
88 | *------------------------------------------------------------------------------ | |
89 | */ | |
90 | ||
91 | /** Get xform chain order */ | |
92 | static enum openssl_chain_order | |
93 | openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) | |
94 | { | |
95 | enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; | |
96 | ||
97 | if (xform != NULL) { | |
98 | if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { | |
99 | if (xform->next == NULL) | |
100 | res = OPENSSL_CHAIN_ONLY_AUTH; | |
101 | else if (xform->next->type == | |
102 | RTE_CRYPTO_SYM_XFORM_CIPHER) | |
103 | res = OPENSSL_CHAIN_AUTH_CIPHER; | |
104 | } | |
105 | if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { | |
106 | if (xform->next == NULL) | |
107 | res = OPENSSL_CHAIN_ONLY_CIPHER; | |
108 | else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) | |
109 | res = OPENSSL_CHAIN_CIPHER_AUTH; | |
110 | } | |
111 | } | |
112 | ||
113 | return res; | |
114 | } | |
115 | ||
116 | /** Get session cipher key from input cipher key */ | |
117 | static void | |
118 | get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) | |
119 | { | |
120 | memcpy(session_key, input_key, keylen); | |
121 | } | |
122 | ||
123 | /** Get key ede 24 bytes standard from input key */ | |
124 | static int | |
125 | get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) | |
126 | { | |
127 | int res = 0; | |
128 | ||
129 | /* Initialize keys - 24 bytes: [key1-key2-key3] */ | |
130 | switch (keylen) { | |
131 | case 24: | |
132 | memcpy(key_ede, key, 24); | |
133 | break; | |
134 | case 16: | |
135 | /* K3 = K1 */ | |
136 | memcpy(key_ede, key, 16); | |
137 | memcpy(key_ede + 16, key, 8); | |
138 | break; | |
139 | case 8: | |
140 | /* K1 = K2 = K3 (DES compatibility) */ | |
141 | memcpy(key_ede, key, 8); | |
142 | memcpy(key_ede + 8, key, 8); | |
143 | memcpy(key_ede + 16, key, 8); | |
144 | break; | |
145 | default: | |
146 | OPENSSL_LOG_ERR("Unsupported key size"); | |
147 | res = -EINVAL; | |
148 | } | |
149 | ||
150 | return res; | |
151 | } | |
152 | ||
153 | /** Get adequate openssl function for input cipher algorithm */ | |
154 | static uint8_t | |
155 | get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, | |
156 | const EVP_CIPHER **algo) | |
157 | { | |
158 | int res = 0; | |
159 | ||
160 | if (algo != NULL) { | |
161 | switch (sess_algo) { | |
162 | case RTE_CRYPTO_CIPHER_3DES_CBC: | |
163 | switch (keylen) { | |
164 | case 16: | |
165 | *algo = EVP_des_ede_cbc(); | |
166 | break; | |
167 | case 24: | |
168 | *algo = EVP_des_ede3_cbc(); | |
169 | break; | |
170 | default: | |
171 | res = -EINVAL; | |
172 | } | |
173 | break; | |
174 | case RTE_CRYPTO_CIPHER_3DES_CTR: | |
175 | break; | |
176 | case RTE_CRYPTO_CIPHER_AES_CBC: | |
177 | switch (keylen) { | |
178 | case 16: | |
179 | *algo = EVP_aes_128_cbc(); | |
180 | break; | |
181 | case 24: | |
182 | *algo = EVP_aes_192_cbc(); | |
183 | break; | |
184 | case 32: | |
185 | *algo = EVP_aes_256_cbc(); | |
186 | break; | |
187 | default: | |
188 | res = -EINVAL; | |
189 | } | |
190 | break; | |
191 | case RTE_CRYPTO_CIPHER_AES_CTR: | |
192 | switch (keylen) { | |
193 | case 16: | |
194 | *algo = EVP_aes_128_ctr(); | |
195 | break; | |
196 | case 24: | |
197 | *algo = EVP_aes_192_ctr(); | |
198 | break; | |
199 | case 32: | |
200 | *algo = EVP_aes_256_ctr(); | |
201 | break; | |
202 | default: | |
203 | res = -EINVAL; | |
204 | } | |
205 | break; | |
206 | case RTE_CRYPTO_CIPHER_AES_GCM: | |
207 | switch (keylen) { | |
208 | case 16: | |
209 | *algo = EVP_aes_128_gcm(); | |
210 | break; | |
211 | case 24: | |
212 | *algo = EVP_aes_192_gcm(); | |
213 | break; | |
214 | case 32: | |
215 | *algo = EVP_aes_256_gcm(); | |
216 | break; | |
217 | default: | |
218 | res = -EINVAL; | |
219 | } | |
220 | break; | |
221 | default: | |
222 | res = -EINVAL; | |
223 | break; | |
224 | } | |
225 | } else { | |
226 | res = -EINVAL; | |
227 | } | |
228 | ||
229 | return res; | |
230 | } | |
231 | ||
232 | /** Get adequate openssl function for input auth algorithm */ | |
233 | static uint8_t | |
234 | get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, | |
235 | const EVP_MD **algo) | |
236 | { | |
237 | int res = 0; | |
238 | ||
239 | if (algo != NULL) { | |
240 | switch (sessalgo) { | |
241 | case RTE_CRYPTO_AUTH_MD5: | |
242 | case RTE_CRYPTO_AUTH_MD5_HMAC: | |
243 | *algo = EVP_md5(); | |
244 | break; | |
245 | case RTE_CRYPTO_AUTH_SHA1: | |
246 | case RTE_CRYPTO_AUTH_SHA1_HMAC: | |
247 | *algo = EVP_sha1(); | |
248 | break; | |
249 | case RTE_CRYPTO_AUTH_SHA224: | |
250 | case RTE_CRYPTO_AUTH_SHA224_HMAC: | |
251 | *algo = EVP_sha224(); | |
252 | break; | |
253 | case RTE_CRYPTO_AUTH_SHA256: | |
254 | case RTE_CRYPTO_AUTH_SHA256_HMAC: | |
255 | *algo = EVP_sha256(); | |
256 | break; | |
257 | case RTE_CRYPTO_AUTH_SHA384: | |
258 | case RTE_CRYPTO_AUTH_SHA384_HMAC: | |
259 | *algo = EVP_sha384(); | |
260 | break; | |
261 | case RTE_CRYPTO_AUTH_SHA512: | |
262 | case RTE_CRYPTO_AUTH_SHA512_HMAC: | |
263 | *algo = EVP_sha512(); | |
264 | break; | |
265 | default: | |
266 | res = -EINVAL; | |
267 | break; | |
268 | } | |
269 | } else { | |
270 | res = -EINVAL; | |
271 | } | |
272 | ||
273 | return res; | |
274 | } | |
275 | ||
276 | /** Set session cipher parameters */ | |
277 | static int | |
278 | openssl_set_session_cipher_parameters(struct openssl_session *sess, | |
279 | const struct rte_crypto_sym_xform *xform) | |
280 | { | |
281 | /* Select cipher direction */ | |
282 | sess->cipher.direction = xform->cipher.op; | |
283 | /* Select cipher key */ | |
284 | sess->cipher.key.length = xform->cipher.key.length; | |
285 | ||
286 | /* Select cipher algo */ | |
287 | switch (xform->cipher.algo) { | |
288 | case RTE_CRYPTO_CIPHER_3DES_CBC: | |
289 | case RTE_CRYPTO_CIPHER_AES_CBC: | |
290 | case RTE_CRYPTO_CIPHER_AES_CTR: | |
291 | case RTE_CRYPTO_CIPHER_AES_GCM: | |
292 | sess->cipher.mode = OPENSSL_CIPHER_LIB; | |
293 | sess->cipher.algo = xform->cipher.algo; | |
294 | sess->cipher.ctx = EVP_CIPHER_CTX_new(); | |
295 | ||
296 | if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, | |
297 | &sess->cipher.evp_algo) != 0) | |
298 | return -EINVAL; | |
299 | ||
300 | get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, | |
301 | sess->cipher.key.data); | |
302 | ||
303 | break; | |
304 | ||
305 | case RTE_CRYPTO_CIPHER_3DES_CTR: | |
306 | sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; | |
307 | sess->cipher.ctx = EVP_CIPHER_CTX_new(); | |
308 | ||
309 | if (get_cipher_key_ede(xform->cipher.key.data, | |
310 | sess->cipher.key.length, | |
311 | sess->cipher.key.data) != 0) | |
312 | return -EINVAL; | |
313 | break; | |
314 | ||
315 | default: | |
316 | sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; | |
317 | return -EINVAL; | |
318 | } | |
319 | ||
320 | return 0; | |
321 | } | |
322 | ||
323 | /* Set session auth parameters */ | |
324 | static int | |
325 | openssl_set_session_auth_parameters(struct openssl_session *sess, | |
326 | const struct rte_crypto_sym_xform *xform) | |
327 | { | |
328 | /* Select auth generate/verify */ | |
329 | sess->auth.operation = xform->auth.op; | |
330 | sess->auth.algo = xform->auth.algo; | |
331 | ||
332 | /* Select auth algo */ | |
333 | switch (xform->auth.algo) { | |
334 | case RTE_CRYPTO_AUTH_AES_GMAC: | |
335 | case RTE_CRYPTO_AUTH_AES_GCM: | |
336 | /* Check additional condition for AES_GMAC/GCM */ | |
337 | if (sess->cipher.algo != RTE_CRYPTO_CIPHER_AES_GCM) | |
338 | return -EINVAL; | |
339 | sess->chain_order = OPENSSL_CHAIN_COMBINED; | |
340 | break; | |
341 | ||
342 | case RTE_CRYPTO_AUTH_MD5: | |
343 | case RTE_CRYPTO_AUTH_SHA1: | |
344 | case RTE_CRYPTO_AUTH_SHA224: | |
345 | case RTE_CRYPTO_AUTH_SHA256: | |
346 | case RTE_CRYPTO_AUTH_SHA384: | |
347 | case RTE_CRYPTO_AUTH_SHA512: | |
348 | sess->auth.mode = OPENSSL_AUTH_AS_AUTH; | |
349 | if (get_auth_algo(xform->auth.algo, | |
350 | &sess->auth.auth.evp_algo) != 0) | |
351 | return -EINVAL; | |
352 | sess->auth.auth.ctx = EVP_MD_CTX_create(); | |
353 | break; | |
354 | ||
355 | case RTE_CRYPTO_AUTH_MD5_HMAC: | |
356 | case RTE_CRYPTO_AUTH_SHA1_HMAC: | |
357 | case RTE_CRYPTO_AUTH_SHA224_HMAC: | |
358 | case RTE_CRYPTO_AUTH_SHA256_HMAC: | |
359 | case RTE_CRYPTO_AUTH_SHA384_HMAC: | |
360 | case RTE_CRYPTO_AUTH_SHA512_HMAC: | |
361 | sess->auth.mode = OPENSSL_AUTH_AS_HMAC; | |
362 | sess->auth.hmac.ctx = EVP_MD_CTX_create(); | |
363 | if (get_auth_algo(xform->auth.algo, | |
364 | &sess->auth.hmac.evp_algo) != 0) | |
365 | return -EINVAL; | |
366 | sess->auth.hmac.pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, | |
367 | xform->auth.key.data, xform->auth.key.length); | |
368 | break; | |
369 | ||
370 | default: | |
371 | return -EINVAL; | |
372 | } | |
373 | ||
374 | return 0; | |
375 | } | |
376 | ||
377 | /** Parse crypto xform chain and set private session parameters */ | |
378 | int | |
379 | openssl_set_session_parameters(struct openssl_session *sess, | |
380 | const struct rte_crypto_sym_xform *xform) | |
381 | { | |
382 | const struct rte_crypto_sym_xform *cipher_xform = NULL; | |
383 | const struct rte_crypto_sym_xform *auth_xform = NULL; | |
384 | ||
385 | sess->chain_order = openssl_get_chain_order(xform); | |
386 | switch (sess->chain_order) { | |
387 | case OPENSSL_CHAIN_ONLY_CIPHER: | |
388 | cipher_xform = xform; | |
389 | break; | |
390 | case OPENSSL_CHAIN_ONLY_AUTH: | |
391 | auth_xform = xform; | |
392 | break; | |
393 | case OPENSSL_CHAIN_CIPHER_AUTH: | |
394 | cipher_xform = xform; | |
395 | auth_xform = xform->next; | |
396 | break; | |
397 | case OPENSSL_CHAIN_AUTH_CIPHER: | |
398 | auth_xform = xform; | |
399 | cipher_xform = xform->next; | |
400 | break; | |
401 | default: | |
402 | return -EINVAL; | |
403 | } | |
404 | ||
405 | /* cipher_xform must be check before auth_xform */ | |
406 | if (cipher_xform) { | |
407 | if (openssl_set_session_cipher_parameters( | |
408 | sess, cipher_xform)) { | |
409 | OPENSSL_LOG_ERR( | |
410 | "Invalid/unsupported cipher parameters"); | |
411 | return -EINVAL; | |
412 | } | |
413 | } | |
414 | ||
415 | if (auth_xform) { | |
416 | if (openssl_set_session_auth_parameters(sess, auth_xform)) { | |
417 | OPENSSL_LOG_ERR( | |
418 | "Invalid/unsupported auth parameters"); | |
419 | return -EINVAL; | |
420 | } | |
421 | } | |
422 | ||
423 | return 0; | |
424 | } | |
425 | ||
426 | /** Reset private session parameters */ | |
427 | void | |
428 | openssl_reset_session(struct openssl_session *sess) | |
429 | { | |
430 | EVP_CIPHER_CTX_free(sess->cipher.ctx); | |
431 | ||
432 | switch (sess->auth.mode) { | |
433 | case OPENSSL_AUTH_AS_AUTH: | |
434 | EVP_MD_CTX_destroy(sess->auth.auth.ctx); | |
435 | break; | |
436 | case OPENSSL_AUTH_AS_HMAC: | |
437 | EVP_PKEY_free(sess->auth.hmac.pkey); | |
438 | EVP_MD_CTX_destroy(sess->auth.hmac.ctx); | |
439 | break; | |
440 | default: | |
441 | break; | |
442 | } | |
443 | } | |
444 | ||
445 | /** Provide session for operation */ | |
446 | static struct openssl_session * | |
447 | get_session(struct openssl_qp *qp, struct rte_crypto_op *op) | |
448 | { | |
449 | struct openssl_session *sess = NULL; | |
450 | ||
451 | if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { | |
452 | /* get existing session */ | |
453 | if (likely(op->sym->session != NULL && | |
454 | op->sym->session->dev_type == | |
455 | RTE_CRYPTODEV_OPENSSL_PMD)) | |
456 | sess = (struct openssl_session *) | |
457 | op->sym->session->_private; | |
458 | } else { | |
459 | /* provide internal session */ | |
460 | void *_sess = NULL; | |
461 | ||
462 | if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { | |
463 | sess = (struct openssl_session *) | |
464 | ((struct rte_cryptodev_sym_session *)_sess) | |
465 | ->_private; | |
466 | ||
467 | if (unlikely(openssl_set_session_parameters( | |
468 | sess, op->sym->xform) != 0)) { | |
469 | rte_mempool_put(qp->sess_mp, _sess); | |
470 | sess = NULL; | |
471 | } else | |
472 | op->sym->session = _sess; | |
473 | } | |
474 | } | |
475 | ||
476 | if (sess == NULL) | |
477 | op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; | |
478 | ||
479 | return sess; | |
480 | } | |
481 | ||
482 | /* | |
483 | *------------------------------------------------------------------------------ | |
484 | * Process Operations | |
485 | *------------------------------------------------------------------------------ | |
486 | */ | |
487 | ||
488 | /** Process standard openssl cipher encryption */ | |
489 | static int | |
490 | process_openssl_cipher_encrypt(uint8_t *src, uint8_t *dst, | |
491 | uint8_t *iv, uint8_t *key, int srclen, | |
492 | EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) | |
493 | { | |
494 | int dstlen, totlen; | |
495 | ||
496 | if (EVP_EncryptInit_ex(ctx, algo, NULL, key, iv) <= 0) | |
497 | goto process_cipher_encrypt_err; | |
498 | ||
499 | if (EVP_EncryptUpdate(ctx, dst, &dstlen, src, srclen) <= 0) | |
500 | goto process_cipher_encrypt_err; | |
501 | ||
502 | if (EVP_EncryptFinal_ex(ctx, dst + dstlen, &totlen) <= 0) | |
503 | goto process_cipher_encrypt_err; | |
504 | ||
505 | return 0; | |
506 | ||
507 | process_cipher_encrypt_err: | |
508 | OPENSSL_LOG_ERR("Process openssl cipher encrypt failed"); | |
509 | return -EINVAL; | |
510 | } | |
511 | ||
512 | /** Process standard openssl cipher decryption */ | |
513 | static int | |
514 | process_openssl_cipher_decrypt(uint8_t *src, uint8_t *dst, | |
515 | uint8_t *iv, uint8_t *key, int srclen, | |
516 | EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) | |
517 | { | |
518 | int dstlen, totlen; | |
519 | ||
520 | if (EVP_DecryptInit_ex(ctx, algo, NULL, key, iv) <= 0) | |
521 | goto process_cipher_decrypt_err; | |
522 | ||
523 | if (EVP_CIPHER_CTX_set_padding(ctx, 0) <= 0) | |
524 | goto process_cipher_decrypt_err; | |
525 | ||
526 | if (EVP_DecryptUpdate(ctx, dst, &dstlen, src, srclen) <= 0) | |
527 | goto process_cipher_decrypt_err; | |
528 | ||
529 | if (EVP_DecryptFinal_ex(ctx, dst + dstlen, &totlen) <= 0) | |
530 | goto process_cipher_decrypt_err; | |
531 | ||
532 | return 0; | |
533 | ||
534 | process_cipher_decrypt_err: | |
535 | OPENSSL_LOG_ERR("Process openssl cipher decrypt failed"); | |
536 | return -EINVAL; | |
537 | } | |
538 | ||
539 | /** Process cipher des 3 ctr encryption, decryption algorithm */ | |
540 | static int | |
541 | process_openssl_cipher_des3ctr(uint8_t *src, uint8_t *dst, | |
542 | uint8_t *iv, uint8_t *key, int srclen, EVP_CIPHER_CTX *ctx) | |
543 | { | |
544 | uint8_t ebuf[8], ctr[8]; | |
545 | int unused, n; | |
546 | ||
547 | /* We use 3DES encryption also for decryption. | |
548 | * IV is not important for 3DES ecb | |
549 | */ | |
550 | if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) | |
551 | goto process_cipher_des3ctr_err; | |
552 | ||
553 | memcpy(ctr, iv, 8); | |
554 | n = 0; | |
555 | ||
556 | while (n < srclen) { | |
557 | if (n % 8 == 0) { | |
558 | if (EVP_EncryptUpdate(ctx, | |
559 | (unsigned char *)&ebuf, &unused, | |
560 | (const unsigned char *)&ctr, 8) <= 0) | |
561 | goto process_cipher_des3ctr_err; | |
562 | ctr_inc(ctr); | |
563 | } | |
564 | dst[n] = src[n] ^ ebuf[n % 8]; | |
565 | n++; | |
566 | } | |
567 | ||
568 | return 0; | |
569 | ||
570 | process_cipher_des3ctr_err: | |
571 | OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed"); | |
572 | return -EINVAL; | |
573 | } | |
574 | ||
575 | /** Process auth/encription aes-gcm algorithm */ | |
576 | static int | |
577 | process_openssl_auth_encryption_gcm(uint8_t *src, int srclen, | |
578 | uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, | |
579 | uint8_t *key, uint8_t *dst, uint8_t *tag, | |
580 | EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) | |
581 | { | |
582 | int len = 0, unused = 0; | |
583 | uint8_t empty[] = {}; | |
584 | ||
585 | if (EVP_EncryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0) | |
586 | goto process_auth_encryption_gcm_err; | |
587 | ||
588 | if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0) | |
589 | goto process_auth_encryption_gcm_err; | |
590 | ||
591 | if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) | |
592 | goto process_auth_encryption_gcm_err; | |
593 | ||
594 | if (aadlen > 0) { | |
595 | if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) | |
596 | goto process_auth_encryption_gcm_err; | |
597 | ||
598 | /* Workaround open ssl bug in version less then 1.0.1f */ | |
599 | if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) | |
600 | goto process_auth_encryption_gcm_err; | |
601 | } | |
602 | ||
603 | if (srclen > 0) | |
604 | if (EVP_EncryptUpdate(ctx, dst, &len, src, srclen) <= 0) | |
605 | goto process_auth_encryption_gcm_err; | |
606 | ||
607 | if (EVP_EncryptFinal_ex(ctx, dst + len, &len) <= 0) | |
608 | goto process_auth_encryption_gcm_err; | |
609 | ||
610 | if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) | |
611 | goto process_auth_encryption_gcm_err; | |
612 | ||
613 | return 0; | |
614 | ||
615 | process_auth_encryption_gcm_err: | |
616 | OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed"); | |
617 | return -EINVAL; | |
618 | } | |
619 | ||
620 | static int | |
621 | process_openssl_auth_decryption_gcm(uint8_t *src, int srclen, | |
622 | uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, | |
623 | uint8_t *key, uint8_t *dst, uint8_t *tag, | |
624 | EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) | |
625 | { | |
626 | int len = 0, unused = 0; | |
627 | uint8_t empty[] = {}; | |
628 | ||
629 | if (EVP_DecryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0) | |
630 | goto process_auth_decryption_gcm_err; | |
631 | ||
632 | if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0) | |
633 | goto process_auth_decryption_gcm_err; | |
634 | ||
635 | if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) | |
636 | goto process_auth_decryption_gcm_err; | |
637 | ||
638 | if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) | |
639 | goto process_auth_decryption_gcm_err; | |
640 | ||
641 | if (aadlen > 0) { | |
642 | if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) | |
643 | goto process_auth_decryption_gcm_err; | |
644 | ||
645 | /* Workaround open ssl bug in version less then 1.0.1f */ | |
646 | if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) | |
647 | goto process_auth_decryption_gcm_err; | |
648 | } | |
649 | ||
650 | if (srclen > 0) | |
651 | if (EVP_DecryptUpdate(ctx, dst, &len, src, srclen) <= 0) | |
652 | goto process_auth_decryption_gcm_err; | |
653 | ||
654 | if (EVP_DecryptFinal_ex(ctx, dst + len, &len) <= 0) | |
655 | goto process_auth_decryption_gcm_final_err; | |
656 | ||
657 | return 0; | |
658 | ||
659 | process_auth_decryption_gcm_err: | |
660 | OPENSSL_LOG_ERR("Process openssl auth description gcm failed"); | |
661 | return -EINVAL; | |
662 | ||
663 | process_auth_decryption_gcm_final_err: | |
664 | return -EFAULT; | |
665 | } | |
666 | ||
667 | /** Process standard openssl auth algorithms */ | |
668 | static int | |
669 | process_openssl_auth(uint8_t *src, uint8_t *dst, | |
670 | __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, | |
671 | int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) | |
672 | { | |
673 | size_t dstlen; | |
674 | ||
675 | if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) | |
676 | goto process_auth_err; | |
677 | ||
678 | if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) | |
679 | goto process_auth_err; | |
680 | ||
681 | if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) | |
682 | goto process_auth_err; | |
683 | ||
684 | return 0; | |
685 | ||
686 | process_auth_err: | |
687 | OPENSSL_LOG_ERR("Process openssl auth failed"); | |
688 | return -EINVAL; | |
689 | } | |
690 | ||
691 | /** Process standard openssl auth algorithms with hmac */ | |
692 | static int | |
693 | process_openssl_auth_hmac(uint8_t *src, uint8_t *dst, | |
694 | __rte_unused uint8_t *iv, EVP_PKEY *pkey, | |
695 | int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) | |
696 | { | |
697 | size_t dstlen; | |
698 | ||
699 | if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0) | |
700 | goto process_auth_err; | |
701 | ||
702 | if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0) | |
703 | goto process_auth_err; | |
704 | ||
705 | if (EVP_DigestSignFinal(ctx, dst, &dstlen) <= 0) | |
706 | goto process_auth_err; | |
707 | ||
708 | return 0; | |
709 | ||
710 | process_auth_err: | |
711 | OPENSSL_LOG_ERR("Process openssl auth failed"); | |
712 | return -EINVAL; | |
713 | } | |
714 | ||
715 | /*----------------------------------------------------------------------------*/ | |
716 | ||
717 | /** Process auth/cipher combined operation */ | |
718 | static void | |
719 | process_openssl_combined_op | |
720 | (struct rte_crypto_op *op, struct openssl_session *sess, | |
721 | struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) | |
722 | { | |
723 | /* cipher */ | |
724 | uint8_t *src = NULL, *dst = NULL, *iv, *tag, *aad; | |
725 | int srclen, ivlen, aadlen, status = -1; | |
726 | ||
727 | iv = op->sym->cipher.iv.data; | |
728 | ivlen = op->sym->cipher.iv.length; | |
729 | aad = op->sym->auth.aad.data; | |
730 | aadlen = op->sym->auth.aad.length; | |
731 | ||
732 | tag = op->sym->auth.digest.data; | |
733 | if (tag == NULL) | |
734 | tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, | |
735 | op->sym->cipher.data.offset + | |
736 | op->sym->cipher.data.length); | |
737 | ||
738 | if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) | |
739 | srclen = 0; | |
740 | else { | |
741 | srclen = op->sym->cipher.data.length; | |
742 | src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, | |
743 | op->sym->cipher.data.offset); | |
744 | dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, | |
745 | op->sym->cipher.data.offset); | |
746 | } | |
747 | ||
748 | if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) | |
749 | status = process_openssl_auth_encryption_gcm( | |
750 | src, srclen, aad, aadlen, iv, ivlen, | |
751 | sess->cipher.key.data, dst, tag, | |
752 | sess->cipher.ctx, sess->cipher.evp_algo); | |
753 | else | |
754 | status = process_openssl_auth_decryption_gcm( | |
755 | src, srclen, aad, aadlen, iv, ivlen, | |
756 | sess->cipher.key.data, dst, tag, | |
757 | sess->cipher.ctx, sess->cipher.evp_algo); | |
758 | ||
759 | if (status != 0) { | |
760 | if (status == (-EFAULT) && | |
761 | sess->auth.operation == | |
762 | RTE_CRYPTO_AUTH_OP_VERIFY) | |
763 | op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; | |
764 | else | |
765 | op->status = RTE_CRYPTO_OP_STATUS_ERROR; | |
766 | } | |
767 | } | |
768 | ||
769 | /** Process cipher operation */ | |
770 | static void | |
771 | process_openssl_cipher_op | |
772 | (struct rte_crypto_op *op, struct openssl_session *sess, | |
773 | struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) | |
774 | { | |
775 | uint8_t *src, *dst, *iv; | |
776 | int srclen, status; | |
777 | ||
778 | srclen = op->sym->cipher.data.length; | |
779 | src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, | |
780 | op->sym->cipher.data.offset); | |
781 | dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, | |
782 | op->sym->cipher.data.offset); | |
783 | ||
784 | iv = op->sym->cipher.iv.data; | |
785 | ||
786 | if (sess->cipher.mode == OPENSSL_CIPHER_LIB) | |
787 | if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) | |
788 | status = process_openssl_cipher_encrypt(src, dst, iv, | |
789 | sess->cipher.key.data, srclen, | |
790 | sess->cipher.ctx, | |
791 | sess->cipher.evp_algo); | |
792 | else | |
793 | status = process_openssl_cipher_decrypt(src, dst, iv, | |
794 | sess->cipher.key.data, srclen, | |
795 | sess->cipher.ctx, | |
796 | sess->cipher.evp_algo); | |
797 | else | |
798 | status = process_openssl_cipher_des3ctr(src, dst, iv, | |
799 | sess->cipher.key.data, srclen, | |
800 | sess->cipher.ctx); | |
801 | ||
802 | if (status != 0) | |
803 | op->status = RTE_CRYPTO_OP_STATUS_ERROR; | |
804 | } | |
805 | ||
806 | /** Process auth operation */ | |
807 | static void | |
808 | process_openssl_auth_op | |
809 | (struct rte_crypto_op *op, struct openssl_session *sess, | |
810 | struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) | |
811 | { | |
812 | uint8_t *src, *dst; | |
813 | int srclen, status; | |
814 | ||
815 | srclen = op->sym->auth.data.length; | |
816 | src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, | |
817 | op->sym->auth.data.offset); | |
818 | ||
819 | if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) | |
820 | dst = (uint8_t *)rte_pktmbuf_append(mbuf_src, | |
821 | op->sym->auth.digest.length); | |
822 | else { | |
823 | dst = op->sym->auth.digest.data; | |
824 | if (dst == NULL) | |
825 | dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, | |
826 | op->sym->auth.data.offset + | |
827 | op->sym->auth.data.length); | |
828 | } | |
829 | ||
830 | switch (sess->auth.mode) { | |
831 | case OPENSSL_AUTH_AS_AUTH: | |
832 | status = process_openssl_auth(src, dst, | |
833 | NULL, NULL, srclen, | |
834 | sess->auth.auth.ctx, sess->auth.auth.evp_algo); | |
835 | break; | |
836 | case OPENSSL_AUTH_AS_HMAC: | |
837 | status = process_openssl_auth_hmac(src, dst, | |
838 | NULL, sess->auth.hmac.pkey, srclen, | |
839 | sess->auth.hmac.ctx, sess->auth.hmac.evp_algo); | |
840 | break; | |
841 | default: | |
842 | status = -1; | |
843 | break; | |
844 | } | |
845 | ||
846 | if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { | |
847 | if (memcmp(dst, op->sym->auth.digest.data, | |
848 | op->sym->auth.digest.length) != 0) { | |
849 | op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; | |
850 | } | |
851 | /* Trim area used for digest from mbuf. */ | |
852 | rte_pktmbuf_trim(mbuf_src, | |
853 | op->sym->auth.digest.length); | |
854 | } | |
855 | ||
856 | if (status != 0) | |
857 | op->status = RTE_CRYPTO_OP_STATUS_ERROR; | |
858 | } | |
859 | ||
860 | /** Process crypto operation for mbuf */ | |
861 | static int | |
862 | process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, | |
863 | struct openssl_session *sess) | |
864 | { | |
865 | struct rte_mbuf *msrc, *mdst; | |
866 | int retval; | |
867 | ||
868 | msrc = op->sym->m_src; | |
869 | mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; | |
870 | ||
871 | op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; | |
872 | ||
873 | switch (sess->chain_order) { | |
874 | case OPENSSL_CHAIN_ONLY_CIPHER: | |
875 | process_openssl_cipher_op(op, sess, msrc, mdst); | |
876 | break; | |
877 | case OPENSSL_CHAIN_ONLY_AUTH: | |
878 | process_openssl_auth_op(op, sess, msrc, mdst); | |
879 | break; | |
880 | case OPENSSL_CHAIN_CIPHER_AUTH: | |
881 | process_openssl_cipher_op(op, sess, msrc, mdst); | |
882 | process_openssl_auth_op(op, sess, mdst, mdst); | |
883 | break; | |
884 | case OPENSSL_CHAIN_AUTH_CIPHER: | |
885 | process_openssl_auth_op(op, sess, msrc, mdst); | |
886 | process_openssl_cipher_op(op, sess, msrc, mdst); | |
887 | break; | |
888 | case OPENSSL_CHAIN_COMBINED: | |
889 | process_openssl_combined_op(op, sess, msrc, mdst); | |
890 | break; | |
891 | default: | |
892 | op->status = RTE_CRYPTO_OP_STATUS_ERROR; | |
893 | break; | |
894 | } | |
895 | ||
896 | /* Free session if a session-less crypto op */ | |
897 | if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { | |
898 | openssl_reset_session(sess); | |
899 | memset(sess, 0, sizeof(struct openssl_session)); | |
900 | rte_mempool_put(qp->sess_mp, op->sym->session); | |
901 | op->sym->session = NULL; | |
902 | } | |
903 | ||
904 | ||
905 | if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) | |
906 | op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; | |
907 | ||
908 | if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) | |
909 | retval = rte_ring_enqueue(qp->processed_ops, (void *)op); | |
910 | else | |
911 | retval = -1; | |
912 | ||
913 | return retval; | |
914 | } | |
915 | ||
916 | /* | |
917 | *------------------------------------------------------------------------------ | |
918 | * PMD Framework | |
919 | *------------------------------------------------------------------------------ | |
920 | */ | |
921 | ||
922 | /** Enqueue burst */ | |
923 | static uint16_t | |
924 | openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, | |
925 | uint16_t nb_ops) | |
926 | { | |
927 | struct openssl_session *sess; | |
928 | struct openssl_qp *qp = queue_pair; | |
929 | int i, retval; | |
930 | ||
931 | for (i = 0; i < nb_ops; i++) { | |
932 | sess = get_session(qp, ops[i]); | |
933 | if (unlikely(sess == NULL)) | |
934 | goto enqueue_err; | |
935 | ||
936 | retval = process_op(qp, ops[i], sess); | |
937 | if (unlikely(retval < 0)) | |
938 | goto enqueue_err; | |
939 | } | |
940 | ||
941 | qp->stats.enqueued_count += i; | |
942 | return i; | |
943 | ||
944 | enqueue_err: | |
945 | qp->stats.enqueue_err_count++; | |
946 | return i; | |
947 | } | |
948 | ||
949 | /** Dequeue burst */ | |
950 | static uint16_t | |
951 | openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, | |
952 | uint16_t nb_ops) | |
953 | { | |
954 | struct openssl_qp *qp = queue_pair; | |
955 | ||
956 | unsigned int nb_dequeued = 0; | |
957 | ||
958 | nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, | |
959 | (void **)ops, nb_ops); | |
960 | qp->stats.dequeued_count += nb_dequeued; | |
961 | ||
962 | return nb_dequeued; | |
963 | } | |
964 | ||
965 | /** Create OPENSSL crypto device */ | |
966 | static int | |
967 | cryptodev_openssl_create(const char *name, | |
968 | struct rte_crypto_vdev_init_params *init_params) | |
969 | { | |
970 | struct rte_cryptodev *dev; | |
971 | char crypto_dev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; | |
972 | struct openssl_private *internals; | |
973 | ||
974 | /* create a unique device name */ | |
975 | if (create_unique_device_name(crypto_dev_name, | |
976 | RTE_CRYPTODEV_NAME_MAX_LEN) != 0) { | |
977 | OPENSSL_LOG_ERR("failed to create unique cryptodev name"); | |
978 | return -EINVAL; | |
979 | } | |
980 | ||
981 | dev = rte_cryptodev_pmd_virtual_dev_init(crypto_dev_name, | |
982 | sizeof(struct openssl_private), | |
983 | init_params->socket_id); | |
984 | if (dev == NULL) { | |
985 | OPENSSL_LOG_ERR("failed to create cryptodev vdev"); | |
986 | goto init_error; | |
987 | } | |
988 | ||
989 | dev->dev_type = RTE_CRYPTODEV_OPENSSL_PMD; | |
990 | dev->dev_ops = rte_openssl_pmd_ops; | |
991 | ||
992 | /* register rx/tx burst functions for data path */ | |
993 | dev->dequeue_burst = openssl_pmd_dequeue_burst; | |
994 | dev->enqueue_burst = openssl_pmd_enqueue_burst; | |
995 | ||
996 | dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | | |
997 | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | | |
998 | RTE_CRYPTODEV_FF_CPU_AESNI; | |
999 | ||
1000 | /* Set vector instructions mode supported */ | |
1001 | internals = dev->data->dev_private; | |
1002 | ||
1003 | internals->max_nb_qpairs = init_params->max_nb_queue_pairs; | |
1004 | internals->max_nb_sessions = init_params->max_nb_sessions; | |
1005 | ||
1006 | return 0; | |
1007 | ||
1008 | init_error: | |
1009 | OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed", name); | |
1010 | ||
1011 | cryptodev_openssl_remove(crypto_dev_name); | |
1012 | return -EFAULT; | |
1013 | } | |
1014 | ||
1015 | /** Initialise OPENSSL crypto device */ | |
1016 | static int | |
1017 | cryptodev_openssl_probe(const char *name, | |
1018 | const char *input_args) | |
1019 | { | |
1020 | struct rte_crypto_vdev_init_params init_params = { | |
1021 | RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, | |
1022 | RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, | |
1023 | rte_socket_id() | |
1024 | }; | |
1025 | ||
1026 | rte_cryptodev_parse_vdev_init_params(&init_params, input_args); | |
1027 | ||
1028 | RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, | |
1029 | init_params.socket_id); | |
1030 | RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", | |
1031 | init_params.max_nb_queue_pairs); | |
1032 | RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", | |
1033 | init_params.max_nb_sessions); | |
1034 | ||
1035 | return cryptodev_openssl_create(name, &init_params); | |
1036 | } | |
1037 | ||
1038 | /** Uninitialise OPENSSL crypto device */ | |
1039 | static int | |
1040 | cryptodev_openssl_remove(const char *name) | |
1041 | { | |
1042 | if (name == NULL) | |
1043 | return -EINVAL; | |
1044 | ||
1045 | RTE_LOG(INFO, PMD, | |
1046 | "Closing OPENSSL crypto device %s on numa socket %u\n", | |
1047 | name, rte_socket_id()); | |
1048 | ||
1049 | return 0; | |
1050 | } | |
1051 | ||
1052 | static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { | |
1053 | .probe = cryptodev_openssl_probe, | |
1054 | .remove = cryptodev_openssl_remove | |
1055 | }; | |
1056 | ||
1057 | RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, | |
1058 | cryptodev_openssl_pmd_drv); | |
1059 | RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, | |
1060 | "max_nb_queue_pairs=<int> " | |
1061 | "max_nb_sessions=<int> " | |
1062 | "socket_id=<int>"); |