4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * This file represents the QAT implementation of checksums and encryption.
24 * Internally, QAT shares the same cryptographic instances for both of these
25 * operations, so the code has been combined here. QAT data compression uses
26 * compression instances, so that code is separated into qat_compress.c
29 #if defined(_KERNEL) && defined(HAVE_QAT)
30 #include <linux/slab.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/completion.h>
34 #include <linux/mod_compat.h>
35 #include <sys/zfs_context.h>
36 #include <sys/zio_crypt.h>
37 #include "lac/cpa_cy_im.h"
38 #include "lac/cpa_cy_common.h"
42 * Max instances in a QAT device, each instance is a channel to submit
43 * jobs to QAT hardware, this is only for pre-allocating instances
44 * and session arrays; the actual number of instances are defined in
45 * the QAT driver's configure file.
47 #define QAT_CRYPT_MAX_INSTANCES 48
49 #define MAX_PAGE_NUM 1024
51 static Cpa32U inst_num
= 0;
52 static Cpa16U num_inst
= 0;
53 static CpaInstanceHandle cy_inst_handles
[QAT_CRYPT_MAX_INSTANCES
];
54 static boolean_t qat_cy_init_done
= B_FALSE
;
55 int zfs_qat_encrypt_disable
= 0;
56 int zfs_qat_checksum_disable
= 0;
58 typedef struct cy_callback
{
59 CpaBoolean verify_result
;
60 struct completion complete
;
64 symcallback(void *p_callback
, CpaStatus status
, const CpaCySymOp operation
,
65 void *op_data
, CpaBufferList
*buf_list_dst
, CpaBoolean verify
)
67 cy_callback_t
*cb
= p_callback
;
70 /* indicate that the function has been called */
71 cb
->verify_result
= verify
;
72 complete(&cb
->complete
);
77 qat_crypt_use_accel(size_t s_len
)
79 return (!zfs_qat_encrypt_disable
&&
81 s_len
>= QAT_MIN_BUF_SIZE
&&
82 s_len
<= QAT_MAX_BUF_SIZE
);
86 qat_checksum_use_accel(size_t s_len
)
88 return (!zfs_qat_checksum_disable
&&
90 s_len
>= QAT_MIN_BUF_SIZE
&&
91 s_len
<= QAT_MAX_BUF_SIZE
);
97 for (Cpa16U i
= 0; i
< num_inst
; i
++)
98 cpaCyStopInstance(cy_inst_handles
[i
]);
101 qat_cy_init_done
= B_FALSE
;
107 CpaStatus status
= CPA_STATUS_FAIL
;
109 if (qat_cy_init_done
)
112 status
= cpaCyGetNumInstances(&num_inst
);
113 if (status
!= CPA_STATUS_SUCCESS
)
116 /* if the user has configured no QAT encryption units just return */
120 if (num_inst
> QAT_CRYPT_MAX_INSTANCES
)
121 num_inst
= QAT_CRYPT_MAX_INSTANCES
;
123 status
= cpaCyGetInstances(num_inst
, &cy_inst_handles
[0]);
124 if (status
!= CPA_STATUS_SUCCESS
)
127 for (Cpa16U i
= 0; i
< num_inst
; i
++) {
128 status
= cpaCySetAddressTranslation(cy_inst_handles
[i
],
129 (void *)virt_to_phys
);
130 if (status
!= CPA_STATUS_SUCCESS
)
133 status
= cpaCyStartInstance(cy_inst_handles
[i
]);
134 if (status
!= CPA_STATUS_SUCCESS
)
138 qat_cy_init_done
= B_TRUE
;
149 if (!qat_cy_init_done
)
156 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir
, CpaInstanceHandle inst_handle
,
157 CpaCySymSessionCtx
**cy_session_ctx
, crypto_key_t
*key
,
158 Cpa64U crypt
, Cpa32U aad_len
)
160 CpaStatus status
= CPA_STATUS_SUCCESS
;
162 Cpa32U ciper_algorithm
;
163 Cpa32U hash_algorithm
;
164 CpaCySymSessionSetupData sd
= { 0 };
166 if (zio_crypt_table
[crypt
].ci_crypt_type
== ZC_TYPE_CCM
) {
167 return (CPA_STATUS_FAIL
);
169 ciper_algorithm
= CPA_CY_SYM_CIPHER_AES_GCM
;
170 hash_algorithm
= CPA_CY_SYM_HASH_AES_GCM
;
173 sd
.cipherSetupData
.cipherAlgorithm
= ciper_algorithm
;
174 sd
.cipherSetupData
.pCipherKey
= key
->ck_data
;
175 sd
.cipherSetupData
.cipherKeyLenInBytes
= key
->ck_length
/ 8;
176 sd
.hashSetupData
.hashAlgorithm
= hash_algorithm
;
177 sd
.hashSetupData
.hashMode
= CPA_CY_SYM_HASH_MODE_AUTH
;
178 sd
.hashSetupData
.digestResultLenInBytes
= ZIO_DATA_MAC_LEN
;
179 sd
.hashSetupData
.authModeSetupData
.aadLenInBytes
= aad_len
;
180 sd
.sessionPriority
= CPA_CY_PRIORITY_NORMAL
;
181 sd
.symOperation
= CPA_CY_SYM_OP_ALGORITHM_CHAINING
;
182 sd
.digestIsAppended
= CPA_FALSE
;
183 sd
.verifyDigest
= CPA_FALSE
;
185 if (dir
== QAT_ENCRYPT
) {
186 sd
.cipherSetupData
.cipherDirection
=
187 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT
;
189 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER
;
191 ASSERT3U(dir
, ==, QAT_DECRYPT
);
192 sd
.cipherSetupData
.cipherDirection
=
193 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT
;
195 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH
;
198 status
= cpaCySymSessionCtxGetSize(inst_handle
, &sd
, &ctx_size
);
199 if (status
!= CPA_STATUS_SUCCESS
)
202 status
= QAT_PHYS_CONTIG_ALLOC(cy_session_ctx
, ctx_size
);
203 if (status
!= CPA_STATUS_SUCCESS
)
206 status
= cpaCySymInitSession(inst_handle
, symcallback
, &sd
,
208 if (status
!= CPA_STATUS_SUCCESS
) {
209 QAT_PHYS_CONTIG_FREE(*cy_session_ctx
);
213 return (CPA_STATUS_SUCCESS
);
217 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle
,
218 CpaCySymSessionCtx
**cy_session_ctx
, Cpa64U cksum
)
220 CpaStatus status
= CPA_STATUS_SUCCESS
;
222 Cpa32U hash_algorithm
;
223 CpaCySymSessionSetupData sd
= { 0 };
226 * ZFS's SHA512 checksum is actually SHA512/256, which uses
227 * a different IV from standard SHA512. QAT does not support
228 * SHA512/256, so we can only support SHA256.
230 if (cksum
== ZIO_CHECKSUM_SHA256
)
231 hash_algorithm
= CPA_CY_SYM_HASH_SHA256
;
233 return (CPA_STATUS_FAIL
);
235 sd
.sessionPriority
= CPA_CY_PRIORITY_NORMAL
;
236 sd
.symOperation
= CPA_CY_SYM_OP_HASH
;
237 sd
.hashSetupData
.hashAlgorithm
= hash_algorithm
;
238 sd
.hashSetupData
.hashMode
= CPA_CY_SYM_HASH_MODE_PLAIN
;
239 sd
.hashSetupData
.digestResultLenInBytes
= sizeof (zio_cksum_t
);
240 sd
.digestIsAppended
= CPA_FALSE
;
241 sd
.verifyDigest
= CPA_FALSE
;
243 status
= cpaCySymSessionCtxGetSize(inst_handle
, &sd
, &ctx_size
);
244 if (status
!= CPA_STATUS_SUCCESS
)
247 status
= QAT_PHYS_CONTIG_ALLOC(cy_session_ctx
, ctx_size
);
248 if (status
!= CPA_STATUS_SUCCESS
)
251 status
= cpaCySymInitSession(inst_handle
, symcallback
, &sd
,
253 if (status
!= CPA_STATUS_SUCCESS
) {
254 QAT_PHYS_CONTIG_FREE(*cy_session_ctx
);
258 return (CPA_STATUS_SUCCESS
);
262 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle
, uint32_t nr_bufs
,
263 CpaBufferList
*src
, CpaBufferList
*dst
)
265 CpaStatus status
= CPA_STATUS_SUCCESS
;
266 Cpa32U meta_size
= 0;
268 status
= cpaCyBufferListGetMetaSize(inst_handle
, nr_bufs
, &meta_size
);
269 if (status
!= CPA_STATUS_SUCCESS
)
272 status
= QAT_PHYS_CONTIG_ALLOC(&src
->pPrivateMetaData
, meta_size
);
273 if (status
!= CPA_STATUS_SUCCESS
)
277 status
= QAT_PHYS_CONTIG_ALLOC(&dst
->pPrivateMetaData
,
279 if (status
!= CPA_STATUS_SUCCESS
)
283 return (CPA_STATUS_SUCCESS
);
286 QAT_PHYS_CONTIG_FREE(src
->pPrivateMetaData
);
288 QAT_PHYS_CONTIG_FREE(dst
->pPrivateMetaData
);
294 qat_crypt(qat_encrypt_dir_t dir
, uint8_t *src_buf
, uint8_t *dst_buf
,
295 uint8_t *aad_buf
, uint32_t aad_len
, uint8_t *iv_buf
, uint8_t *digest_buf
,
296 crypto_key_t
*key
, uint64_t crypt
, uint32_t enc_len
)
298 CpaStatus status
= CPA_STATUS_SUCCESS
;
300 CpaInstanceHandle cy_inst_handle
;
301 Cpa16U nr_bufs
= (enc_len
>> PAGE_SHIFT
) + 2;
302 Cpa32U bytes_left
= 0;
304 CpaCySymSessionCtx
*cy_session_ctx
= NULL
;
306 CpaCySymOpData op_data
= { 0 };
307 CpaBufferList src_buffer_list
= { 0 };
308 CpaBufferList dst_buffer_list
= { 0 };
309 CpaFlatBuffer
*flat_src_buf_array
= NULL
;
310 CpaFlatBuffer
*flat_src_buf
= NULL
;
311 CpaFlatBuffer
*flat_dst_buf_array
= NULL
;
312 CpaFlatBuffer
*flat_dst_buf
= NULL
;
313 struct page
*in_pages
[MAX_PAGE_NUM
];
314 struct page
*out_pages
[MAX_PAGE_NUM
];
315 Cpa32U in_page_num
= 0;
316 Cpa32U out_page_num
= 0;
317 Cpa32U in_page_off
= 0;
318 Cpa32U out_page_off
= 0;
320 if (dir
== QAT_ENCRYPT
) {
321 QAT_STAT_BUMP(encrypt_requests
);
322 QAT_STAT_INCR(encrypt_total_in_bytes
, enc_len
);
324 QAT_STAT_BUMP(decrypt_requests
);
325 QAT_STAT_INCR(decrypt_total_in_bytes
, enc_len
);
328 i
= (Cpa32U
)atomic_inc_32_nv(&inst_num
) % num_inst
;
329 cy_inst_handle
= cy_inst_handles
[i
];
331 status
= qat_init_crypt_session_ctx(dir
, cy_inst_handle
,
332 &cy_session_ctx
, key
, crypt
, aad_len
);
333 if (status
!= CPA_STATUS_SUCCESS
) {
334 /* don't count CCM as a failure since it's not supported */
335 if (zio_crypt_table
[crypt
].ci_crypt_type
== ZC_TYPE_GCM
)
336 QAT_STAT_BUMP(crypt_fails
);
341 * We increment nr_bufs by 2 to allow us to handle non
342 * page-aligned buffer addresses and buffers whose sizes
343 * are not divisible by PAGE_SIZE.
345 status
= qat_init_cy_buffer_lists(cy_inst_handle
, nr_bufs
,
346 &src_buffer_list
, &dst_buffer_list
);
347 if (status
!= CPA_STATUS_SUCCESS
)
350 status
= QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array
,
351 nr_bufs
* sizeof (CpaFlatBuffer
));
352 if (status
!= CPA_STATUS_SUCCESS
)
354 status
= QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array
,
355 nr_bufs
* sizeof (CpaFlatBuffer
));
356 if (status
!= CPA_STATUS_SUCCESS
)
358 status
= QAT_PHYS_CONTIG_ALLOC(&op_data
.pDigestResult
,
360 if (status
!= CPA_STATUS_SUCCESS
)
362 status
= QAT_PHYS_CONTIG_ALLOC(&op_data
.pIv
,
364 if (status
!= CPA_STATUS_SUCCESS
)
367 status
= QAT_PHYS_CONTIG_ALLOC(&op_data
.pAdditionalAuthData
,
369 if (status
!= CPA_STATUS_SUCCESS
)
371 bcopy(aad_buf
, op_data
.pAdditionalAuthData
, aad_len
);
374 bytes_left
= enc_len
;
376 flat_src_buf
= flat_src_buf_array
;
377 while (bytes_left
> 0) {
378 in_page_off
= ((long)data
& ~PAGE_MASK
);
379 in_pages
[in_page_num
] = qat_mem_to_page(data
);
380 flat_src_buf
->pData
= kmap(in_pages
[in_page_num
]) + in_page_off
;
381 flat_src_buf
->dataLenInBytes
=
382 min((long)PAGE_SIZE
- in_page_off
, (long)bytes_left
);
383 data
+= flat_src_buf
->dataLenInBytes
;
384 bytes_left
-= flat_src_buf
->dataLenInBytes
;
388 src_buffer_list
.pBuffers
= flat_src_buf_array
;
389 src_buffer_list
.numBuffers
= in_page_num
;
391 bytes_left
= enc_len
;
393 flat_dst_buf
= flat_dst_buf_array
;
394 while (bytes_left
> 0) {
395 out_page_off
= ((long)data
& ~PAGE_MASK
);
396 out_pages
[out_page_num
] = qat_mem_to_page(data
);
397 flat_dst_buf
->pData
= kmap(out_pages
[out_page_num
]) +
399 flat_dst_buf
->dataLenInBytes
=
400 min((long)PAGE_SIZE
- out_page_off
, (long)bytes_left
);
401 data
+= flat_dst_buf
->dataLenInBytes
;
402 bytes_left
-= flat_dst_buf
->dataLenInBytes
;
406 dst_buffer_list
.pBuffers
= flat_dst_buf_array
;
407 dst_buffer_list
.numBuffers
= out_page_num
;
409 op_data
.sessionCtx
= cy_session_ctx
;
410 op_data
.packetType
= CPA_CY_SYM_PACKET_TYPE_FULL
;
411 op_data
.cryptoStartSrcOffsetInBytes
= 0;
412 op_data
.messageLenToCipherInBytes
= 0;
413 op_data
.hashStartSrcOffsetInBytes
= 0;
414 op_data
.messageLenToHashInBytes
= 0;
415 op_data
.messageLenToCipherInBytes
= enc_len
;
416 op_data
.ivLenInBytes
= ZIO_DATA_IV_LEN
;
417 bcopy(iv_buf
, op_data
.pIv
, ZIO_DATA_IV_LEN
);
419 cb
.verify_result
= CPA_FALSE
;
420 init_completion(&cb
.complete
);
421 status
= cpaCySymPerformOp(cy_inst_handle
, &cb
, &op_data
,
422 &src_buffer_list
, &dst_buffer_list
, NULL
);
423 if (status
!= CPA_STATUS_SUCCESS
)
426 if (!wait_for_completion_interruptible_timeout(&cb
.complete
,
428 status
= CPA_STATUS_FAIL
;
432 if (cb
.verify_result
== CPA_FALSE
) {
433 status
= CPA_STATUS_FAIL
;
437 /* save digest result to digest_buf */
438 bcopy(op_data
.pDigestResult
, digest_buf
, ZIO_DATA_MAC_LEN
);
439 if (dir
== QAT_ENCRYPT
)
440 QAT_STAT_INCR(encrypt_total_out_bytes
, enc_len
);
442 QAT_STAT_INCR(decrypt_total_out_bytes
, enc_len
);
445 if (status
!= CPA_STATUS_SUCCESS
)
446 QAT_STAT_BUMP(crypt_fails
);
448 for (i
= 0; i
< in_page_num
; i
++)
450 for (i
= 0; i
< out_page_num
; i
++)
451 kunmap(out_pages
[i
]);
453 cpaCySymRemoveSession(cy_inst_handle
, cy_session_ctx
);
455 QAT_PHYS_CONTIG_FREE(op_data
.pAdditionalAuthData
);
456 QAT_PHYS_CONTIG_FREE(op_data
.pIv
);
457 QAT_PHYS_CONTIG_FREE(op_data
.pDigestResult
);
458 QAT_PHYS_CONTIG_FREE(src_buffer_list
.pPrivateMetaData
);
459 QAT_PHYS_CONTIG_FREE(dst_buffer_list
.pPrivateMetaData
);
460 QAT_PHYS_CONTIG_FREE(cy_session_ctx
);
461 QAT_PHYS_CONTIG_FREE(flat_src_buf_array
);
462 QAT_PHYS_CONTIG_FREE(flat_dst_buf_array
);
468 qat_checksum(uint64_t cksum
, uint8_t *buf
, uint64_t size
, zio_cksum_t
*zcp
)
472 CpaInstanceHandle cy_inst_handle
;
473 Cpa16U nr_bufs
= (size
>> PAGE_SHIFT
) + 2;
474 Cpa32U bytes_left
= 0;
476 CpaCySymSessionCtx
*cy_session_ctx
= NULL
;
478 Cpa8U
*digest_buffer
= NULL
;
479 CpaCySymOpData op_data
= { 0 };
480 CpaBufferList src_buffer_list
= { 0 };
481 CpaFlatBuffer
*flat_src_buf_array
= NULL
;
482 CpaFlatBuffer
*flat_src_buf
= NULL
;
483 struct page
*in_pages
[MAX_PAGE_NUM
];
487 QAT_STAT_BUMP(cksum_requests
);
488 QAT_STAT_INCR(cksum_total_in_bytes
, size
);
490 i
= (Cpa32U
)atomic_inc_32_nv(&inst_num
) % num_inst
;
491 cy_inst_handle
= cy_inst_handles
[i
];
493 status
= qat_init_checksum_session_ctx(cy_inst_handle
,
494 &cy_session_ctx
, cksum
);
495 if (status
!= CPA_STATUS_SUCCESS
) {
496 /* don't count unsupported checksums as a failure */
497 if (cksum
== ZIO_CHECKSUM_SHA256
||
498 cksum
== ZIO_CHECKSUM_SHA512
)
499 QAT_STAT_BUMP(cksum_fails
);
504 * We increment nr_bufs by 2 to allow us to handle non
505 * page-aligned buffer addresses and buffers whose sizes
506 * are not divisible by PAGE_SIZE.
508 status
= qat_init_cy_buffer_lists(cy_inst_handle
, nr_bufs
,
509 &src_buffer_list
, &src_buffer_list
);
510 if (status
!= CPA_STATUS_SUCCESS
)
513 status
= QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array
,
514 nr_bufs
* sizeof (CpaFlatBuffer
));
515 if (status
!= CPA_STATUS_SUCCESS
)
517 status
= QAT_PHYS_CONTIG_ALLOC(&digest_buffer
,
518 sizeof (zio_cksum_t
));
519 if (status
!= CPA_STATUS_SUCCESS
)
524 flat_src_buf
= flat_src_buf_array
;
525 while (bytes_left
> 0) {
526 page_off
= ((long)data
& ~PAGE_MASK
);
527 in_pages
[page_num
] = qat_mem_to_page(data
);
528 flat_src_buf
->pData
= kmap(in_pages
[page_num
]) + page_off
;
529 flat_src_buf
->dataLenInBytes
=
530 min((long)PAGE_SIZE
- page_off
, (long)bytes_left
);
531 data
+= flat_src_buf
->dataLenInBytes
;
532 bytes_left
-= flat_src_buf
->dataLenInBytes
;
536 src_buffer_list
.pBuffers
= flat_src_buf_array
;
537 src_buffer_list
.numBuffers
= page_num
;
539 op_data
.sessionCtx
= cy_session_ctx
;
540 op_data
.packetType
= CPA_CY_SYM_PACKET_TYPE_FULL
;
541 op_data
.hashStartSrcOffsetInBytes
= 0;
542 op_data
.messageLenToHashInBytes
= size
;
543 op_data
.pDigestResult
= digest_buffer
;
545 cb
.verify_result
= CPA_FALSE
;
546 init_completion(&cb
.complete
);
547 status
= cpaCySymPerformOp(cy_inst_handle
, &cb
, &op_data
,
548 &src_buffer_list
, &src_buffer_list
, NULL
);
549 if (status
!= CPA_STATUS_SUCCESS
)
552 if (!wait_for_completion_interruptible_timeout(&cb
.complete
,
554 status
= CPA_STATUS_FAIL
;
557 if (cb
.verify_result
== CPA_FALSE
) {
558 status
= CPA_STATUS_FAIL
;
562 bcopy(digest_buffer
, zcp
, sizeof (zio_cksum_t
));
565 if (status
!= CPA_STATUS_SUCCESS
)
566 QAT_STAT_BUMP(cksum_fails
);
568 for (i
= 0; i
< page_num
; i
++)
571 cpaCySymRemoveSession(cy_inst_handle
, cy_session_ctx
);
572 QAT_PHYS_CONTIG_FREE(digest_buffer
);
573 QAT_PHYS_CONTIG_FREE(src_buffer_list
.pPrivateMetaData
);
574 QAT_PHYS_CONTIG_FREE(cy_session_ctx
);
575 QAT_PHYS_CONTIG_FREE(flat_src_buf_array
);
581 param_set_qat_encrypt(const char *val
, struct kernel_param
*kp
)
584 int *pvalue
= kp
->arg
;
585 ret
= param_set_int(val
, kp
);
589 * zfs_qat_encrypt_disable = 0: enable qat encrypt
590 * try to initialize qat instance if it has not been done
592 if (*pvalue
== 0 && !qat_cy_init_done
) {
595 zfs_qat_encrypt_disable
= 1;
603 param_set_qat_checksum(const char *val
, struct kernel_param
*kp
)
606 int *pvalue
= kp
->arg
;
607 ret
= param_set_int(val
, kp
);
611 * set_checksum_param_ops = 0: enable qat checksum
612 * try to initialize qat instance if it has not been done
614 if (*pvalue
== 0 && !qat_cy_init_done
) {
617 zfs_qat_checksum_disable
= 1;
624 module_param_call(zfs_qat_encrypt_disable
, param_set_qat_encrypt
,
625 param_get_int
, &zfs_qat_encrypt_disable
, 0644);
626 MODULE_PARM_DESC(zfs_qat_encrypt_disable
, "Enable/Disable QAT encryption");
628 module_param_call(zfs_qat_checksum_disable
, param_set_qat_checksum
,
629 param_get_int
, &zfs_qat_checksum_disable
, 0644);
630 MODULE_PARM_DESC(zfs_qat_checksum_disable
, "Enable/Disable QAT checksumming");