4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * This file represents the QAT implementation of checksums and encryption.
24 * Internally, QAT shares the same cryptographic instances for both of these
25 * operations, so the code has been combined here. QAT data compression uses
26 * compression instances, so that code is separated into qat_compress.c
29 #if defined(_KERNEL) && defined(HAVE_QAT)
30 #include <linux/slab.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/completion.h>
34 #include <sys/zfs_context.h>
35 #include <sys/zio_crypt.h>
36 #include "lac/cpa_cy_im.h"
37 #include "lac/cpa_cy_common.h"
41 * Max instances in a QAT device, each instance is a channel to submit
42 * jobs to QAT hardware, this is only for pre-allocating instances
43 * and session arrays; the actual number of instances are defined in
44 * the QAT driver's configure file.
46 #define QAT_CRYPT_MAX_INSTANCES 48
48 #define MAX_PAGE_NUM 1024
50 static Cpa32U inst_num
= 0;
51 static Cpa16U num_inst
= 0;
52 static CpaInstanceHandle cy_inst_handles
[QAT_CRYPT_MAX_INSTANCES
];
53 static boolean_t qat_crypt_init_done
= B_FALSE
;
54 int zfs_qat_encrypt_disable
= 0;
55 int zfs_qat_checksum_disable
= 0;
57 typedef struct cy_callback
{
58 CpaBoolean verify_result
;
59 struct completion complete
;
63 symcallback(void *p_callback
, CpaStatus status
, const CpaCySymOp operation
,
64 void *op_data
, CpaBufferList
*buf_list_dst
, CpaBoolean verify
)
66 cy_callback_t
*cb
= p_callback
;
69 /* indicate that the function has been called */
70 cb
->verify_result
= verify
;
71 complete(&cb
->complete
);
76 qat_crypt_use_accel(size_t s_len
)
78 return (!zfs_qat_encrypt_disable
&&
79 qat_crypt_init_done
&&
80 s_len
>= QAT_MIN_BUF_SIZE
&&
81 s_len
<= QAT_MAX_BUF_SIZE
);
85 qat_checksum_use_accel(size_t s_len
)
87 return (!zfs_qat_checksum_disable
&&
88 qat_crypt_init_done
&&
89 s_len
>= QAT_MIN_BUF_SIZE
&&
90 s_len
<= QAT_MAX_BUF_SIZE
);
96 for (Cpa16U i
= 0; i
< num_inst
; i
++)
97 cpaCyStopInstance(cy_inst_handles
[i
]);
100 qat_crypt_init_done
= B_FALSE
;
106 CpaStatus status
= CPA_STATUS_FAIL
;
108 status
= cpaCyGetNumInstances(&num_inst
);
109 if (status
!= CPA_STATUS_SUCCESS
)
112 /* if the user has configured no QAT encryption units just return */
116 if (num_inst
> QAT_CRYPT_MAX_INSTANCES
)
117 num_inst
= QAT_CRYPT_MAX_INSTANCES
;
119 status
= cpaCyGetInstances(num_inst
, &cy_inst_handles
[0]);
120 if (status
!= CPA_STATUS_SUCCESS
)
123 for (Cpa16U i
= 0; i
< num_inst
; i
++) {
124 status
= cpaCySetAddressTranslation(cy_inst_handles
[i
],
125 (void *)virt_to_phys
);
126 if (status
!= CPA_STATUS_SUCCESS
)
129 status
= cpaCyStartInstance(cy_inst_handles
[i
]);
130 if (status
!= CPA_STATUS_SUCCESS
)
134 qat_crypt_init_done
= B_TRUE
;
145 if (!qat_crypt_init_done
)
152 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir
, CpaInstanceHandle inst_handle
,
153 CpaCySymSessionCtx
**cy_session_ctx
, crypto_key_t
*key
,
154 Cpa64U crypt
, Cpa32U aad_len
)
156 CpaStatus status
= CPA_STATUS_SUCCESS
;
158 Cpa32U ciper_algorithm
;
159 Cpa32U hash_algorithm
;
160 CpaCySymSessionSetupData sd
= { 0 };
162 if (zio_crypt_table
[crypt
].ci_crypt_type
== ZC_TYPE_CCM
) {
163 return (CPA_STATUS_FAIL
);
165 ciper_algorithm
= CPA_CY_SYM_CIPHER_AES_GCM
;
166 hash_algorithm
= CPA_CY_SYM_HASH_AES_GCM
;
169 sd
.cipherSetupData
.cipherAlgorithm
= ciper_algorithm
;
170 sd
.cipherSetupData
.pCipherKey
= key
->ck_data
;
171 sd
.cipherSetupData
.cipherKeyLenInBytes
= key
->ck_length
/ 8;
172 sd
.hashSetupData
.hashAlgorithm
= hash_algorithm
;
173 sd
.hashSetupData
.hashMode
= CPA_CY_SYM_HASH_MODE_AUTH
;
174 sd
.hashSetupData
.digestResultLenInBytes
= ZIO_DATA_MAC_LEN
;
175 sd
.hashSetupData
.authModeSetupData
.aadLenInBytes
= aad_len
;
176 sd
.sessionPriority
= CPA_CY_PRIORITY_NORMAL
;
177 sd
.symOperation
= CPA_CY_SYM_OP_ALGORITHM_CHAINING
;
178 sd
.digestIsAppended
= CPA_FALSE
;
179 sd
.verifyDigest
= CPA_FALSE
;
181 if (dir
== QAT_ENCRYPT
) {
182 sd
.cipherSetupData
.cipherDirection
=
183 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT
;
185 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER
;
187 ASSERT3U(dir
, ==, QAT_DECRYPT
);
188 sd
.cipherSetupData
.cipherDirection
=
189 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT
;
191 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH
;
194 status
= cpaCySymSessionCtxGetSize(inst_handle
, &sd
, &ctx_size
);
195 if (status
!= CPA_STATUS_SUCCESS
)
198 status
= QAT_PHYS_CONTIG_ALLOC(cy_session_ctx
, ctx_size
);
199 if (status
!= CPA_STATUS_SUCCESS
)
202 status
= cpaCySymInitSession(inst_handle
, symcallback
, &sd
,
204 if (status
!= CPA_STATUS_SUCCESS
) {
205 QAT_PHYS_CONTIG_FREE(*cy_session_ctx
);
209 return (CPA_STATUS_SUCCESS
);
213 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle
,
214 CpaCySymSessionCtx
**cy_session_ctx
, Cpa64U cksum
)
216 CpaStatus status
= CPA_STATUS_SUCCESS
;
218 Cpa32U hash_algorithm
;
219 CpaCySymSessionSetupData sd
= { 0 };
222 * ZFS's SHA512 checksum is actually SHA512/256, which uses
223 * a different IV from standard SHA512. QAT does not support
224 * SHA512/256, so we can only support SHA256.
226 if (cksum
== ZIO_CHECKSUM_SHA256
)
227 hash_algorithm
= CPA_CY_SYM_HASH_SHA256
;
229 return (CPA_STATUS_FAIL
);
231 sd
.sessionPriority
= CPA_CY_PRIORITY_NORMAL
;
232 sd
.symOperation
= CPA_CY_SYM_OP_HASH
;
233 sd
.hashSetupData
.hashAlgorithm
= hash_algorithm
;
234 sd
.hashSetupData
.hashMode
= CPA_CY_SYM_HASH_MODE_PLAIN
;
235 sd
.hashSetupData
.digestResultLenInBytes
= sizeof (zio_cksum_t
);
236 sd
.digestIsAppended
= CPA_FALSE
;
237 sd
.verifyDigest
= CPA_FALSE
;
239 status
= cpaCySymSessionCtxGetSize(inst_handle
, &sd
, &ctx_size
);
240 if (status
!= CPA_STATUS_SUCCESS
)
243 status
= QAT_PHYS_CONTIG_ALLOC(cy_session_ctx
, ctx_size
);
244 if (status
!= CPA_STATUS_SUCCESS
)
247 status
= cpaCySymInitSession(inst_handle
, symcallback
, &sd
,
249 if (status
!= CPA_STATUS_SUCCESS
) {
250 QAT_PHYS_CONTIG_FREE(*cy_session_ctx
);
254 return (CPA_STATUS_SUCCESS
);
258 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle
, uint32_t nr_bufs
,
259 CpaBufferList
*src
, CpaBufferList
*dst
)
261 CpaStatus status
= CPA_STATUS_SUCCESS
;
262 Cpa32U meta_size
= 0;
264 status
= cpaCyBufferListGetMetaSize(inst_handle
, nr_bufs
, &meta_size
);
265 if (status
!= CPA_STATUS_SUCCESS
)
268 status
= QAT_PHYS_CONTIG_ALLOC(&src
->pPrivateMetaData
, meta_size
);
269 if (status
!= CPA_STATUS_SUCCESS
)
273 status
= QAT_PHYS_CONTIG_ALLOC(&dst
->pPrivateMetaData
,
275 if (status
!= CPA_STATUS_SUCCESS
)
279 return (CPA_STATUS_SUCCESS
);
282 QAT_PHYS_CONTIG_FREE(src
->pPrivateMetaData
);
284 QAT_PHYS_CONTIG_FREE(dst
->pPrivateMetaData
);
290 qat_crypt(qat_encrypt_dir_t dir
, uint8_t *src_buf
, uint8_t *dst_buf
,
291 uint8_t *aad_buf
, uint32_t aad_len
, uint8_t *iv_buf
, uint8_t *digest_buf
,
292 crypto_key_t
*key
, uint64_t crypt
, uint32_t enc_len
)
294 CpaStatus status
= CPA_STATUS_SUCCESS
;
296 CpaInstanceHandle cy_inst_handle
;
297 Cpa16U nr_bufs
= (enc_len
>> PAGE_SHIFT
) + 2;
298 Cpa32U bytes_left
= 0;
300 CpaCySymSessionCtx
*cy_session_ctx
= NULL
;
302 CpaCySymOpData op_data
= { 0 };
303 CpaBufferList src_buffer_list
= { 0 };
304 CpaBufferList dst_buffer_list
= { 0 };
305 CpaFlatBuffer
*flat_src_buf_array
= NULL
;
306 CpaFlatBuffer
*flat_src_buf
= NULL
;
307 CpaFlatBuffer
*flat_dst_buf_array
= NULL
;
308 CpaFlatBuffer
*flat_dst_buf
= NULL
;
309 struct page
*in_pages
[MAX_PAGE_NUM
];
310 struct page
*out_pages
[MAX_PAGE_NUM
];
311 Cpa32U in_page_num
= 0;
312 Cpa32U out_page_num
= 0;
313 Cpa32U in_page_off
= 0;
314 Cpa32U out_page_off
= 0;
316 if (dir
== QAT_ENCRYPT
) {
317 QAT_STAT_BUMP(encrypt_requests
);
318 QAT_STAT_INCR(encrypt_total_in_bytes
, enc_len
);
320 QAT_STAT_BUMP(decrypt_requests
);
321 QAT_STAT_INCR(decrypt_total_in_bytes
, enc_len
);
324 i
= (Cpa32U
)atomic_inc_32_nv(&inst_num
) % num_inst
;
325 cy_inst_handle
= cy_inst_handles
[i
];
327 status
= qat_init_crypt_session_ctx(dir
, cy_inst_handle
,
328 &cy_session_ctx
, key
, crypt
, aad_len
);
329 if (status
!= CPA_STATUS_SUCCESS
) {
330 /* don't count CCM as a failure since it's not supported */
331 if (zio_crypt_table
[crypt
].ci_crypt_type
== ZC_TYPE_GCM
)
332 QAT_STAT_BUMP(crypt_fails
);
337 * We increment nr_bufs by 2 to allow us to handle non
338 * page-aligned buffer addresses and buffers whose sizes
339 * are not divisible by PAGE_SIZE.
341 status
= qat_init_cy_buffer_lists(cy_inst_handle
, nr_bufs
,
342 &src_buffer_list
, &dst_buffer_list
);
343 if (status
!= CPA_STATUS_SUCCESS
)
346 status
= QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array
,
347 nr_bufs
* sizeof (CpaFlatBuffer
));
348 if (status
!= CPA_STATUS_SUCCESS
)
350 status
= QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array
,
351 nr_bufs
* sizeof (CpaFlatBuffer
));
352 if (status
!= CPA_STATUS_SUCCESS
)
355 bytes_left
= enc_len
;
357 flat_src_buf
= flat_src_buf_array
;
358 while (bytes_left
> 0) {
359 in_page_off
= ((long)data
& ~PAGE_MASK
);
360 in_pages
[in_page_num
] = qat_mem_to_page(data
);
361 flat_src_buf
->pData
= kmap(in_pages
[in_page_num
]) + in_page_off
;
362 flat_src_buf
->dataLenInBytes
=
363 min((long)PAGE_SIZE
- in_page_off
, (long)bytes_left
);
364 data
+= flat_src_buf
->dataLenInBytes
;
365 bytes_left
-= flat_src_buf
->dataLenInBytes
;
369 src_buffer_list
.pBuffers
= flat_src_buf_array
;
370 src_buffer_list
.numBuffers
= in_page_num
;
372 bytes_left
= enc_len
;
374 flat_dst_buf
= flat_dst_buf_array
;
375 while (bytes_left
> 0) {
376 out_page_off
= ((long)data
& ~PAGE_MASK
);
377 out_pages
[out_page_num
] = qat_mem_to_page(data
);
378 flat_dst_buf
->pData
= kmap(out_pages
[out_page_num
]) +
380 flat_dst_buf
->dataLenInBytes
=
381 min((long)PAGE_SIZE
- out_page_off
, (long)bytes_left
);
382 data
+= flat_dst_buf
->dataLenInBytes
;
383 bytes_left
-= flat_dst_buf
->dataLenInBytes
;
387 dst_buffer_list
.pBuffers
= flat_dst_buf_array
;
388 dst_buffer_list
.numBuffers
= out_page_num
;
390 op_data
.sessionCtx
= cy_session_ctx
;
391 op_data
.packetType
= CPA_CY_SYM_PACKET_TYPE_FULL
;
392 op_data
.pIv
= NULL
; /* set this later as the J0 block */
393 op_data
.ivLenInBytes
= 0;
394 op_data
.cryptoStartSrcOffsetInBytes
= 0;
395 op_data
.messageLenToCipherInBytes
= 0;
396 op_data
.hashStartSrcOffsetInBytes
= 0;
397 op_data
.messageLenToHashInBytes
= 0;
398 op_data
.pDigestResult
= 0;
399 op_data
.messageLenToCipherInBytes
= enc_len
;
400 op_data
.ivLenInBytes
= ZIO_DATA_IV_LEN
;
401 op_data
.pDigestResult
= digest_buf
;
402 op_data
.pAdditionalAuthData
= aad_buf
;
403 op_data
.pIv
= iv_buf
;
405 cb
.verify_result
= CPA_FALSE
;
406 init_completion(&cb
.complete
);
407 status
= cpaCySymPerformOp(cy_inst_handle
, &cb
, &op_data
,
408 &src_buffer_list
, &dst_buffer_list
, NULL
);
409 if (status
!= CPA_STATUS_SUCCESS
)
412 if (!wait_for_completion_interruptible_timeout(&cb
.complete
,
414 status
= CPA_STATUS_FAIL
;
418 if (cb
.verify_result
== CPA_FALSE
) {
419 status
= CPA_STATUS_FAIL
;
423 if (dir
== QAT_ENCRYPT
)
424 QAT_STAT_INCR(encrypt_total_out_bytes
, enc_len
);
426 QAT_STAT_INCR(decrypt_total_out_bytes
, enc_len
);
429 if (status
!= CPA_STATUS_SUCCESS
)
430 QAT_STAT_BUMP(crypt_fails
);
432 for (i
= 0; i
< in_page_num
; i
++)
434 for (i
= 0; i
< out_page_num
; i
++)
435 kunmap(out_pages
[i
]);
437 cpaCySymRemoveSession(cy_inst_handle
, cy_session_ctx
);
438 QAT_PHYS_CONTIG_FREE(src_buffer_list
.pPrivateMetaData
);
439 QAT_PHYS_CONTIG_FREE(dst_buffer_list
.pPrivateMetaData
);
440 QAT_PHYS_CONTIG_FREE(cy_session_ctx
);
441 QAT_PHYS_CONTIG_FREE(flat_src_buf_array
);
442 QAT_PHYS_CONTIG_FREE(flat_dst_buf_array
);
448 qat_checksum(uint64_t cksum
, uint8_t *buf
, uint64_t size
, zio_cksum_t
*zcp
)
452 CpaInstanceHandle cy_inst_handle
;
453 Cpa16U nr_bufs
= (size
>> PAGE_SHIFT
) + 2;
454 Cpa32U bytes_left
= 0;
456 CpaCySymSessionCtx
*cy_session_ctx
= NULL
;
458 Cpa8U digest_buffer
[sizeof (zio_cksum_t
)];
459 CpaCySymOpData op_data
= { 0 };
460 CpaBufferList src_buffer_list
= { 0 };
461 CpaFlatBuffer
*flat_src_buf_array
= NULL
;
462 CpaFlatBuffer
*flat_src_buf
= NULL
;
463 struct page
*in_pages
[MAX_PAGE_NUM
];
467 QAT_STAT_BUMP(cksum_requests
);
468 QAT_STAT_INCR(cksum_total_in_bytes
, size
);
470 i
= (Cpa32U
)atomic_inc_32_nv(&inst_num
) % num_inst
;
471 cy_inst_handle
= cy_inst_handles
[i
];
473 status
= qat_init_checksum_session_ctx(cy_inst_handle
,
474 &cy_session_ctx
, cksum
);
475 if (status
!= CPA_STATUS_SUCCESS
) {
476 /* don't count unsupported checksums as a failure */
477 if (cksum
== ZIO_CHECKSUM_SHA256
||
478 cksum
== ZIO_CHECKSUM_SHA512
)
479 QAT_STAT_BUMP(cksum_fails
);
484 * We increment nr_bufs by 2 to allow us to handle non
485 * page-aligned buffer addresses and buffers whose sizes
486 * are not divisible by PAGE_SIZE.
488 status
= qat_init_cy_buffer_lists(cy_inst_handle
, nr_bufs
,
489 &src_buffer_list
, &src_buffer_list
);
490 if (status
!= CPA_STATUS_SUCCESS
)
493 status
= QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array
,
494 nr_bufs
* sizeof (CpaFlatBuffer
));
495 if (status
!= CPA_STATUS_SUCCESS
)
500 flat_src_buf
= flat_src_buf_array
;
501 while (bytes_left
> 0) {
502 page_off
= ((long)data
& ~PAGE_MASK
);
503 in_pages
[page_num
] = qat_mem_to_page(data
);
504 flat_src_buf
->pData
= kmap(in_pages
[page_num
]) + page_off
;
505 flat_src_buf
->dataLenInBytes
=
506 min((long)PAGE_SIZE
- page_off
, (long)bytes_left
);
507 data
+= flat_src_buf
->dataLenInBytes
;
508 bytes_left
-= flat_src_buf
->dataLenInBytes
;
512 src_buffer_list
.pBuffers
= flat_src_buf_array
;
513 src_buffer_list
.numBuffers
= page_num
;
515 op_data
.sessionCtx
= cy_session_ctx
;
516 op_data
.packetType
= CPA_CY_SYM_PACKET_TYPE_FULL
;
517 op_data
.hashStartSrcOffsetInBytes
= 0;
518 op_data
.messageLenToHashInBytes
= size
;
519 op_data
.pDigestResult
= digest_buffer
;
521 cb
.verify_result
= CPA_FALSE
;
522 init_completion(&cb
.complete
);
523 status
= cpaCySymPerformOp(cy_inst_handle
, &cb
, &op_data
,
524 &src_buffer_list
, &src_buffer_list
, NULL
);
525 if (status
!= CPA_STATUS_SUCCESS
)
528 if (!wait_for_completion_interruptible_timeout(&cb
.complete
,
530 status
= CPA_STATUS_FAIL
;
534 bcopy(digest_buffer
, zcp
, sizeof (zio_cksum_t
));
537 if (status
!= CPA_STATUS_SUCCESS
)
538 QAT_STAT_BUMP(cksum_fails
);
540 for (i
= 0; i
< page_num
; i
++)
543 cpaCySymRemoveSession(cy_inst_handle
, cy_session_ctx
);
544 QAT_PHYS_CONTIG_FREE(src_buffer_list
.pPrivateMetaData
);
545 QAT_PHYS_CONTIG_FREE(cy_session_ctx
);
546 QAT_PHYS_CONTIG_FREE(flat_src_buf_array
);
551 module_param(zfs_qat_encrypt_disable
, int, 0644);
552 MODULE_PARM_DESC(zfs_qat_encrypt_disable
, "Disable QAT encryption");
554 module_param(zfs_qat_checksum_disable
, int, 0644);
555 MODULE_PARM_DESC(zfs_qat_checksum_disable
, "Disable QAT checksumming");