4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * This file represents the QAT implementation of checksums and encryption.
24 * Internally, QAT shares the same cryptographic instances for both of these
25 * operations, so the code has been combined here. QAT data compression uses
26 * compression instances, so that code is separated into qat_compress.c
29 #if defined(_KERNEL) && defined(HAVE_QAT)
30 #include <linux/slab.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/completion.h>
34 #include <sys/zfs_context.h>
35 #include <sys/zio_crypt.h>
36 #include "lac/cpa_cy_im.h"
37 #include "lac/cpa_cy_common.h"
41 * Max instances in a QAT device, each instance is a channel to submit
42 * jobs to QAT hardware, this is only for pre-allocating instances
43 * and session arrays; the actual number of instances are defined in
44 * the QAT driver's configure file.
46 #define QAT_CRYPT_MAX_INSTANCES 48
48 #define MAX_PAGE_NUM 1024
50 static Cpa16U inst_num
= 0;
51 static Cpa16U num_inst
= 0;
52 static CpaInstanceHandle cy_inst_handles
[QAT_CRYPT_MAX_INSTANCES
];
53 static boolean_t qat_crypt_init_done
= B_FALSE
;
54 int zfs_qat_encrypt_disable
= 0;
55 int zfs_qat_checksum_disable
= 0;
57 typedef struct cy_callback
{
58 CpaBoolean verify_result
;
59 struct completion complete
;
63 symcallback(void *p_callback
, CpaStatus status
, const CpaCySymOp operation
,
64 void *op_data
, CpaBufferList
*buf_list_dst
, CpaBoolean verify
)
66 cy_callback_t
*cb
= p_callback
;
69 /* indicate that the function has been called */
70 cb
->verify_result
= verify
;
71 complete(&cb
->complete
);
76 qat_crypt_use_accel(size_t s_len
)
78 return (!zfs_qat_encrypt_disable
&&
79 qat_crypt_init_done
&&
80 s_len
>= QAT_MIN_BUF_SIZE
&&
81 s_len
<= QAT_MAX_BUF_SIZE
);
85 qat_checksum_use_accel(size_t s_len
)
87 return (!zfs_qat_checksum_disable
&&
88 qat_crypt_init_done
&&
89 s_len
>= QAT_MIN_BUF_SIZE
&&
90 s_len
<= QAT_MAX_BUF_SIZE
);
96 for (Cpa32U i
= 0; i
< num_inst
; i
++)
97 cpaCyStopInstance(cy_inst_handles
[i
]);
100 qat_crypt_init_done
= B_FALSE
;
107 CpaStatus status
= CPA_STATUS_FAIL
;
109 status
= cpaCyGetNumInstances(&num_inst
);
110 if (status
!= CPA_STATUS_SUCCESS
)
113 /* if the user has configured no QAT encryption units just return */
117 if (num_inst
> QAT_CRYPT_MAX_INSTANCES
)
118 num_inst
= QAT_CRYPT_MAX_INSTANCES
;
120 status
= cpaCyGetInstances(num_inst
, &cy_inst_handles
[0]);
121 if (status
!= CPA_STATUS_SUCCESS
)
124 for (i
= 0; i
< num_inst
; i
++) {
125 status
= cpaCySetAddressTranslation(cy_inst_handles
[i
],
126 (void *)virt_to_phys
);
127 if (status
!= CPA_STATUS_SUCCESS
)
130 status
= cpaCyStartInstance(cy_inst_handles
[i
]);
131 if (status
!= CPA_STATUS_SUCCESS
)
135 qat_crypt_init_done
= B_TRUE
;
146 if (!qat_crypt_init_done
)
153 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir
, CpaInstanceHandle inst_handle
,
154 CpaCySymSessionCtx
**cy_session_ctx
, crypto_key_t
*key
,
155 Cpa64U crypt
, Cpa32U aad_len
)
157 CpaStatus status
= CPA_STATUS_SUCCESS
;
159 Cpa32U ciper_algorithm
;
160 Cpa32U hash_algorithm
;
161 CpaCySymSessionSetupData sd
= { 0 };
163 if (zio_crypt_table
[crypt
].ci_crypt_type
== ZC_TYPE_CCM
) {
164 return (CPA_STATUS_FAIL
);
166 ciper_algorithm
= CPA_CY_SYM_CIPHER_AES_GCM
;
167 hash_algorithm
= CPA_CY_SYM_HASH_AES_GCM
;
170 sd
.cipherSetupData
.cipherAlgorithm
= ciper_algorithm
;
171 sd
.cipherSetupData
.pCipherKey
= key
->ck_data
;
172 sd
.cipherSetupData
.cipherKeyLenInBytes
= key
->ck_length
/ 8;
173 sd
.hashSetupData
.hashAlgorithm
= hash_algorithm
;
174 sd
.hashSetupData
.hashMode
= CPA_CY_SYM_HASH_MODE_AUTH
;
175 sd
.hashSetupData
.digestResultLenInBytes
= ZIO_DATA_MAC_LEN
;
176 sd
.hashSetupData
.authModeSetupData
.aadLenInBytes
= aad_len
;
177 sd
.sessionPriority
= CPA_CY_PRIORITY_NORMAL
;
178 sd
.symOperation
= CPA_CY_SYM_OP_ALGORITHM_CHAINING
;
179 sd
.digestIsAppended
= CPA_FALSE
;
180 sd
.verifyDigest
= CPA_FALSE
;
182 if (dir
== QAT_ENCRYPT
) {
183 sd
.cipherSetupData
.cipherDirection
=
184 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT
;
186 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER
;
188 ASSERT3U(dir
, ==, QAT_DECRYPT
);
189 sd
.cipherSetupData
.cipherDirection
=
190 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT
;
192 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH
;
195 status
= cpaCySymSessionCtxGetSize(inst_handle
, &sd
, &ctx_size
);
196 if (status
!= CPA_STATUS_SUCCESS
)
199 status
= QAT_PHYS_CONTIG_ALLOC(cy_session_ctx
, ctx_size
);
200 if (status
!= CPA_STATUS_SUCCESS
)
203 status
= cpaCySymInitSession(inst_handle
, symcallback
, &sd
,
205 if (status
!= CPA_STATUS_SUCCESS
) {
206 QAT_PHYS_CONTIG_FREE(*cy_session_ctx
);
210 return (CPA_STATUS_SUCCESS
);
214 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle
,
215 CpaCySymSessionCtx
**cy_session_ctx
, Cpa64U cksum
)
217 CpaStatus status
= CPA_STATUS_SUCCESS
;
219 Cpa32U hash_algorithm
;
220 CpaCySymSessionSetupData sd
= { 0 };
223 * ZFS's SHA512 checksum is actually SHA512/256, which uses
224 * a different IV from standard SHA512. QAT does not support
225 * SHA512/256, so we can only support SHA256.
227 if (cksum
== ZIO_CHECKSUM_SHA256
)
228 hash_algorithm
= CPA_CY_SYM_HASH_SHA256
;
230 return (CPA_STATUS_FAIL
);
232 sd
.sessionPriority
= CPA_CY_PRIORITY_NORMAL
;
233 sd
.symOperation
= CPA_CY_SYM_OP_HASH
;
234 sd
.hashSetupData
.hashAlgorithm
= hash_algorithm
;
235 sd
.hashSetupData
.hashMode
= CPA_CY_SYM_HASH_MODE_PLAIN
;
236 sd
.hashSetupData
.digestResultLenInBytes
= sizeof (zio_cksum_t
);
237 sd
.digestIsAppended
= CPA_FALSE
;
238 sd
.verifyDigest
= CPA_FALSE
;
240 status
= cpaCySymSessionCtxGetSize(inst_handle
, &sd
, &ctx_size
);
241 if (status
!= CPA_STATUS_SUCCESS
)
244 status
= QAT_PHYS_CONTIG_ALLOC(cy_session_ctx
, ctx_size
);
245 if (status
!= CPA_STATUS_SUCCESS
)
248 status
= cpaCySymInitSession(inst_handle
, symcallback
, &sd
,
250 if (status
!= CPA_STATUS_SUCCESS
) {
251 QAT_PHYS_CONTIG_FREE(*cy_session_ctx
);
255 return (CPA_STATUS_SUCCESS
);
259 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle
, uint32_t nr_bufs
,
260 CpaBufferList
*src
, CpaBufferList
*dst
)
262 CpaStatus status
= CPA_STATUS_SUCCESS
;
263 Cpa32U meta_size
= 0;
265 status
= cpaCyBufferListGetMetaSize(inst_handle
, nr_bufs
, &meta_size
);
266 if (status
!= CPA_STATUS_SUCCESS
)
269 status
= QAT_PHYS_CONTIG_ALLOC(&src
->pPrivateMetaData
, meta_size
);
270 if (status
!= CPA_STATUS_SUCCESS
)
274 status
= QAT_PHYS_CONTIG_ALLOC(&dst
->pPrivateMetaData
,
276 if (status
!= CPA_STATUS_SUCCESS
)
280 return (CPA_STATUS_SUCCESS
);
283 QAT_PHYS_CONTIG_FREE(src
->pPrivateMetaData
);
285 QAT_PHYS_CONTIG_FREE(dst
->pPrivateMetaData
);
291 qat_crypt(qat_encrypt_dir_t dir
, uint8_t *src_buf
, uint8_t *dst_buf
,
292 uint8_t *aad_buf
, uint32_t aad_len
, uint8_t *iv_buf
, uint8_t *digest_buf
,
293 crypto_key_t
*key
, uint64_t crypt
, uint32_t enc_len
)
295 CpaStatus status
= CPA_STATUS_SUCCESS
;
297 CpaInstanceHandle cy_inst_handle
;
298 Cpa16U nr_bufs
= (enc_len
>> PAGE_SHIFT
) + 2;
299 Cpa32U bytes_left
= 0;
301 CpaCySymSessionCtx
*cy_session_ctx
= NULL
;
303 CpaCySymOpData op_data
= { 0 };
304 CpaBufferList src_buffer_list
= { 0 };
305 CpaBufferList dst_buffer_list
= { 0 };
306 CpaFlatBuffer
*flat_src_buf_array
= NULL
;
307 CpaFlatBuffer
*flat_src_buf
= NULL
;
308 CpaFlatBuffer
*flat_dst_buf_array
= NULL
;
309 CpaFlatBuffer
*flat_dst_buf
= NULL
;
310 struct page
*in_pages
[MAX_PAGE_NUM
];
311 struct page
*out_pages
[MAX_PAGE_NUM
];
312 Cpa32U in_page_num
= 0;
313 Cpa32U out_page_num
= 0;
314 Cpa32U in_page_off
= 0;
315 Cpa32U out_page_off
= 0;
317 if (dir
== QAT_ENCRYPT
) {
318 QAT_STAT_BUMP(encrypt_requests
);
319 QAT_STAT_INCR(encrypt_total_in_bytes
, enc_len
);
321 QAT_STAT_BUMP(decrypt_requests
);
322 QAT_STAT_INCR(decrypt_total_in_bytes
, enc_len
);
325 i
= atomic_inc_32_nv(&inst_num
) % num_inst
;
326 cy_inst_handle
= cy_inst_handles
[i
];
328 status
= qat_init_crypt_session_ctx(dir
, cy_inst_handle
,
329 &cy_session_ctx
, key
, crypt
, aad_len
);
330 if (status
!= CPA_STATUS_SUCCESS
) {
331 /* don't count CCM as a failure since it's not supported */
332 if (zio_crypt_table
[crypt
].ci_crypt_type
== ZC_TYPE_GCM
)
333 QAT_STAT_BUMP(crypt_fails
);
338 * We increment nr_bufs by 2 to allow us to handle non
339 * page-aligned buffer addresses and buffers whose sizes
340 * are not divisible by PAGE_SIZE.
342 status
= qat_init_cy_buffer_lists(cy_inst_handle
, nr_bufs
,
343 &src_buffer_list
, &dst_buffer_list
);
344 if (status
!= CPA_STATUS_SUCCESS
)
347 status
= QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array
,
348 nr_bufs
* sizeof (CpaFlatBuffer
));
349 if (status
!= CPA_STATUS_SUCCESS
)
351 status
= QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array
,
352 nr_bufs
* sizeof (CpaFlatBuffer
));
353 if (status
!= CPA_STATUS_SUCCESS
)
356 bytes_left
= enc_len
;
358 flat_src_buf
= flat_src_buf_array
;
359 while (bytes_left
> 0) {
360 in_page_off
= ((long)data
& ~PAGE_MASK
);
361 in_pages
[in_page_num
] = qat_mem_to_page(data
);
362 flat_src_buf
->pData
= kmap(in_pages
[in_page_num
]) + in_page_off
;
363 flat_src_buf
->dataLenInBytes
=
364 min((long)PAGE_SIZE
- in_page_off
, (long)bytes_left
);
365 data
+= flat_src_buf
->dataLenInBytes
;
366 bytes_left
-= flat_src_buf
->dataLenInBytes
;
370 src_buffer_list
.pBuffers
= flat_src_buf_array
;
371 src_buffer_list
.numBuffers
= in_page_num
;
373 bytes_left
= enc_len
;
375 flat_dst_buf
= flat_dst_buf_array
;
376 while (bytes_left
> 0) {
377 out_page_off
= ((long)data
& ~PAGE_MASK
);
378 out_pages
[out_page_num
] = qat_mem_to_page(data
);
379 flat_dst_buf
->pData
= kmap(out_pages
[out_page_num
]) +
381 flat_dst_buf
->dataLenInBytes
=
382 min((long)PAGE_SIZE
- out_page_off
, (long)bytes_left
);
383 data
+= flat_dst_buf
->dataLenInBytes
;
384 bytes_left
-= flat_dst_buf
->dataLenInBytes
;
388 dst_buffer_list
.pBuffers
= flat_dst_buf_array
;
389 dst_buffer_list
.numBuffers
= out_page_num
;
391 op_data
.sessionCtx
= cy_session_ctx
;
392 op_data
.packetType
= CPA_CY_SYM_PACKET_TYPE_FULL
;
393 op_data
.pIv
= NULL
; /* set this later as the J0 block */
394 op_data
.ivLenInBytes
= 0;
395 op_data
.cryptoStartSrcOffsetInBytes
= 0;
396 op_data
.messageLenToCipherInBytes
= 0;
397 op_data
.hashStartSrcOffsetInBytes
= 0;
398 op_data
.messageLenToHashInBytes
= 0;
399 op_data
.pDigestResult
= 0;
400 op_data
.messageLenToCipherInBytes
= enc_len
;
401 op_data
.ivLenInBytes
= ZIO_DATA_IV_LEN
;
402 op_data
.pDigestResult
= digest_buf
;
403 op_data
.pAdditionalAuthData
= aad_buf
;
404 op_data
.pIv
= iv_buf
;
406 cb
.verify_result
= CPA_FALSE
;
407 init_completion(&cb
.complete
);
408 status
= cpaCySymPerformOp(cy_inst_handle
, &cb
, &op_data
,
409 &src_buffer_list
, &dst_buffer_list
, NULL
);
410 if (status
!= CPA_STATUS_SUCCESS
)
413 if (!wait_for_completion_interruptible_timeout(&cb
.complete
,
415 status
= CPA_STATUS_FAIL
;
419 if (cb
.verify_result
== CPA_FALSE
) {
420 status
= CPA_STATUS_FAIL
;
424 if (dir
== QAT_ENCRYPT
)
425 QAT_STAT_INCR(encrypt_total_out_bytes
, enc_len
);
427 QAT_STAT_INCR(decrypt_total_out_bytes
, enc_len
);
430 if (status
!= CPA_STATUS_SUCCESS
)
431 QAT_STAT_BUMP(crypt_fails
);
433 for (i
= 0; i
< in_page_num
; i
++)
435 for (i
= 0; i
< out_page_num
; i
++)
436 kunmap(out_pages
[i
]);
438 cpaCySymRemoveSession(cy_inst_handle
, cy_session_ctx
);
439 QAT_PHYS_CONTIG_FREE(src_buffer_list
.pPrivateMetaData
);
440 QAT_PHYS_CONTIG_FREE(dst_buffer_list
.pPrivateMetaData
);
441 QAT_PHYS_CONTIG_FREE(cy_session_ctx
);
442 QAT_PHYS_CONTIG_FREE(flat_src_buf_array
);
443 QAT_PHYS_CONTIG_FREE(flat_dst_buf_array
);
449 qat_checksum(uint64_t cksum
, uint8_t *buf
, uint64_t size
, zio_cksum_t
*zcp
)
453 CpaInstanceHandle cy_inst_handle
;
454 Cpa16U nr_bufs
= (size
>> PAGE_SHIFT
) + 2;
455 Cpa32U bytes_left
= 0;
457 CpaCySymSessionCtx
*cy_session_ctx
= NULL
;
459 Cpa8U digest_buffer
[sizeof (zio_cksum_t
)];
460 CpaCySymOpData op_data
= { 0 };
461 CpaBufferList src_buffer_list
= { 0 };
462 CpaFlatBuffer
*flat_src_buf_array
= NULL
;
463 CpaFlatBuffer
*flat_src_buf
= NULL
;
464 struct page
*in_pages
[MAX_PAGE_NUM
];
468 QAT_STAT_BUMP(cksum_requests
);
469 QAT_STAT_INCR(cksum_total_in_bytes
, size
);
471 i
= atomic_inc_32_nv(&inst_num
) % num_inst
;
472 cy_inst_handle
= cy_inst_handles
[i
];
474 status
= qat_init_checksum_session_ctx(cy_inst_handle
,
475 &cy_session_ctx
, cksum
);
476 if (status
!= CPA_STATUS_SUCCESS
) {
477 /* don't count unsupported checksums as a failure */
478 if (cksum
== ZIO_CHECKSUM_SHA256
||
479 cksum
== ZIO_CHECKSUM_SHA512
)
480 QAT_STAT_BUMP(cksum_fails
);
485 * We increment nr_bufs by 2 to allow us to handle non
486 * page-aligned buffer addresses and buffers whose sizes
487 * are not divisible by PAGE_SIZE.
489 status
= qat_init_cy_buffer_lists(cy_inst_handle
, nr_bufs
,
490 &src_buffer_list
, &src_buffer_list
);
491 if (status
!= CPA_STATUS_SUCCESS
)
494 status
= QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array
,
495 nr_bufs
* sizeof (CpaFlatBuffer
));
496 if (status
!= CPA_STATUS_SUCCESS
)
501 flat_src_buf
= flat_src_buf_array
;
502 while (bytes_left
> 0) {
503 page_off
= ((long)data
& ~PAGE_MASK
);
504 in_pages
[page_num
] = qat_mem_to_page(data
);
505 flat_src_buf
->pData
= kmap(in_pages
[page_num
]) + page_off
;
506 flat_src_buf
->dataLenInBytes
=
507 min((long)PAGE_SIZE
- page_off
, (long)bytes_left
);
508 data
+= flat_src_buf
->dataLenInBytes
;
509 bytes_left
-= flat_src_buf
->dataLenInBytes
;
513 src_buffer_list
.pBuffers
= flat_src_buf_array
;
514 src_buffer_list
.numBuffers
= page_num
;
516 op_data
.sessionCtx
= cy_session_ctx
;
517 op_data
.packetType
= CPA_CY_SYM_PACKET_TYPE_FULL
;
518 op_data
.hashStartSrcOffsetInBytes
= 0;
519 op_data
.messageLenToHashInBytes
= size
;
520 op_data
.pDigestResult
= digest_buffer
;
522 cb
.verify_result
= CPA_FALSE
;
523 init_completion(&cb
.complete
);
524 status
= cpaCySymPerformOp(cy_inst_handle
, &cb
, &op_data
,
525 &src_buffer_list
, &src_buffer_list
, NULL
);
526 if (status
!= CPA_STATUS_SUCCESS
)
529 if (!wait_for_completion_interruptible_timeout(&cb
.complete
,
531 status
= CPA_STATUS_FAIL
;
535 bcopy(digest_buffer
, zcp
, sizeof (zio_cksum_t
));
538 if (status
!= CPA_STATUS_SUCCESS
)
539 QAT_STAT_BUMP(cksum_fails
);
541 for (i
= 0; i
< page_num
; i
++)
544 cpaCySymRemoveSession(cy_inst_handle
, cy_session_ctx
);
545 QAT_PHYS_CONTIG_FREE(src_buffer_list
.pPrivateMetaData
);
546 QAT_PHYS_CONTIG_FREE(cy_session_ctx
);
547 QAT_PHYS_CONTIG_FREE(flat_src_buf_array
);
552 module_param(zfs_qat_encrypt_disable
, int, 0644);
553 MODULE_PARM_DESC(zfs_qat_encrypt_disable
, "Disable QAT encryption");
555 module_param(zfs_qat_checksum_disable
, int, 0644);
556 MODULE_PARM_DESC(zfs_qat_checksum_disable
, "Disable QAT checksumming");