]>
Commit | Line | Data |
---|---|---|
cf637391 TC |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
9 | * or http://www.opensolaris.org/os/licensing. | |
10 | * See the License for the specific language governing permissions | |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | ||
22 | #if defined(_KERNEL) && defined(HAVE_QAT) | |
23 | #include <linux/slab.h> | |
24 | #include <linux/vmalloc.h> | |
25 | #include <linux/pagemap.h> | |
26 | #include <linux/completion.h> | |
27 | #include <sys/zfs_context.h> | |
28 | #include <sys/zio_crypt.h> | |
29 | #include "lac/cpa_cy_im.h" | |
30 | #include "qat.h" | |
31 | ||
32 | /* | |
33 | * Max instances in QAT device, each instance is a channel to submit | |
34 | * jobs to QAT hardware, this is only for pre-allocating instance, | |
35 | * and session arrays, the actual number of instances are defined in | |
36 | * the QAT driver's configure file. | |
37 | */ | |
38 | #define QAT_CRYPT_MAX_INSTANCES 48 | |
39 | ||
40 | #define MAX_PAGE_NUM 1024 | |
41 | ||
42 | static boolean_t qat_crypt_init_done = B_FALSE; | |
43 | static Cpa16U inst_num = 0; | |
44 | static Cpa16U num_inst = 0; | |
45 | static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES]; | |
46 | ||
47 | typedef struct cy_callback { | |
48 | CpaBoolean verify_result; | |
49 | struct completion complete; | |
50 | } cy_callback_t; | |
51 | ||
52 | static void | |
53 | symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation, | |
54 | void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify) | |
55 | { | |
56 | cy_callback_t *cb = p_callback; | |
57 | ||
58 | if (cb != NULL) { | |
59 | /* indicate that the function has been called */ | |
60 | cb->verify_result = verify; | |
61 | complete(&cb->complete); | |
62 | } | |
63 | } | |
64 | ||
65 | boolean_t | |
66 | qat_crypt_use_accel(size_t s_len) | |
67 | { | |
68 | return (!zfs_qat_disable && | |
69 | qat_crypt_init_done && | |
70 | s_len >= QAT_MIN_BUF_SIZE && | |
71 | s_len <= QAT_MAX_BUF_SIZE); | |
72 | } | |
73 | ||
74 | void | |
75 | qat_crypt_clean(void) | |
76 | { | |
77 | for (Cpa32U i = 0; i < num_inst; i++) | |
78 | cpaCyStopInstance(cy_inst_handles[i]); | |
79 | ||
80 | num_inst = 0; | |
81 | qat_crypt_init_done = B_FALSE; | |
82 | } | |
83 | ||
84 | int | |
85 | qat_crypt_init(void) | |
86 | { | |
87 | Cpa32U i; | |
88 | CpaStatus status = CPA_STATUS_FAIL; | |
89 | ||
90 | status = cpaCyGetNumInstances(&num_inst); | |
91 | if (status != CPA_STATUS_SUCCESS) | |
92 | return (-1); | |
93 | ||
94 | /* if the user has configured no QAT encryption units just return */ | |
95 | if (num_inst == 0) | |
96 | return (0); | |
97 | ||
98 | if (num_inst > QAT_CRYPT_MAX_INSTANCES) | |
99 | num_inst = QAT_CRYPT_MAX_INSTANCES; | |
100 | ||
101 | status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]); | |
102 | if (status != CPA_STATUS_SUCCESS) | |
103 | return (-1); | |
104 | ||
105 | for (i = 0; i < num_inst; i++) { | |
106 | status = cpaCySetAddressTranslation(cy_inst_handles[i], | |
107 | (void *)virt_to_phys); | |
108 | if (status != CPA_STATUS_SUCCESS) | |
109 | goto error; | |
110 | ||
111 | status = cpaCyStartInstance(cy_inst_handles[i]); | |
112 | if (status != CPA_STATUS_SUCCESS) | |
113 | goto error; | |
114 | } | |
115 | ||
116 | qat_crypt_init_done = B_TRUE; | |
117 | return (0); | |
118 | ||
119 | error: | |
120 | qat_crypt_clean(); | |
121 | return (-1); | |
122 | } | |
123 | ||
124 | void | |
125 | qat_crypt_fini(void) | |
126 | { | |
127 | if (!qat_crypt_init_done) | |
128 | return; | |
129 | ||
130 | qat_crypt_clean(); | |
131 | } | |
132 | ||
133 | static CpaStatus | |
134 | init_cy_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle, | |
135 | CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key, | |
136 | Cpa64U crypt, Cpa32U aad_len) | |
137 | { | |
138 | CpaStatus status = CPA_STATUS_SUCCESS; | |
139 | Cpa32U ctx_size; | |
140 | Cpa32U ciper_algorithm; | |
141 | Cpa32U hash_algorithm; | |
142 | CpaCySymSessionSetupData sd = { 0 }; | |
143 | ||
144 | if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) { | |
145 | return (CPA_STATUS_FAIL); | |
146 | } else { | |
147 | ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM; | |
148 | hash_algorithm = CPA_CY_SYM_HASH_AES_GCM; | |
149 | } | |
150 | ||
151 | sd.cipherSetupData.cipherAlgorithm = ciper_algorithm; | |
152 | sd.cipherSetupData.pCipherKey = key->ck_data; | |
153 | sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8; | |
154 | sd.hashSetupData.hashAlgorithm = hash_algorithm; | |
155 | sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH; | |
156 | sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN; | |
157 | sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len; | |
158 | sd.sessionPriority = CPA_CY_PRIORITY_NORMAL; | |
159 | sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING; | |
160 | sd.digestIsAppended = CPA_FALSE; | |
161 | sd.verifyDigest = CPA_FALSE; | |
162 | ||
163 | if (dir == QAT_ENCRYPT) { | |
164 | sd.cipherSetupData.cipherDirection = | |
165 | CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; | |
166 | sd.algChainOrder = | |
167 | CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; | |
168 | } else { | |
169 | ASSERT3U(dir, ==, QAT_DECRYPT); | |
170 | sd.cipherSetupData.cipherDirection = | |
171 | CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; | |
172 | sd.algChainOrder = | |
173 | CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; | |
174 | } | |
175 | ||
176 | status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size); | |
177 | if (status != CPA_STATUS_SUCCESS) | |
178 | return (status); | |
179 | ||
180 | status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size); | |
181 | if (status != CPA_STATUS_SUCCESS) | |
182 | return (status); | |
183 | ||
184 | status = cpaCySymInitSession(inst_handle, symcallback, &sd, | |
185 | *cy_session_ctx); | |
186 | if (status != CPA_STATUS_SUCCESS) { | |
187 | QAT_PHYS_CONTIG_FREE(*cy_session_ctx); | |
188 | return (status); | |
189 | } | |
190 | ||
191 | return (CPA_STATUS_SUCCESS); | |
192 | } | |
193 | ||
194 | static CpaStatus | |
195 | init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs, | |
196 | CpaBufferList *src, CpaBufferList *dst) | |
197 | { | |
198 | CpaStatus status = CPA_STATUS_SUCCESS; | |
199 | Cpa32U meta_size = 0; | |
200 | ||
201 | status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size); | |
202 | if (status != CPA_STATUS_SUCCESS) | |
203 | return (status); | |
204 | ||
205 | src->numBuffers = nr_bufs; | |
206 | status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size); | |
207 | if (status != CPA_STATUS_SUCCESS) | |
208 | goto error; | |
209 | ||
210 | if (src != dst) { | |
211 | dst->numBuffers = nr_bufs; | |
212 | status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData, | |
213 | meta_size); | |
214 | if (status != CPA_STATUS_SUCCESS) | |
215 | goto error; | |
216 | } | |
217 | ||
218 | return (CPA_STATUS_SUCCESS); | |
219 | ||
220 | error: | |
221 | QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData); | |
222 | if (src != dst) | |
223 | QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData); | |
224 | ||
225 | return (status); | |
226 | } | |
227 | ||
228 | int | |
229 | qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, | |
230 | uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf, | |
231 | crypto_key_t *key, uint64_t crypt, uint32_t enc_len) | |
232 | { | |
233 | CpaStatus status = CPA_STATUS_SUCCESS; | |
234 | Cpa16U i; | |
235 | CpaInstanceHandle cy_inst_handle; | |
236 | Cpa16U nr_bufs; | |
237 | Cpa32U bytes_left = 0; | |
238 | Cpa8S *in = NULL; | |
239 | Cpa8S *out = NULL; | |
240 | CpaCySymSessionCtx *cy_session_ctx = NULL; | |
241 | cy_callback_t cb; | |
242 | CpaCySymOpData op_data = { 0 }; | |
243 | CpaBufferList src_buffer_list = { 0 }; | |
244 | CpaBufferList dst_buffer_list = { 0 }; | |
245 | CpaFlatBuffer *flat_src_buf_array = NULL; | |
246 | CpaFlatBuffer *flat_src_buf = NULL; | |
247 | CpaFlatBuffer *flat_dst_buf_array = NULL; | |
248 | CpaFlatBuffer *flat_dst_buf = NULL; | |
249 | struct page *in_pages[MAX_PAGE_NUM]; | |
250 | struct page *out_pages[MAX_PAGE_NUM]; | |
251 | Cpa32S page_num = 0; | |
252 | ||
253 | if (dir == QAT_ENCRYPT) { | |
254 | QAT_STAT_BUMP(encrypt_requests); | |
255 | QAT_STAT_INCR(encrypt_total_in_bytes, enc_len); | |
256 | } else { | |
257 | QAT_STAT_BUMP(decrypt_requests); | |
258 | QAT_STAT_INCR(decrypt_total_in_bytes, enc_len); | |
259 | } | |
260 | ||
261 | i = atomic_inc_32_nv(&inst_num) % num_inst; | |
262 | cy_inst_handle = cy_inst_handles[i]; | |
263 | ||
264 | status = init_cy_session_ctx(dir, cy_inst_handle, &cy_session_ctx, key, | |
265 | crypt, aad_len); | |
266 | if (status != CPA_STATUS_SUCCESS) | |
267 | return (status); | |
268 | ||
269 | nr_bufs = enc_len / PAGE_CACHE_SIZE + | |
270 | (enc_len % PAGE_CACHE_SIZE == 0 ? 0 : 1); | |
271 | status = init_cy_buffer_lists(cy_inst_handle, nr_bufs, &src_buffer_list, | |
272 | &dst_buffer_list); | |
273 | if (status != CPA_STATUS_SUCCESS) | |
274 | goto fail; | |
275 | ||
276 | status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array, | |
277 | nr_bufs * sizeof (CpaFlatBuffer)); | |
278 | if (status != CPA_STATUS_SUCCESS) | |
279 | goto fail; | |
280 | status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array, | |
281 | nr_bufs * sizeof (CpaFlatBuffer)); | |
282 | if (status != CPA_STATUS_SUCCESS) | |
283 | goto fail; | |
284 | ||
285 | bytes_left = enc_len; | |
286 | in = src_buf; | |
287 | out = dst_buf; | |
288 | flat_src_buf = flat_src_buf_array; | |
289 | flat_dst_buf = flat_dst_buf_array; | |
290 | while (bytes_left > 0) { | |
291 | in_pages[page_num] = qat_mem_to_page(in); | |
292 | out_pages[page_num] = qat_mem_to_page(out); | |
293 | flat_src_buf->pData = kmap(in_pages[page_num]); | |
294 | flat_dst_buf->pData = kmap(out_pages[page_num]); | |
295 | flat_src_buf->dataLenInBytes = min((long)PAGE_CACHE_SIZE, | |
296 | (long)bytes_left); | |
297 | flat_dst_buf->dataLenInBytes = min((long)PAGE_CACHE_SIZE, | |
298 | (long)bytes_left); | |
299 | in += flat_src_buf->dataLenInBytes; | |
300 | out += flat_dst_buf->dataLenInBytes; | |
301 | bytes_left -= flat_src_buf->dataLenInBytes; | |
302 | flat_src_buf++; | |
303 | flat_dst_buf++; | |
304 | page_num++; | |
305 | } | |
306 | src_buffer_list.pBuffers = flat_src_buf_array; | |
307 | dst_buffer_list.pBuffers = flat_dst_buf_array; | |
308 | ||
309 | op_data.sessionCtx = cy_session_ctx; | |
310 | op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; | |
311 | op_data.pIv = NULL; /* set this later as the J0 block */ | |
312 | op_data.ivLenInBytes = 0; | |
313 | op_data.cryptoStartSrcOffsetInBytes = 0; | |
314 | op_data.messageLenToCipherInBytes = 0; | |
315 | op_data.hashStartSrcOffsetInBytes = 0; | |
316 | op_data.messageLenToHashInBytes = 0; | |
317 | op_data.pDigestResult = 0; | |
318 | op_data.messageLenToCipherInBytes = enc_len; | |
319 | op_data.ivLenInBytes = ZIO_DATA_IV_LEN; | |
320 | op_data.pDigestResult = digest_buf; | |
321 | op_data.pAdditionalAuthData = aad_buf; | |
322 | op_data.pIv = iv_buf; | |
323 | ||
324 | cb.verify_result = CPA_FALSE; | |
325 | init_completion(&cb.complete); | |
326 | status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data, | |
327 | &src_buffer_list, &dst_buffer_list, NULL); | |
328 | if (status != CPA_STATUS_SUCCESS) | |
329 | goto fail; | |
330 | ||
331 | if (!wait_for_completion_interruptible_timeout(&cb.complete, | |
332 | QAT_TIMEOUT_MS)) { | |
333 | status = CPA_STATUS_FAIL; | |
334 | goto fail; | |
335 | } | |
336 | ||
337 | if (cb.verify_result == CPA_FALSE) { | |
338 | status = CPA_STATUS_FAIL; | |
339 | goto fail; | |
340 | } | |
341 | ||
342 | if (dir == QAT_ENCRYPT) | |
343 | QAT_STAT_INCR(encrypt_total_out_bytes, enc_len); | |
344 | else | |
345 | QAT_STAT_INCR(decrypt_total_out_bytes, enc_len); | |
346 | ||
347 | fail: | |
348 | /* don't count CCM as a failure since it's not supported */ | |
349 | if (status != CPA_STATUS_SUCCESS && | |
350 | zio_crypt_table[crypt].ci_crypt_type != ZC_TYPE_CCM) | |
351 | QAT_STAT_BUMP(crypt_fails); | |
352 | ||
353 | for (i = 0; i < page_num; i ++) { | |
354 | kunmap(in_pages[i]); | |
355 | kunmap(out_pages[i]); | |
356 | } | |
357 | ||
358 | cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx); | |
359 | QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData); | |
360 | QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData); | |
361 | QAT_PHYS_CONTIG_FREE(cy_session_ctx); | |
362 | QAT_PHYS_CONTIG_FREE(flat_src_buf_array); | |
363 | QAT_PHYS_CONTIG_FREE(flat_dst_buf_array); | |
364 | ||
365 | return (status); | |
366 | } | |
367 | ||
368 | module_param(zfs_qat_disable, int, 0644); | |
369 | MODULE_PARM_DESC(zfs_qat_disable, "Disable QAT acceleration"); | |
370 | ||
371 | #endif |