]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | #include "qcccrypto.h" |
2 | #include <iostream> | |
3 | #include "string.h" | |
4 | #include <pthread.h> | |
5 | #include "common/debug.h" | |
6 | #include "include/scope_guard.h" | |
1d09f67e TL |
7 | #include "common/dout.h" |
8 | #include "common/errno.h" | |
11fdf7f2 TL |
9 | |
10 | // ----------------------------------------------------------------------------- | |
11 | #define dout_context g_ceph_context | |
12 | #define dout_subsys ceph_subsys_rgw | |
13 | #undef dout_prefix | |
14 | #define dout_prefix _prefix(_dout) | |
15 | ||
1d09f67e | 16 | static std::ostream& _prefix(std::ostream* _dout) |
11fdf7f2 TL |
17 | { |
18 | return *_dout << "QccCrypto: "; | |
19 | } | |
20 | // ----------------------------------------------------------------------------- | |
21 | ||
22 | /* | |
23 | * Poller thread & functions | |
24 | */ | |
25 | static std::mutex qcc_alloc_mutex; | |
26 | static std::mutex qcc_eng_mutex; | |
27 | static std::atomic<bool> init_called = { false }; | |
28 | ||
29 | void* QccCrypto::crypt_thread(void *args) { | |
30 | struct qcc_thread_args *thread_args = (struct qcc_thread_args *)args; | |
31 | thread_args->qccinstance->do_crypt(thread_args); | |
1d09f67e | 32 | return thread_args; |
11fdf7f2 TL |
33 | } |
34 | ||
35 | void QccCrypto::QccFreeInstance(int entry) { | |
36 | std::lock_guard<std::mutex> freeinst(qcc_alloc_mutex); | |
37 | open_instances.push(entry); | |
38 | } | |
39 | ||
40 | int QccCrypto::QccGetFreeInstance() { | |
41 | int ret = -1; | |
42 | std::lock_guard<std::mutex> getinst(qcc_alloc_mutex); | |
43 | if (!open_instances.empty()) { | |
44 | ret = open_instances.front(); | |
45 | open_instances.pop(); | |
46 | } | |
47 | return ret; | |
48 | } | |
49 | ||
50 | void QccCrypto::cleanup() { | |
51 | icp_sal_userStop(); | |
52 | qaeMemDestroy(); | |
53 | is_init = false; | |
54 | init_stat = stat; | |
55 | init_called = false; | |
56 | derr << "Failure during QAT init sequence. Quitting" << dendl; | |
57 | } | |
58 | ||
59 | /* | |
60 | * We initialize QAT instance and everything that is common for all ops | |
61 | */ | |
62 | bool QccCrypto::init() | |
63 | { | |
64 | ||
65 | std::lock_guard<std::mutex> l(qcc_eng_mutex); | |
66 | ||
67 | if(init_called) { | |
68 | dout(10) << "Init sequence already called. Skipping duplicate call" << dendl; | |
69 | return true; | |
70 | } | |
71 | ||
72 | // First call to init | |
73 | dout(15) << "First init for QAT" << dendl; | |
74 | init_called = true; | |
75 | ||
76 | // Find if the usermode memory driver is available. We need to this to | |
77 | // create contiguous memory needed by QAT. | |
78 | stat = qaeMemInit(); | |
79 | if(stat != CPA_STATUS_SUCCESS) { | |
80 | derr << "Unable to load memory driver" << dendl; | |
81 | this->cleanup(); | |
82 | return false; | |
83 | } | |
84 | ||
85 | stat = icp_sal_userStart("CEPH"); | |
86 | if(stat != CPA_STATUS_SUCCESS) { | |
87 | derr << "Unable to start qat device" << dendl; | |
88 | this->cleanup(); | |
89 | return false; | |
90 | } | |
91 | ||
92 | qcc_os_mem_alloc((void **)&qcc_inst, sizeof(QCCINST)); | |
93 | if(qcc_inst == NULL) { | |
94 | derr << "Unable to alloc mem for instance struct" << dendl; | |
95 | this->cleanup(); | |
96 | return false; | |
97 | } | |
98 | ||
99 | // Initialize contents of qcc_inst | |
100 | qcc_inst->num_instances = 0; | |
101 | qcc_inst->cy_inst_handles = NULL; | |
102 | ||
103 | stat = cpaCyGetNumInstances(&(qcc_inst->num_instances)); | |
104 | if ((stat != CPA_STATUS_SUCCESS) || (qcc_inst->num_instances <= 0)) { | |
105 | derr << "Unable to find available instances" << dendl; | |
106 | this->cleanup(); | |
107 | return false; | |
108 | } | |
109 | ||
110 | qcc_os_mem_alloc((void **)&qcc_inst->cy_inst_handles, | |
111 | ((int)qcc_inst->num_instances * sizeof(CpaInstanceHandle))); | |
112 | if (qcc_inst->cy_inst_handles == NULL) { | |
113 | derr << "Unable to allocate instances array memory" << dendl; | |
114 | this->cleanup(); | |
115 | return false; | |
116 | } | |
117 | ||
118 | stat = cpaCyGetInstances(qcc_inst->num_instances, qcc_inst->cy_inst_handles); | |
119 | if (stat != CPA_STATUS_SUCCESS) { | |
120 | derr << "Unable to get instances" << dendl; | |
121 | this->cleanup(); | |
122 | return false; | |
123 | } | |
124 | ||
125 | int iter = 0; | |
126 | //Start Instances | |
127 | for(iter = 0; iter < qcc_inst->num_instances; iter++) { | |
128 | stat = cpaCyStartInstance(qcc_inst->cy_inst_handles[iter]); | |
129 | if(stat != CPA_STATUS_SUCCESS) { | |
130 | derr << "Unable to start instance" << dendl; | |
131 | this->cleanup(); | |
132 | return false; | |
133 | } | |
134 | } | |
135 | ||
136 | qcc_os_mem_alloc((void **)&qcc_inst->is_polled, | |
137 | ((int)qcc_inst->num_instances * sizeof(CpaBoolean))); | |
138 | CpaInstanceInfo2 info; | |
139 | for(iter = 0; iter < qcc_inst->num_instances; iter++) { | |
140 | qcc_inst->is_polled[iter] = cpaCyInstanceGetInfo2(qcc_inst->cy_inst_handles[iter], | |
141 | &info) == CPA_STATUS_SUCCESS ? info.isPolled : CPA_FALSE; | |
142 | } | |
143 | ||
144 | // Allocate memory structures for all instances | |
145 | qcc_os_mem_alloc((void **)&qcc_sess, | |
146 | ((int)qcc_inst->num_instances * sizeof(QCCSESS))); | |
147 | if(qcc_sess == NULL) { | |
148 | derr << "Unable to allocate memory for session struct" << dendl; | |
149 | this->cleanup(); | |
150 | return false; | |
151 | } | |
152 | ||
153 | qcc_os_mem_alloc((void **)&qcc_op_mem, | |
154 | ((int)qcc_inst->num_instances * sizeof(QCCOPMEM))); | |
155 | if(qcc_sess == NULL) { | |
156 | derr << "Unable to allocate memory for opmem struct" << dendl; | |
157 | this->cleanup(); | |
158 | return false; | |
159 | } | |
160 | ||
161 | qcc_os_mem_alloc((void **)&cypollthreads, | |
162 | ((int)qcc_inst->num_instances * sizeof(pthread_t))); | |
163 | if(cypollthreads == NULL) { | |
164 | derr << "Unable to allocate memory for pthreads" << dendl; | |
165 | this->cleanup(); | |
166 | return false; | |
167 | } | |
168 | ||
169 | //At this point we are only doing an user-space version. | |
170 | //To-Do: Maybe a kernel based one | |
171 | for(iter = 0; iter < qcc_inst->num_instances; iter++) { | |
172 | stat = cpaCySetAddressTranslation(qcc_inst->cy_inst_handles[iter], | |
173 | qaeVirtToPhysNUMA); | |
174 | if(stat == CPA_STATUS_SUCCESS) { | |
175 | // Start HW Polling Thread | |
176 | // To-Do: Enable epoll & interrupt based later? | |
177 | // QccCyStartPoll(iter); | |
178 | // Setup the session structures for crypto operation and populate | |
179 | // whatever we can now. Rest will be filled in when crypto operation | |
180 | // happens. | |
181 | qcc_sess[iter].sess_ctx_sz = 0; | |
182 | qcc_sess[iter].sess_ctx = NULL; | |
183 | qcc_sess[iter].sess_stp_data.sessionPriority = CPA_CY_PRIORITY_NORMAL; | |
184 | qcc_sess[iter].sess_stp_data.symOperation = CPA_CY_SYM_OP_CIPHER; | |
185 | open_instances.push(iter); | |
186 | qcc_op_mem[iter].is_mem_alloc = false; | |
187 | qcc_op_mem[iter].op_complete = false; | |
188 | qcc_op_mem[iter].op_result = CPA_STATUS_SUCCESS; | |
189 | qcc_op_mem[iter].sym_op_data = NULL; | |
190 | qcc_op_mem[iter].buff_meta_size = qcc_op_mem[iter].buff_size = 0; | |
191 | qcc_op_mem[iter].src_buff_meta = qcc_op_mem[iter].src_buff | |
192 | = qcc_op_mem[iter].iv_buff = NULL; | |
193 | qcc_op_mem[iter].src_buff_list = NULL; | |
194 | qcc_op_mem[iter].src_buff_flat = NULL; | |
195 | qcc_op_mem[iter].num_buffers = 1; | |
196 | } else { | |
197 | derr << "Unable to find address translations of instance " << iter << dendl; | |
198 | this->cleanup(); | |
199 | return false; | |
200 | } | |
201 | } | |
202 | is_init = true; | |
203 | dout(10) << "Init complete" << dendl; | |
204 | return true; | |
205 | } | |
206 | ||
207 | bool QccCrypto::destroy() { | |
208 | if((!is_init) || (!init_called)) { | |
209 | dout(15) << "QAT not initialized here. Nothing to do" << dendl; | |
210 | return false; | |
211 | } | |
212 | ||
213 | unsigned int retry = 0; | |
214 | while(retry <= QCC_MAX_RETRIES) { | |
215 | if(open_instances.size() == qcc_inst->num_instances) { | |
216 | break; | |
217 | } else { | |
218 | retry++; | |
219 | } | |
220 | dout(5) << "QAT is still busy and cannot free resources yet" << dendl; | |
221 | return false; | |
222 | } | |
223 | ||
224 | dout(10) << "Destroying QAT crypto & related memory" << dendl; | |
225 | int iter = 0; | |
226 | ||
227 | // Free up op related memory | |
228 | for (iter =0; iter < qcc_inst->num_instances; iter++) { | |
229 | qcc_contig_mem_free((void **)&(qcc_op_mem[iter].src_buff)); | |
230 | qcc_contig_mem_free((void **)&(qcc_op_mem[iter].iv_buff)); | |
231 | qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_list)); | |
232 | qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_flat)); | |
233 | qcc_contig_mem_free((void **)&(qcc_op_mem[iter].sym_op_data)); | |
234 | } | |
235 | ||
236 | // Free up Session memory | |
237 | for(iter = 0; iter < qcc_inst->num_instances; iter++) { | |
238 | cpaCySymRemoveSession(qcc_inst->cy_inst_handles[iter], qcc_sess[iter].sess_ctx); | |
239 | qcc_contig_mem_free((void **)&(qcc_sess[iter].sess_ctx)); | |
240 | } | |
241 | ||
242 | // Stop QAT Instances | |
243 | for(iter = 0; iter < qcc_inst->num_instances; iter++) { | |
244 | cpaCyStopInstance(qcc_inst->cy_inst_handles[iter]); | |
245 | } | |
246 | ||
247 | // Free up the base structures we use | |
248 | qcc_os_mem_free((void **)&qcc_op_mem); | |
249 | qcc_os_mem_free((void **)&qcc_sess); | |
250 | qcc_os_mem_free((void **)&(qcc_inst->cy_inst_handles)); | |
251 | qcc_os_mem_free((void **)&(qcc_inst->is_polled)); | |
252 | qcc_os_mem_free((void **)&cypollthreads); | |
253 | qcc_os_mem_free((void **)&qcc_inst); | |
254 | ||
255 | //Un-init memory driver and QAT HW | |
256 | icp_sal_userStop(); | |
257 | qaeMemDestroy(); | |
258 | init_called = false; | |
259 | is_init = false; | |
260 | return true; | |
261 | } | |
262 | ||
263 | void QccCrypto::do_crypt(qcc_thread_args *thread_args) { | |
264 | auto entry = thread_args->entry; | |
265 | qcc_op_mem[entry].op_result = cpaCySymPerformOp(qcc_inst->cy_inst_handles[entry], | |
266 | NULL, | |
267 | qcc_op_mem[entry].sym_op_data, | |
268 | qcc_op_mem[entry].src_buff_list, | |
269 | qcc_op_mem[entry].src_buff_list, | |
270 | NULL); | |
271 | qcc_op_mem[entry].op_complete = true; | |
272 | free(thread_args); | |
273 | } | |
274 | ||
275 | bool QccCrypto::perform_op(unsigned char* out, const unsigned char* in, | |
276 | size_t size, uint8_t *iv, uint8_t *key, CpaCySymCipherDirection op_type) | |
277 | { | |
278 | if (!init_called) { | |
279 | dout(10) << "QAT not intialized yet. Initializing now..." << dendl; | |
280 | if(!QccCrypto::init()) { | |
281 | derr << "QAT init failed" << dendl; | |
282 | return false; | |
283 | } | |
284 | } | |
285 | ||
286 | if(!is_init) | |
287 | { | |
288 | dout(10) << "QAT not initialized in this instance or init failed with possible error " << (int)init_stat << dendl; | |
289 | return is_init; | |
290 | } | |
291 | ||
292 | int avail_inst = -1; | |
293 | unsigned int retrycount = 0; | |
294 | while(retrycount <= QCC_MAX_RETRIES) { | |
295 | avail_inst = QccGetFreeInstance(); | |
296 | if(avail_inst != -1) { | |
297 | break; | |
298 | } else { | |
299 | retrycount++; | |
300 | usleep(qcc_sleep_duration); | |
301 | } | |
302 | } | |
303 | ||
304 | if(avail_inst == -1) { | |
305 | derr << "Unable to get an QAT instance. Failing request" << dendl; | |
306 | return false; | |
307 | } | |
308 | ||
309 | dout(15) << "Using inst " << avail_inst << dendl; | |
310 | // Start polling threads for this instance | |
311 | //QccCyStartPoll(avail_inst); | |
312 | ||
313 | auto sg = make_scope_guard([=] { | |
314 | //free up the instance irrespective of the op status | |
315 | dout(15) << "Completed task under " << avail_inst << dendl; | |
316 | qcc_op_mem[avail_inst].op_complete = false; | |
317 | QccCrypto::QccFreeInstance(avail_inst); | |
318 | }); | |
319 | ||
320 | /* | |
321 | * Allocate buffers for this version of the instance if not already done. | |
322 | * Hold onto to most of them until destructor is called. | |
323 | */ | |
324 | if (qcc_op_mem[avail_inst].is_mem_alloc == false) { | |
325 | ||
326 | qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherAlgorithm = | |
327 | CPA_CY_SYM_CIPHER_AES_CBC; | |
328 | qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherKeyLenInBytes = | |
329 | AES_256_KEY_SIZE; | |
330 | ||
331 | // Allocate contig memory for buffers that are independent of the | |
332 | // input/output | |
333 | stat = cpaCyBufferListGetMetaSize(qcc_inst->cy_inst_handles[avail_inst], | |
334 | qcc_op_mem[avail_inst].num_buffers, &(qcc_op_mem[avail_inst].buff_meta_size)); | |
335 | if(stat != CPA_STATUS_SUCCESS) { | |
336 | derr << "Unable to get buff meta size" << dendl; | |
337 | return false; | |
338 | } | |
339 | ||
340 | // Allocate Buffer List Private metadata | |
341 | stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_meta), | |
342 | qcc_op_mem[avail_inst].buff_meta_size, 1); | |
343 | if(stat != CPA_STATUS_SUCCESS) { | |
344 | derr << "Unable to allocate private metadata memory" << dendl; | |
345 | return false; | |
346 | } | |
347 | ||
348 | // Allocate Buffer List Memory | |
349 | qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_list), sizeof(CpaBufferList)); | |
350 | qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_flat), | |
351 | (qcc_op_mem[avail_inst].num_buffers * sizeof(CpaFlatBuffer))); | |
352 | if(qcc_op_mem[avail_inst].src_buff_list == NULL || qcc_op_mem[avail_inst].src_buff_flat == NULL) { | |
353 | derr << "Unable to allocate bufferlist memory" << dendl; | |
354 | return false; | |
355 | } | |
356 | ||
357 | // Allocate IV memory | |
358 | stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].iv_buff), AES_256_IV_LEN); | |
359 | if(stat != CPA_STATUS_SUCCESS) { | |
360 | derr << "Unable to allocate bufferlist memory" << dendl; | |
361 | return false; | |
362 | } | |
363 | ||
364 | //Assign src stuff for the operation | |
365 | (qcc_op_mem[avail_inst].src_buff_list)->pBuffers = qcc_op_mem[avail_inst].src_buff_flat; | |
366 | (qcc_op_mem[avail_inst].src_buff_list)->numBuffers = qcc_op_mem[avail_inst].num_buffers; | |
367 | (qcc_op_mem[avail_inst].src_buff_list)->pPrivateMetaData = qcc_op_mem[avail_inst].src_buff_meta; | |
368 | ||
369 | //Setup OpData | |
370 | stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].sym_op_data), | |
371 | sizeof(CpaCySymOpData)); | |
372 | if(stat != CPA_STATUS_SUCCESS) { | |
373 | derr << "Unable to allocate opdata memory" << dendl; | |
374 | return false; | |
375 | } | |
376 | ||
377 | // Assuming op to be encryption for initiation. This will be reset when we | |
378 | // exit this block | |
379 | qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection = | |
380 | CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; | |
381 | // Allocate Session memory | |
382 | stat = cpaCySymSessionCtxGetSize(qcc_inst->cy_inst_handles[avail_inst], | |
383 | &(qcc_sess[avail_inst].sess_stp_data), &(qcc_sess[avail_inst].sess_ctx_sz)); | |
384 | if(stat != CPA_STATUS_SUCCESS) { | |
385 | derr << "Unable to find session size" << dendl; | |
386 | return false; | |
387 | } | |
388 | ||
389 | stat = qcc_contig_mem_alloc((void **)&(qcc_sess[avail_inst].sess_ctx), | |
390 | qcc_sess[avail_inst].sess_ctx_sz); | |
391 | if(stat != CPA_STATUS_SUCCESS) { | |
392 | derr << "Unable to allocate contig memory" << dendl; | |
393 | return false; | |
394 | } | |
395 | ||
396 | // Set memalloc flag so that we don't go through this exercise again. | |
397 | qcc_op_mem[avail_inst].is_mem_alloc = true; | |
398 | dout(15) << "Instantiation complete for " << avail_inst << dendl; | |
399 | } | |
400 | ||
401 | // Section that runs on every call | |
402 | // Identify the operation and assign to session | |
403 | qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection = op_type; | |
404 | qcc_sess[avail_inst].sess_stp_data.cipherSetupData.pCipherKey = (Cpa8U *)key; | |
405 | ||
406 | stat = cpaCySymInitSession(qcc_inst->cy_inst_handles[avail_inst], | |
407 | NULL, | |
408 | &(qcc_sess[avail_inst].sess_stp_data), | |
409 | qcc_sess[avail_inst].sess_ctx); | |
410 | if (stat != CPA_STATUS_SUCCESS) { | |
411 | derr << "Unable to init session" << dendl; | |
412 | return false; | |
413 | } | |
414 | ||
415 | // Allocate actual buffers that will hold data | |
416 | if (qcc_op_mem[avail_inst].buff_size != (Cpa32U)size) { | |
417 | qcc_contig_mem_free((void **)&(qcc_op_mem[avail_inst].src_buff)); | |
418 | qcc_op_mem[avail_inst].buff_size = (Cpa32U)size; | |
419 | stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff), | |
420 | qcc_op_mem[avail_inst].buff_size); | |
421 | if(stat != CPA_STATUS_SUCCESS) { | |
422 | derr << "Unable to allocate contig memory" << dendl; | |
423 | return false; | |
424 | } | |
425 | } | |
426 | ||
427 | // Copy src & iv into the respective buffers | |
428 | memcpy(qcc_op_mem[avail_inst].src_buff, in, size); | |
429 | memcpy(qcc_op_mem[avail_inst].iv_buff, iv, AES_256_IV_LEN); | |
430 | ||
431 | //Assign the reminder of the stuff | |
432 | qcc_op_mem[avail_inst].src_buff_flat->dataLenInBytes = qcc_op_mem[avail_inst].buff_size; | |
433 | qcc_op_mem[avail_inst].src_buff_flat->pData = qcc_op_mem[avail_inst].src_buff; | |
434 | ||
435 | //OpData assignment | |
436 | qcc_op_mem[avail_inst].sym_op_data->sessionCtx = qcc_sess[avail_inst].sess_ctx; | |
437 | qcc_op_mem[avail_inst].sym_op_data->packetType = CPA_CY_SYM_PACKET_TYPE_FULL; | |
438 | qcc_op_mem[avail_inst].sym_op_data->pIv = qcc_op_mem[avail_inst].iv_buff; | |
439 | qcc_op_mem[avail_inst].sym_op_data->ivLenInBytes = AES_256_IV_LEN; | |
440 | qcc_op_mem[avail_inst].sym_op_data->cryptoStartSrcOffsetInBytes = 0; | |
441 | qcc_op_mem[avail_inst].sym_op_data->messageLenToCipherInBytes = qcc_op_mem[avail_inst].buff_size; | |
442 | ||
443 | // Perform cipher operation in a thread | |
444 | qcc_thread_args* thread_args = new qcc_thread_args(); | |
445 | thread_args->qccinstance = this; | |
446 | thread_args->entry = avail_inst; | |
447 | ||
448 | if (pthread_create(&cypollthreads[avail_inst], NULL, crypt_thread, (void *)thread_args) != 0) { | |
449 | derr << "Unable to create thread for crypt operation" << dendl; | |
450 | return false; | |
451 | } | |
452 | if (qcc_inst->is_polled[avail_inst] == CPA_TRUE) { | |
453 | while (!qcc_op_mem[avail_inst].op_complete) { | |
454 | icp_sal_CyPollInstance(qcc_inst->cy_inst_handles[avail_inst], 0); | |
455 | } | |
456 | } | |
457 | pthread_join(cypollthreads[avail_inst], NULL); | |
458 | ||
459 | if(qcc_op_mem[avail_inst].op_result != CPA_STATUS_SUCCESS) { | |
460 | derr << "Unable to perform crypt operation" << dendl; | |
461 | return false; | |
462 | } | |
463 | ||
464 | //Copy data back to out buffer | |
465 | memcpy(out, qcc_op_mem[avail_inst].src_buff, size); | |
466 | //Always cleanup memory holding user-data at the end | |
467 | memset(qcc_op_mem[avail_inst].iv_buff, 0, AES_256_IV_LEN); | |
468 | memset(qcc_op_mem[avail_inst].src_buff, 0, qcc_op_mem[avail_inst].buff_size); | |
469 | ||
470 | return true; | |
471 | } |