]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/intel-ipsec-mb/mb_mgr_code.h
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
update sources to ceph Nautilus 14.2.1
[ceph.git] / ceph / src / spdk / intel-ipsec-mb / mb_mgr_code.h
1 /*******************************************************************************
2 Copyright (c) 2012-2018, Intel Corporation
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions are met:
6
7 * Redistributions of source code must retain the above copyright notice,
8 this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in the
11 documentation and/or other materials provided with the distribution.
12 * Neither the name of Intel Corporation nor the names of its contributors
13 may be used to endorse or promote products derived from this software
14 without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *******************************************************************************/
27
28 /*
29 * This contains the bulk of the mb_mgr code, with #define's to build
30 * an SSE, AVX, AVX2 or AVX512 version (see mb_mgr_sse.c, mb_mgr_avx.c, etc.)
31 *
32 * get_next_job() returns a job object. This must be filled in and returned
33 * via submit_job() before get_next_job() is called again.
34 *
35 * submit_job() and flush_job() returns a job object. This job object ceases
36 * to be usable at the next call to get_next_job()
37 *
38 * Assume JOBS() and ADV_JOBS() from mb_mgr_code.h are available
39 */
40
41 #include <string.h> /* memcpy(), memset() */
42
43 /* ========================================================================= */
44 /* Lower level "out of order" schedulers */
45 /* ========================================================================= */
46
47 __forceinline
48 JOB_AES_HMAC *
49 SUBMIT_JOB_AES128_DEC(JOB_AES_HMAC *job)
50 {
51 AES_CBC_DEC_128(job->src + job->cipher_start_src_offset_in_bytes,
52 job->iv,
53 job->aes_dec_key_expanded,
54 job->dst,
55 job->msg_len_to_cipher_in_bytes & (~15));
56 job->status |= STS_COMPLETED_AES;
57 return job;
58 }
59
60 __forceinline
61 JOB_AES_HMAC *
62 SUBMIT_JOB_AES192_DEC(JOB_AES_HMAC *job)
63 {
64 AES_CBC_DEC_192(job->src + job->cipher_start_src_offset_in_bytes,
65 job->iv,
66 job->aes_dec_key_expanded,
67 job->dst,
68 job->msg_len_to_cipher_in_bytes);
69 job->status |= STS_COMPLETED_AES;
70 return job;
71 }
72
73 __forceinline
74 JOB_AES_HMAC *
75 SUBMIT_JOB_AES256_DEC(JOB_AES_HMAC *job)
76 {
77 AES_CBC_DEC_256(job->src + job->cipher_start_src_offset_in_bytes,
78 job->iv,
79 job->aes_dec_key_expanded,
80 job->dst,
81 job->msg_len_to_cipher_in_bytes);
82 job->status |= STS_COMPLETED_AES;
83 return job;
84 }
85
86 __forceinline
87 JOB_AES_HMAC *
88 SUBMIT_JOB_AES128_CNTR(JOB_AES_HMAC *job)
89 {
90 AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes,
91 job->iv,
92 job->aes_enc_key_expanded,
93 job->dst,
94 job->msg_len_to_cipher_in_bytes,
95 job->iv_len_in_bytes);
96 job->status |= STS_COMPLETED_AES;
97 return job;
98 }
99
100 __forceinline
101 JOB_AES_HMAC *
102 SUBMIT_JOB_AES192_CNTR(JOB_AES_HMAC *job)
103 {
104 AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes,
105 job->iv,
106 job->aes_enc_key_expanded,
107 job->dst,
108 job->msg_len_to_cipher_in_bytes,
109 job->iv_len_in_bytes);
110 job->status |= STS_COMPLETED_AES;
111 return job;
112 }
113
114 __forceinline
115 JOB_AES_HMAC *
116 SUBMIT_JOB_AES256_CNTR(JOB_AES_HMAC *job)
117 {
118 AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes,
119 job->iv,
120 job->aes_enc_key_expanded,
121 job->dst,
122 job->msg_len_to_cipher_in_bytes,
123 job->iv_len_in_bytes);
124 job->status |= STS_COMPLETED_AES;
125 return job;
126 }
127
128 /* ========================================================================= */
129 /* AES-CCM */
130 /* ========================================================================= */
131
132 __forceinline
133 JOB_AES_HMAC *
134 submit_flush_job_aes_ccm(MB_MGR_CCM_OOO *state, JOB_AES_HMAC *job,
135 const unsigned max_jobs, const int is_submit)
136 {
137 const unsigned lane_blocks_size = 64;
138 const unsigned aad_len_size = 2;
139 unsigned lane, min_len, min_idx;
140 JOB_AES_HMAC *ret_job = NULL;
141 uint8_t *pb = NULL;
142 unsigned i;
143
144 if (is_submit) {
145 /*
146 * SUBMIT
147 * - get a free lane id
148 */
149 const unsigned L = AES_BLOCK_SIZE - 1 -
150 (unsigned) job->iv_len_in_bytes;
151
152 lane = state->unused_lanes & 15;
153 state->unused_lanes >>= 4;
154 pb = &state->init_blocks[lane * lane_blocks_size];
155
156 /*
157 * Build IV for AES-CTR-128.
158 * - byte 0: flags with L'
159 * - bytes 1 to 13: nonce
160 * - zero bytes after nonce (up to byte 15)
161 *
162 * First AES block of init_blocks will always hold this format
163 * throughtout job processing.
164 */
165 memset(&pb[8], 0, 8);
166 pb[0] = (uint8_t) L - 1; /* flags = L` = L - 1 */
167 /* nonce 7 to 13 */
168 memcpy(&pb[1], job->iv, job->iv_len_in_bytes);
169
170 if (job->cipher_direction != ENCRYPT) {
171 /* decrypt before authentication */
172 pb[15] = 1;
173 AES_CNTR_128(job->src +
174 job->cipher_start_src_offset_in_bytes,
175 pb, job->aes_enc_key_expanded, job->dst,
176 job->msg_len_to_cipher_in_bytes,
177 AES_BLOCK_SIZE);
178 }
179
180 /* copy job data in and set up inital blocks */
181 state->job_in_lane[lane] = job;
182 state->lens[lane] = AES_BLOCK_SIZE;
183 state->init_done[lane] = 0;
184 state->args.in[lane] = pb;
185 state->args.keys[lane] = job->aes_enc_key_expanded;
186 memset(&state->args.IV[lane], 0, sizeof(state->args.IV[0]));
187
188 /*
189 * Convert AES-CTR IV into BLOCK 0 for CBC-MAC-128:
190 * - correct flags by adding M' (AAD later)
191 * - put message length
192 */
193 pb[0] |= ((job->auth_tag_output_len_in_bytes - 2) >> 1) << 3;
194 pb[14] = (uint8_t) (job->msg_len_to_hash_in_bytes >> 8);
195 pb[15] = (uint8_t) job->msg_len_to_hash_in_bytes;
196
197 /* Make AAD correction and put together AAD blocks, if any */
198 if (job->u.CCM.aad_len_in_bytes != 0) {
199 /*
200 * - increment length by length of AAD and
201 * AAD length size
202 * - add AAD present flag
203 * - copy AAD to the lane initial blocks
204 * - zero trailing block bytes
205 */
206 const unsigned aadl =
207 (unsigned) job->u.CCM.aad_len_in_bytes +
208 aad_len_size;
209
210 state->lens[lane] +=
211 (aadl + AES_BLOCK_SIZE - 1) &
212 (~(AES_BLOCK_SIZE - 1));
213 pb[0] |= 0x40;
214 pb[AES_BLOCK_SIZE + 0] =
215 (uint8_t) (job->u.CCM.aad_len_in_bytes >> 8);
216 pb[AES_BLOCK_SIZE + 1] =
217 (uint8_t) job->u.CCM.aad_len_in_bytes;
218 memcpy(&pb[AES_BLOCK_SIZE + aad_len_size],
219 job->u.CCM.aad,
220 job->u.CCM.aad_len_in_bytes);
221 memset(&pb[AES_BLOCK_SIZE + aadl], 0,
222 state->lens[lane] - aadl);
223 }
224
225 /* enough jobs to start processing? */
226 if (state->unused_lanes != 0xf)
227 return NULL;
228 } else {
229 /*
230 * FLUSH
231 * - find 1st non null job
232 */
233 for (lane = 0; lane < max_jobs; lane++)
234 if (state->job_in_lane[lane] != NULL)
235 break;
236 if (lane >= max_jobs)
237 return NULL; /* no not null job */
238 }
239
240 ccm_round:
241 if (is_submit) {
242 /*
243 * SUBMIT
244 * - find min common length to process
245 */
246 min_idx = 0;
247 min_len = state->lens[0];
248
249 for (i = 1; i < max_jobs; i++) {
250 if (min_len > state->lens[i]) {
251 min_idx = i;
252 min_len = state->lens[i];
253 }
254 }
255 } else {
256 /*
257 * FLUSH
258 * - copy good (not null) lane onto empty lanes
259 * - find min common length to process across not null lanes
260 */
261 min_idx = lane;
262 min_len = state->lens[lane];
263
264 for (i = 0; i < max_jobs; i++) {
265 if (i == lane)
266 continue;
267
268 if (state->job_in_lane[i] != NULL) {
269 if (min_len > state->lens[i]) {
270 min_idx = i;
271 min_len = state->lens[i];
272 }
273 } else {
274 state->args.in[i] = state->args.in[lane];
275 state->args.keys[i] = state->args.keys[lane];
276 state->args.IV[i] = state->args.IV[lane];
277 state->lens[i] = UINT16_MAX;
278 state->init_done[i] = state->init_done[lane];
279 }
280 }
281 }
282
283 /* subtract min len from all lanes */
284 for (i = 0; i < max_jobs; i++)
285 state->lens[i] -= min_len;
286
287 /* run the algorythmic code on selected blocks */
288 if (min_len != 0)
289 AES128_CBC_MAC(&state->args, min_len);
290
291 ret_job = state->job_in_lane[min_idx];
292 pb = &state->init_blocks[min_idx * lane_blocks_size];
293
294 if (state->init_done[min_idx] == 0) {
295 /*
296 * First block and AAD blocks are done.
297 * Full message blocks are to do.
298 */
299 if (ret_job->cipher_direction == ENCRYPT)
300 state->args.in[min_idx] = ret_job->src +
301 ret_job->hash_start_src_offset_in_bytes;
302 else
303 state->args.in[min_idx] = ret_job->dst;
304
305 state->init_done[min_idx] = 1;
306
307 if (ret_job->msg_len_to_hash_in_bytes & (~15)) {
308 /* first block + AAD done - process message blocks */
309 state->lens[min_idx] =
310 ret_job->msg_len_to_hash_in_bytes & (~15);
311 goto ccm_round;
312 }
313 }
314
315 if (state->init_done[min_idx] == 1 &&
316 (ret_job->msg_len_to_hash_in_bytes & 15)) {
317 /*
318 * First block, AAD, message blocks are done.
319 * Partial message block is still to do.
320 */
321 state->init_done[min_idx] = 2;
322 state->lens[min_idx] = AES_BLOCK_SIZE;
323 memset(&pb[AES_BLOCK_SIZE], 0, AES_BLOCK_SIZE);
324 memcpy(&pb[AES_BLOCK_SIZE], state->args.in[min_idx],
325 (size_t) ret_job->msg_len_to_hash_in_bytes & 15);
326 state->args.in[min_idx] = &pb[AES_BLOCK_SIZE];
327 goto ccm_round;
328 }
329
330 /*
331 * Final XOR with AES-CNTR on B_0
332 * - remove M' and AAD presence bits from flags
333 * - set counter to 0
334 */
335 pb[0] = pb[0] & 7;
336 pb[14] = 0;
337 pb[15] = 0;
338
339 /*
340 * Clever use of AES-CTR mode saves a few ops here.
341 * What AES-CCM authentication requires us to do is:
342 * AES-CCM: E(KEY,B_0) XOR IV_CBC_MAC
343 *
344 * And what AES_CTR offers is:
345 * AES_CTR: E(KEY, NONCE|COUNTER) XOR PLAIN_TEXT
346 *
347 * So if:
348 * B_0 is passed instead of NONCE|COUNTER and IV instead of PLAIN_TESXT
349 * then AES_CTR function is doing pretty much what we need.
350 * On top of it can truncate the authentication tag and copy to
351 * destination.
352 */
353 AES_CNTR_128(&state->args.IV[min_idx] /* src = IV */,
354 pb /* nonce/iv = B_0 */,
355 state->args.keys[min_idx],
356 ret_job->auth_tag_output /* dst */,
357 ret_job->auth_tag_output_len_in_bytes /* num_bytes */,
358 AES_BLOCK_SIZE /* nonce/iv len */);
359
360 if (ret_job->cipher_direction == ENCRYPT) {
361 /* encrypt after authentication */
362 pb[15] = 1; /* start from counter 1, not 0 */
363 AES_CNTR_128(ret_job->src +
364 ret_job->cipher_start_src_offset_in_bytes,
365 pb, ret_job->aes_enc_key_expanded, ret_job->dst,
366 ret_job->msg_len_to_cipher_in_bytes,
367 AES_BLOCK_SIZE);
368 }
369
370 /* put back processed packet into unused lanes, set job as complete */
371 state->unused_lanes = (state->unused_lanes << 4) | min_idx;
372 ret_job = state->job_in_lane[min_idx];
373 ret_job->status |= (STS_COMPLETED_HMAC|STS_COMPLETED_AES);
374 state->job_in_lane[min_idx] = NULL;
375 return ret_job;
376 }
377
378 static
379 JOB_AES_HMAC *
380 submit_job_aes_ccm_auth_arch(MB_MGR_CCM_OOO *state, JOB_AES_HMAC *job)
381 {
382 return submit_flush_job_aes_ccm(state, job, AES_CCM_MAX_JOBS, 1);
383 }
384
385 static
386 JOB_AES_HMAC *
387 flush_job_aes_ccm_auth_arch(MB_MGR_CCM_OOO *state)
388 {
389 return submit_flush_job_aes_ccm(state, NULL, AES_CCM_MAX_JOBS, 0);
390 }
391 /* ========================================================================= */
392 /* AES-CMAC */
393 /* ========================================================================= */
394
395 /*
396 * Implementation follows Figure 2.3 from RFC 4493
397 *
398 * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
399 * + Algorithm AES-CMAC +
400 * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
401 * + +
402 * + Input : K ( 128-bit key ) +
403 * + : M ( message to be authenticated ) +
404 * + : len ( length of the message in octets ) +
405 * + Output : T ( message authentication code ) +
406 * + +
407 * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
408 * + Constants: const_Zero is 0x00000000000000000000000000000000 +
409 * + const_Bsize is 16 +
410 * + +
411 * + Variables: K1, K2 for 128-bit subkeys +
412 * + M_i is the i-th block (i=1..ceil(len/const_Bsize)) +
413 * + M_last is the last block xor-ed with K1 or K2 +
414 * + n for number of blocks to be processed +
415 * + r for number of octets of last block +
416 * + flag for denoting if last block is complete or not +
417 * + +
418 * + Step 1. (K1,K2) := Generate_Subkey(K); +
419 * + Step 2. n := ceil(len/const_Bsize); +
420 * + Step 3. if n = 0 +
421 * + then +
422 * + n := 1; +
423 * + flag := false; +
424 * + else +
425 * + if len mod const_Bsize is 0 +
426 * + then flag := true; +
427 * + else flag := false; +
428 * + +
429 * + Step 4. if flag is true +
430 * + then M_last := M_n XOR K1; +
431 * + else M_last := padding(M_n) XOR K2; +
432 * + Step 5. X := const_Zero; +
433 * + Step 6. for i := 1 to n-1 do +
434 * + begin +
435 * + Y := X XOR M_i; +
436 * + X := AES-128(K,Y); +
437 * + end +
438 * + Y := M_last XOR X; +
439 * + T := AES-128(K,Y); +
440 * + Step 7. return T; +
441 * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
442 */
443
444 __forceinline
445 JOB_AES_HMAC *
446 submit_flush_job_aes_cmac(MB_MGR_CMAC_OOO *state, JOB_AES_HMAC *job,
447 const unsigned max_jobs, const int is_submit)
448 {
449 const unsigned lane_scratch_size = 16;
450 unsigned lane, min_len, min_idx;
451 JOB_AES_HMAC *ret_job = NULL;
452 uint8_t *M_last = NULL;
453 unsigned i;
454
455 if (is_submit) {
456 /*
457 * SUBMIT
458 * - get a free lane id
459 */
460 unsigned n = (unsigned) (job->msg_len_to_hash_in_bytes + 15) /
461 AES_BLOCK_SIZE;
462 const unsigned r = (unsigned) job->msg_len_to_hash_in_bytes &
463 (AES_BLOCK_SIZE - 1);
464 const uint64_t *p_key;
465 const uint8_t *job_src =
466 job->src + job->hash_start_src_offset_in_bytes;
467 uint64_t *p_dst;
468 int flag;
469
470 lane = state->unused_lanes & 15;
471 state->unused_lanes >>= 4;
472
473 /* copy job into the lane */
474 state->job_in_lane[lane] = job;
475 state->args.keys[lane] = job->u.CMAC._key_expanded;
476 memset(&state->args.IV[lane], 0, sizeof(state->args.IV[0]));
477
478 M_last = &state->scratch[lane * lane_scratch_size];
479 p_dst = (uint64_t *) M_last;
480
481 if (n == 0) {
482 /* only one partial block */
483 state->init_done[lane] = 1;
484 state->args.in[lane] = M_last;
485 state->lens[lane] = AES_BLOCK_SIZE;
486 n = 1;
487 flag = 0;
488 } else {
489 /* one or more blocks, potentially partial */
490 state->init_done[lane] = 0;
491 state->args.in[lane] = job_src;
492 state->lens[lane] = (n - 1) * AES_BLOCK_SIZE;
493 flag = (r == 0);
494 }
495
496 if (flag) {
497 p_key = job->u.CMAC._skey1;
498 memcpy(M_last, job_src + ((n - 1) * AES_BLOCK_SIZE),
499 AES_BLOCK_SIZE);
500 } else {
501 p_key = job->u.CMAC._skey2;
502 memcpy(M_last, job_src + ((n - 1) * AES_BLOCK_SIZE), r);
503 M_last[r] = 0x80;
504 memset(&M_last[r + 1], 0, AES_BLOCK_SIZE - r - 1);
505 }
506
507 p_dst[0] = p_dst[0] ^ p_key[0];
508 p_dst[1] = p_dst[1] ^ p_key[1];
509
510 /* enough jobs to start processing? */
511 if (state->unused_lanes != 0xf)
512 return NULL;
513 } else {
514 /*
515 * FLUSH
516 * - find 1st non null job
517 */
518 for (lane = 0; lane < max_jobs; lane++)
519 if (state->job_in_lane[lane] != NULL)
520 break;
521 if (lane >= max_jobs)
522 return NULL; /* no not null job */
523 }
524
525 cmac_round:
526 if (is_submit) {
527 /*
528 * SUBMIT
529 * - find min common length to process
530 */
531 min_idx = 0;
532 min_len = state->lens[0];
533
534 for (i = 1; i < max_jobs; i++) {
535 if (min_len > state->lens[i]) {
536 min_idx = i;
537 min_len = state->lens[i];
538 }
539 }
540 } else {
541 /*
542 * FLUSH
543 * - copy good (not null) lane onto empty lanes
544 * - find min common length to process across not null lanes
545 */
546 min_idx = lane;
547 min_len = state->lens[lane];
548
549 for (i = 0; i < max_jobs; i++) {
550 if (i == lane)
551 continue;
552
553 if (state->job_in_lane[i] != NULL) {
554 if (min_len > state->lens[i]) {
555 min_idx = i;
556 min_len = state->lens[i];
557 }
558 } else {
559 state->args.in[i] = state->args.in[lane];
560 state->args.keys[i] = state->args.keys[lane];
561 state->args.IV[i] = state->args.IV[lane];
562 state->lens[i] = UINT16_MAX;
563 state->init_done[i] = state->init_done[lane];
564 }
565 }
566 }
567
568 /* subtract min len from all lanes */
569 for (i = 0; i < max_jobs; i++)
570 state->lens[i] -= min_len;
571
572 /* run the algorythmic code on selected blocks */
573 if (min_len != 0)
574 AES128_CBC_MAC(&state->args, min_len);
575
576 ret_job = state->job_in_lane[min_idx];
577 M_last = &state->scratch[min_idx * lane_scratch_size];
578
579 if (state->init_done[min_idx] == 0) {
580 /*
581 * Finish step 6
582 * Y := M_last XOR X;
583 * T := AES-128(K,Y);
584 */
585 state->init_done[min_idx] = 1;
586 state->args.in[min_idx] = M_last;
587 state->lens[min_idx] = AES_BLOCK_SIZE;
588 goto cmac_round;
589 }
590
591 if (ret_job->auth_tag_output_len_in_bytes == 16)
592 memcpy(ret_job->auth_tag_output, &state->args.IV[min_idx], 16);
593 else
594 memcpy(ret_job->auth_tag_output, &state->args.IV[min_idx], 12);
595
596
597 /* put back processed packet into unused lanes, set job as complete */
598 state->unused_lanes = (state->unused_lanes << 4) | min_idx;
599 ret_job = state->job_in_lane[min_idx];
600 ret_job->status |= STS_COMPLETED_HMAC;
601 state->job_in_lane[min_idx] = NULL;
602 return ret_job;
603 }
604
605 static
606 JOB_AES_HMAC *
607 submit_job_aes_cmac_auth_arch(MB_MGR_CMAC_OOO *state, JOB_AES_HMAC *job)
608 {
609 return submit_flush_job_aes_cmac(state, job, AES_CMAC_MAX_JOBS, 1);
610 }
611
612 static
613 JOB_AES_HMAC *
614 flush_job_aes_cmac_auth_arch(MB_MGR_CMAC_OOO *state)
615 {
616 return submit_flush_job_aes_cmac(state, NULL, AES_CMAC_MAX_JOBS, 0);
617 }
618
619 /* ========================================================================= */
620 /* AES-GCM */
621 /* ========================================================================= */
622
623 #ifndef NO_GCM
624 __forceinline
625 JOB_AES_HMAC *
626 SUBMIT_JOB_AES_GCM_DEC(JOB_AES_HMAC *job)
627 {
628 DECLARE_ALIGNED(struct gcm_context_data ctx, 16);
629
630 if (16 == job->aes_key_len_in_bytes)
631 AES_GCM_DEC_128(job->aes_dec_key_expanded, &ctx, job->dst,
632 job->src +
633 job->cipher_start_src_offset_in_bytes,
634 job->msg_len_to_cipher_in_bytes,
635 job->iv,
636 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
637 job->auth_tag_output,
638 job->auth_tag_output_len_in_bytes);
639 else if (24 == job->aes_key_len_in_bytes)
640 AES_GCM_DEC_192(job->aes_dec_key_expanded, &ctx, job->dst,
641 job->src +
642 job->cipher_start_src_offset_in_bytes,
643 job->msg_len_to_cipher_in_bytes,
644 job->iv,
645 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
646 job->auth_tag_output,
647 job->auth_tag_output_len_in_bytes);
648 else
649 AES_GCM_DEC_256(job->aes_dec_key_expanded, &ctx, job->dst,
650 job->src +
651 job->cipher_start_src_offset_in_bytes,
652 job->msg_len_to_cipher_in_bytes,
653 job->iv,
654 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
655 job->auth_tag_output,
656 job->auth_tag_output_len_in_bytes);
657
658 job->status = STS_COMPLETED;
659 return job;
660 }
661
662 __forceinline
663 JOB_AES_HMAC *
664 SUBMIT_JOB_AES_GCM_ENC(JOB_AES_HMAC *job)
665 {
666 DECLARE_ALIGNED(struct gcm_context_data ctx, 16);
667
668 if (16 == job->aes_key_len_in_bytes)
669 AES_GCM_ENC_128(job->aes_enc_key_expanded, &ctx, job->dst,
670 job->src +
671 job->cipher_start_src_offset_in_bytes,
672 job->msg_len_to_cipher_in_bytes, job->iv,
673 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
674 job->auth_tag_output,
675 job->auth_tag_output_len_in_bytes);
676 else if (24 == job->aes_key_len_in_bytes)
677 AES_GCM_ENC_192(job->aes_enc_key_expanded, &ctx, job->dst,
678 job->src +
679 job->cipher_start_src_offset_in_bytes,
680 job->msg_len_to_cipher_in_bytes, job->iv,
681 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
682 job->auth_tag_output,
683 job->auth_tag_output_len_in_bytes);
684 else
685 AES_GCM_ENC_256(job->aes_enc_key_expanded, &ctx, job->dst,
686 job->src +
687 job->cipher_start_src_offset_in_bytes,
688 job->msg_len_to_cipher_in_bytes, job->iv,
689 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
690 job->auth_tag_output,
691 job->auth_tag_output_len_in_bytes);
692
693 job->status = STS_COMPLETED;
694 return job;
695 }
696 #endif /* !NO_GCM */
697
698 /* ========================================================================= */
699 /* Custom hash / cipher */
700 /* ========================================================================= */
701
702 __forceinline
703 JOB_AES_HMAC *
704 JOB_CUSTOM_CIPHER(JOB_AES_HMAC *job)
705 {
706 if (!(job->status & STS_COMPLETED_AES)) {
707 if (job->cipher_func(job))
708 job->status = STS_INTERNAL_ERROR;
709 else
710 job->status |= STS_COMPLETED_AES;
711 }
712 return job;
713 }
714
715 __forceinline
716 JOB_AES_HMAC *
717 SUBMIT_JOB_CUSTOM_CIPHER(JOB_AES_HMAC *job)
718 {
719 return JOB_CUSTOM_CIPHER(job);
720 }
721
722 __forceinline
723 JOB_AES_HMAC *
724 FLUSH_JOB_CUSTOM_CIPHER(JOB_AES_HMAC *job)
725 {
726 return JOB_CUSTOM_CIPHER(job);
727 }
728
729 __forceinline
730 JOB_AES_HMAC *
731 JOB_CUSTOM_HASH(JOB_AES_HMAC *job)
732 {
733 if (!(job->status & STS_COMPLETED_HMAC)) {
734 if (job->hash_func(job))
735 job->status = STS_INTERNAL_ERROR;
736 else
737 job->status |= STS_COMPLETED_HMAC;
738 }
739 return job;
740 }
741
742 __forceinline
743 JOB_AES_HMAC *
744 SUBMIT_JOB_CUSTOM_HASH(JOB_AES_HMAC *job)
745 {
746 return JOB_CUSTOM_HASH(job);
747 }
748
749 __forceinline
750 JOB_AES_HMAC *
751 FLUSH_JOB_CUSTOM_HASH(JOB_AES_HMAC *job)
752 {
753 return JOB_CUSTOM_HASH(job);
754 }
755
756 /* ========================================================================= */
757 /* DOCSIS AES (AES128 CBC + AES128 CFB) */
758 /* ========================================================================= */
759
760 #define AES_BLOCK_SIZE 16
761
762 /**
763 * @brief Encrypts/decrypts the last partial block for DOCSIS SEC v3.1 BPI
764 *
765 * The last partial block is encrypted/decrypted using AES CFB128.
766 * IV is always the next last ciphered block.
767 *
768 * @note It is assumed that length is bigger than one AES 128 block.
769 *
770 * @param job desriptor of performed crypto operation
771 * @return It always returns value passed in \a job
772 */
773 __forceinline
774 JOB_AES_HMAC *
775 DOCSIS_LAST_BLOCK(JOB_AES_HMAC *job)
776 {
777 const void *iv = NULL;
778 uint64_t offset = 0;
779 uint64_t partial_bytes = 0;
780
781 if (job == NULL)
782 return job;
783
784 IMB_ASSERT((job->cipher_direction == DECRYPT) ||
785 (job->status & STS_COMPLETED_AES));
786
787 partial_bytes = job->msg_len_to_cipher_in_bytes & (AES_BLOCK_SIZE - 1);
788 offset = job->msg_len_to_cipher_in_bytes & (~(AES_BLOCK_SIZE - 1));
789
790 if (!partial_bytes)
791 return job;
792
793 /* in either case IV has to be next last ciphered block */
794 if (job->cipher_direction == ENCRYPT)
795 iv = job->dst + offset - AES_BLOCK_SIZE;
796 else
797 iv = job->src + job->cipher_start_src_offset_in_bytes +
798 offset - AES_BLOCK_SIZE;
799
800 IMB_ASSERT(partial_bytes <= AES_BLOCK_SIZE);
801 AES_CFB_128_ONE(job->dst + offset,
802 job->src + job->cipher_start_src_offset_in_bytes +
803 offset,
804 iv, job->aes_enc_key_expanded, partial_bytes);
805
806 return job;
807 }
808
809 /**
810 * @brief Encrypts/decrypts the first and only partial block for
811 * DOCSIS SEC v3.1 BPI
812 *
813 * The first partial block is encrypted/decrypted using AES CFB128.
814 *
815 * @param job desriptor of performed crypto operation
816 * @return It always returns value passed in \a job
817 */
818 __forceinline
819 JOB_AES_HMAC *
820 DOCSIS_FIRST_BLOCK(JOB_AES_HMAC *job)
821 {
822 IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
823 IMB_ASSERT(job->msg_len_to_cipher_in_bytes <= AES_BLOCK_SIZE);
824 AES_CFB_128_ONE(job->dst,
825 job->src + job->cipher_start_src_offset_in_bytes,
826 job->iv, job->aes_enc_key_expanded,
827 job->msg_len_to_cipher_in_bytes);
828 job->status |= STS_COMPLETED_AES;
829 return job;
830 }
831
832 /* ========================================================================= */
833 /* DES, 3DES and DOCSIS DES (DES CBC + DES CFB) */
834 /* ========================================================================= */
835
836 /**
837 * @brief DOCSIS DES cipher encryption
838 *
839 * @param job desriptor of performed crypto operation
840 * @return It always returns value passed in \a job
841 */
842 __forceinline
843 JOB_AES_HMAC *
844 DOCSIS_DES_ENC(JOB_AES_HMAC *job)
845 {
846 IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
847 docsis_des_enc_basic(job->src + job->cipher_start_src_offset_in_bytes,
848 job->dst,
849 (int) job->msg_len_to_cipher_in_bytes,
850 job->aes_enc_key_expanded,
851 (const uint64_t *)job->iv);
852 job->status |= STS_COMPLETED_AES;
853 return job;
854 }
855
856 /**
857 * @brief DOCSIS DES cipher decryption
858 *
859 * @param job desriptor of performed crypto operation
860 * @return It always returns value passed in \a job
861 */
862 __forceinline
863 JOB_AES_HMAC *
864 DOCSIS_DES_DEC(JOB_AES_HMAC *job)
865 {
866 IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
867 docsis_des_dec_basic(job->src + job->cipher_start_src_offset_in_bytes,
868 job->dst,
869 (int) job->msg_len_to_cipher_in_bytes,
870 job->aes_dec_key_expanded,
871 (const uint64_t *)job->iv);
872 job->status |= STS_COMPLETED_AES;
873 return job;
874 }
875
876 /**
877 * @brief DES cipher encryption
878 *
879 * @param job desriptor of performed crypto operation
880 * @return It always returns value passed in \a job
881 */
882 __forceinline
883 JOB_AES_HMAC *
884 DES_CBC_ENC(JOB_AES_HMAC *job)
885 {
886 IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
887 des_enc_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
888 job->dst,
889 job->msg_len_to_cipher_in_bytes &
890 (~(DES_BLOCK_SIZE - 1)),
891 job->aes_enc_key_expanded, (const uint64_t *)job->iv);
892 job->status |= STS_COMPLETED_AES;
893 return job;
894 }
895
896 /**
897 * @brief DES cipher decryption
898 *
899 * @param job desriptor of performed crypto operation
900 * @return It always returns value passed in \a job
901 */
902 __forceinline
903 JOB_AES_HMAC *
904 DES_CBC_DEC(JOB_AES_HMAC *job)
905 {
906 IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
907 des_dec_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
908 job->dst,
909 job->msg_len_to_cipher_in_bytes &
910 (~(DES_BLOCK_SIZE - 1)),
911 job->aes_dec_key_expanded, (const uint64_t *)job->iv);
912 job->status |= STS_COMPLETED_AES;
913 return job;
914 }
915
916 /**
917 * @brief 3DES cipher encryption
918 *
919 * @param job desriptor of performed crypto operation
920 * @return It always returns value passed in \a job
921 */
922 __forceinline
923 JOB_AES_HMAC *
924 DES3_CBC_ENC(JOB_AES_HMAC *job)
925 {
926 const void * const *ks_ptr =
927 (const void * const *)job->aes_enc_key_expanded;
928
929 IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
930 des3_enc_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
931 job->dst,
932 job->msg_len_to_cipher_in_bytes &
933 (~(DES_BLOCK_SIZE - 1)),
934 ks_ptr[0], ks_ptr[1], ks_ptr[2],
935 (const uint64_t *)job->iv);
936 job->status |= STS_COMPLETED_AES;
937 return job;
938 }
939
940 /**
941 * @brief 3DES cipher decryption
942 *
943 * @param job desriptor of performed crypto operation
944 * @return It always returns value passed in \a job
945 */
946 __forceinline
947 JOB_AES_HMAC *
948 DES3_CBC_DEC(JOB_AES_HMAC *job)
949 {
950 const void * const *ks_ptr =
951 (const void * const *)job->aes_dec_key_expanded;
952
953 IMB_ASSERT(!(job->status & STS_COMPLETED_AES));
954 des3_dec_cbc_basic(job->src + job->cipher_start_src_offset_in_bytes,
955 job->dst,
956 job->msg_len_to_cipher_in_bytes &
957 (~(DES_BLOCK_SIZE - 1)),
958 ks_ptr[0], ks_ptr[1], ks_ptr[2],
959 (const uint64_t *)job->iv);
960 job->status |= STS_COMPLETED_AES;
961 return job;
962 }
963
964 /* ========================================================================= */
965 /* Cipher submit & flush functions */
966 /* ========================================================================= */
967
968 __forceinline
969 JOB_AES_HMAC *
970 SUBMIT_JOB_AES_ENC(MB_MGR *state, JOB_AES_HMAC *job)
971 {
972 if (CBC == job->cipher_mode) {
973 if (16 == job->aes_key_len_in_bytes) {
974 return SUBMIT_JOB_AES128_ENC(&state->aes128_ooo, job);
975 } else if (24 == job->aes_key_len_in_bytes) {
976 return SUBMIT_JOB_AES192_ENC(&state->aes192_ooo, job);
977 } else { /* assume 32 */
978 return SUBMIT_JOB_AES256_ENC(&state->aes256_ooo, job);
979 }
980 } else if (CNTR == job->cipher_mode) {
981 if (16 == job->aes_key_len_in_bytes) {
982 return SUBMIT_JOB_AES128_CNTR(job);
983 } else if (24 == job->aes_key_len_in_bytes) {
984 return SUBMIT_JOB_AES192_CNTR(job);
985 } else { /* assume 32 */
986 return SUBMIT_JOB_AES256_CNTR(job);
987 }
988 } else if (DOCSIS_SEC_BPI == job->cipher_mode) {
989 if (job->msg_len_to_cipher_in_bytes >= AES_BLOCK_SIZE) {
990 JOB_AES_HMAC *tmp;
991
992 tmp = SUBMIT_JOB_AES128_ENC(&state->docsis_sec_ooo,
993 job);
994 return DOCSIS_LAST_BLOCK(tmp);
995 } else
996 return DOCSIS_FIRST_BLOCK(job);
997 #ifndef NO_GCM
998 } else if (GCM == job->cipher_mode) {
999 return SUBMIT_JOB_AES_GCM_ENC(job);
1000 #endif /* NO_GCM */
1001 } else if (CUSTOM_CIPHER == job->cipher_mode) {
1002 return SUBMIT_JOB_CUSTOM_CIPHER(job);
1003 } else if (DES == job->cipher_mode) {
1004 #ifdef SUBMIT_JOB_DES_CBC_ENC
1005 return SUBMIT_JOB_DES_CBC_ENC(&state->des_enc_ooo, job);
1006 #else
1007 return DES_CBC_ENC(job);
1008 #endif /* SUBMIT_JOB_DES_CBC_ENC */
1009 } else if (DOCSIS_DES == job->cipher_mode) {
1010 #ifdef SUBMIT_JOB_DOCSIS_DES_ENC
1011 return SUBMIT_JOB_DOCSIS_DES_ENC(&state->docsis_des_enc_ooo,
1012 job);
1013 #else
1014 return DOCSIS_DES_ENC(job);
1015 #endif /* SUBMIT_JOB_DOCSIS_DES_ENC */
1016 } else if (DES3 == job->cipher_mode) {
1017 #ifdef SUBMIT_JOB_3DES_CBC_ENC
1018 return SUBMIT_JOB_3DES_CBC_ENC(&state->des3_enc_ooo, job);
1019 #else
1020 return DES3_CBC_ENC(job);
1021 #endif
1022 } else { /* assume NUL_CIPHER or CCM */
1023 job->status |= STS_COMPLETED_AES;
1024 return job;
1025 }
1026 }
1027
1028 __forceinline
1029 JOB_AES_HMAC *
1030 FLUSH_JOB_AES_ENC(MB_MGR *state, JOB_AES_HMAC *job)
1031 {
1032 if (CBC == job->cipher_mode) {
1033 if (16 == job->aes_key_len_in_bytes) {
1034 return FLUSH_JOB_AES128_ENC(&state->aes128_ooo);
1035 } else if (24 == job->aes_key_len_in_bytes) {
1036 return FLUSH_JOB_AES192_ENC(&state->aes192_ooo);
1037 } else { /* assume 32 */
1038 return FLUSH_JOB_AES256_ENC(&state->aes256_ooo);
1039 }
1040 } else if (DOCSIS_SEC_BPI == job->cipher_mode) {
1041 JOB_AES_HMAC *tmp;
1042
1043 tmp = FLUSH_JOB_AES128_ENC(&state->docsis_sec_ooo);
1044 return DOCSIS_LAST_BLOCK(tmp);
1045 #ifdef FLUSH_JOB_DES_CBC_ENC
1046 } else if (DES == job->cipher_mode) {
1047 return FLUSH_JOB_DES_CBC_ENC(&state->des_enc_ooo);
1048 #endif /* FLUSH_JOB_DES_CBC_ENC */
1049 #ifdef FLUSH_JOB_3DES_CBC_ENC
1050 } else if (DES3 == job->cipher_mode) {
1051 return FLUSH_JOB_3DES_CBC_ENC(&state->des3_enc_ooo);
1052 #endif /* FLUSH_JOB_3DES_CBC_ENC */
1053 #ifdef FLUSH_JOB_DOCSIS_DES_ENC
1054 } else if (DOCSIS_DES == job->cipher_mode) {
1055 return FLUSH_JOB_DOCSIS_DES_ENC(&state->docsis_des_enc_ooo);
1056 #endif /* FLUSH_JOB_DOCSIS_DES_ENC */
1057 } else if (CUSTOM_CIPHER == job->cipher_mode) {
1058 return FLUSH_JOB_CUSTOM_CIPHER(job);
1059 } else { /* assume CNTR, CCM or NULL_CIPHER */
1060 return NULL;
1061 }
1062 }
1063
1064 __forceinline
1065 JOB_AES_HMAC *
1066 SUBMIT_JOB_AES_DEC(MB_MGR *state, JOB_AES_HMAC *job)
1067 {
1068 if (CBC == job->cipher_mode) {
1069 if (16 == job->aes_key_len_in_bytes) {
1070 return SUBMIT_JOB_AES128_DEC(job);
1071 } else if (24 == job->aes_key_len_in_bytes) {
1072 return SUBMIT_JOB_AES192_DEC(job);
1073 } else { /* assume 32 */
1074 return SUBMIT_JOB_AES256_DEC(job);
1075 }
1076 } else if (CNTR == job->cipher_mode) {
1077 if (16 == job->aes_key_len_in_bytes) {
1078 return SUBMIT_JOB_AES128_CNTR(job);
1079 } else if (24 == job->aes_key_len_in_bytes) {
1080 return SUBMIT_JOB_AES192_CNTR(job);
1081 } else { /* assume 32 */
1082 return SUBMIT_JOB_AES256_CNTR(job);
1083 }
1084 } else if (DOCSIS_SEC_BPI == job->cipher_mode) {
1085 if (job->msg_len_to_cipher_in_bytes >= AES_BLOCK_SIZE) {
1086 DOCSIS_LAST_BLOCK(job);
1087 return SUBMIT_JOB_AES128_DEC(job);
1088 } else {
1089 return DOCSIS_FIRST_BLOCK(job);
1090 }
1091 #ifndef NO_GCM
1092 } else if (GCM == job->cipher_mode) {
1093 return SUBMIT_JOB_AES_GCM_DEC(job);
1094 #endif /* NO_GCM */
1095 } else if (DES == job->cipher_mode) {
1096 #ifdef SUBMIT_JOB_DES_CBC_DEC
1097 return SUBMIT_JOB_DES_CBC_DEC(&state->des_dec_ooo, job);
1098 #else
1099 (void) state;
1100 return DES_CBC_DEC(job);
1101 #endif /* SUBMIT_JOB_DES_CBC_DEC */
1102 } else if (DOCSIS_DES == job->cipher_mode) {
1103 #ifdef SUBMIT_JOB_DOCSIS_DES_DEC
1104 return SUBMIT_JOB_DOCSIS_DES_DEC(&state->docsis_des_dec_ooo,
1105 job);
1106 #else
1107 return DOCSIS_DES_DEC(job);
1108 #endif /* SUBMIT_JOB_DOCSIS_DES_DEC */
1109 } else if (DES3 == job->cipher_mode) {
1110 #ifdef SUBMIT_JOB_3DES_CBC_DEC
1111 return SUBMIT_JOB_3DES_CBC_DEC(&state->des3_dec_ooo, job);
1112 #else
1113 return DES3_CBC_DEC(job);
1114 #endif
1115 } else if (CUSTOM_CIPHER == job->cipher_mode) {
1116 return SUBMIT_JOB_CUSTOM_CIPHER(job);
1117 } else {
1118 /* assume NULL cipher or CCM */
1119 job->status |= STS_COMPLETED_AES;
1120 return job;
1121 }
1122 }
1123
1124 __forceinline
1125 JOB_AES_HMAC *
1126 FLUSH_JOB_AES_DEC(MB_MGR *state, JOB_AES_HMAC *job)
1127 {
1128 #ifdef FLUSH_JOB_DES_CBC_DEC
1129 if (DES == job->cipher_mode)
1130 return FLUSH_JOB_DES_CBC_DEC(&state->des_dec_ooo);
1131 #endif /* FLUSH_JOB_DES_CBC_DEC */
1132 #ifdef FLUSH_JOB_3DES_CBC_DEC
1133 if (DES3 == job->cipher_mode)
1134 return FLUSH_JOB_3DES_CBC_DEC(&state->des3_dec_ooo);
1135 #endif /* FLUSH_JOB_3DES_CBC_DEC */
1136 #ifdef FLUSH_JOB_DOCSIS_DES_DEC
1137 if (DOCSIS_DES == job->cipher_mode)
1138 return FLUSH_JOB_DOCSIS_DES_DEC(&state->docsis_des_dec_ooo);
1139 #endif /* FLUSH_JOB_DOCSIS_DES_DEC */
1140 (void) state;
1141 return SUBMIT_JOB_AES_DEC(state, job);
1142 }
1143
1144 /* ========================================================================= */
1145 /* Hash submit & flush functions */
1146 /* ========================================================================= */
1147
1148 __forceinline
1149 JOB_AES_HMAC *
1150 SUBMIT_JOB_HASH(MB_MGR *state, JOB_AES_HMAC *job)
1151 {
1152 #ifdef VERBOSE
1153 printf("--------Enter SUBMIT_JOB_HASH --------------\n");
1154 #endif
1155 switch (job->hash_alg) {
1156 case SHA1:
1157 #ifdef HASH_USE_SHAEXT
1158 if (state->features & IMB_FEATURE_SHANI)
1159 return SUBMIT_JOB_HMAC_NI(&state->hmac_sha_1_ooo, job);
1160 #endif
1161 return SUBMIT_JOB_HMAC(&state->hmac_sha_1_ooo, job);
1162 case SHA_224:
1163 #ifdef HASH_USE_SHAEXT
1164 if (state->features & IMB_FEATURE_SHANI)
1165 return SUBMIT_JOB_HMAC_SHA_224_NI
1166 (&state->hmac_sha_224_ooo, job);
1167 #endif
1168 return SUBMIT_JOB_HMAC_SHA_224(&state->hmac_sha_224_ooo, job);
1169 case SHA_256:
1170 #ifdef HASH_USE_SHAEXT
1171 if (state->features & IMB_FEATURE_SHANI)
1172 return SUBMIT_JOB_HMAC_SHA_256_NI
1173 (&state->hmac_sha_256_ooo, job);
1174 #endif
1175 return SUBMIT_JOB_HMAC_SHA_256(&state->hmac_sha_256_ooo, job);
1176 case SHA_384:
1177 return SUBMIT_JOB_HMAC_SHA_384(&state->hmac_sha_384_ooo, job);
1178 case SHA_512:
1179 return SUBMIT_JOB_HMAC_SHA_512(&state->hmac_sha_512_ooo, job);
1180 case AES_XCBC:
1181 return SUBMIT_JOB_AES_XCBC(&state->aes_xcbc_ooo, job);
1182 case MD5:
1183 return SUBMIT_JOB_HMAC_MD5(&state->hmac_md5_ooo, job);
1184 case CUSTOM_HASH:
1185 return SUBMIT_JOB_CUSTOM_HASH(job);
1186 case AES_CCM:
1187 return SUBMIT_JOB_AES_CCM_AUTH(&state->aes_ccm_ooo, job);
1188 case AES_CMAC:
1189 return SUBMIT_JOB_AES_CMAC_AUTH(&state->aes_cmac_ooo, job);
1190 default: /* assume NULL_HASH */
1191 job->status |= STS_COMPLETED_HMAC;
1192 return job;
1193 }
1194 }
1195
1196 __forceinline
1197 JOB_AES_HMAC *
1198 FLUSH_JOB_HASH(MB_MGR *state, JOB_AES_HMAC *job)
1199 {
1200 switch (job->hash_alg) {
1201 case SHA1:
1202 #ifdef HASH_USE_SHAEXT
1203 if (state->features & IMB_FEATURE_SHANI)
1204 return FLUSH_JOB_HMAC_NI(&state->hmac_sha_1_ooo);
1205 #endif
1206 return FLUSH_JOB_HMAC(&state->hmac_sha_1_ooo);
1207 case SHA_224:
1208 #ifdef HASH_USE_SHAEXT
1209 if (state->features & IMB_FEATURE_SHANI)
1210 return FLUSH_JOB_HMAC_SHA_224_NI
1211 (&state->hmac_sha_224_ooo);
1212 #endif
1213 return FLUSH_JOB_HMAC_SHA_224(&state->hmac_sha_224_ooo);
1214 case SHA_256:
1215 #ifdef HASH_USE_SHAEXT
1216 if (state->features & IMB_FEATURE_SHANI)
1217 return FLUSH_JOB_HMAC_SHA_256_NI
1218 (&state->hmac_sha_256_ooo);
1219 #endif
1220 return FLUSH_JOB_HMAC_SHA_256(&state->hmac_sha_256_ooo);
1221 case SHA_384:
1222 return FLUSH_JOB_HMAC_SHA_384(&state->hmac_sha_384_ooo);
1223 case SHA_512:
1224 return FLUSH_JOB_HMAC_SHA_512(&state->hmac_sha_512_ooo);
1225 case AES_XCBC:
1226 return FLUSH_JOB_AES_XCBC(&state->aes_xcbc_ooo);
1227 case MD5:
1228 return FLUSH_JOB_HMAC_MD5(&state->hmac_md5_ooo);
1229 case CUSTOM_HASH:
1230 return FLUSH_JOB_CUSTOM_HASH(job);
1231 case AES_CCM:
1232 return FLUSH_JOB_AES_CCM_AUTH(&state->aes_ccm_ooo);
1233 case AES_CMAC:
1234 return FLUSH_JOB_AES_CMAC_AUTH(&state->aes_cmac_ooo);
1235 default: /* assume NULL_HASH */
1236 if (!(job->status & STS_COMPLETED_HMAC)) {
1237 job->status |= STS_COMPLETED_HMAC;
1238 return job;
1239 }
1240 /* if HMAC is complete then return NULL */
1241 return NULL;
1242 }
1243 }
1244
1245
1246 /* ========================================================================= */
1247 /* Job submit & flush functions */
1248 /* ========================================================================= */
1249
1250 #ifdef DEBUG
1251 #ifdef _WIN32
1252 #define INVALID_PRN(_fmt, ...) \
1253 fprintf(stderr, "%s():%d: " _fmt, __FUNCTION__, __LINE__, __VA_ARGS__)
1254 #else
1255 #define INVALID_PRN(_fmt, ...) \
1256 fprintf(stderr, "%s():%d: " _fmt, __func__, __LINE__, __VA_ARGS__)
1257 #endif
1258 #else
1259 #define INVALID_PRN(_fmt, ...)
1260 #endif
1261
1262 __forceinline int
1263 is_job_invalid(const JOB_AES_HMAC *job)
1264 {
1265 const uint64_t auth_tag_len_max[] = {
1266 0, /* INVALID selection */
1267 12, /* SHA1 */
1268 14, /* SHA_224 */
1269 16, /* SHA_256 */
1270 24, /* SHA_384 */
1271 32, /* SHA_512 */
1272 12, /* AES_XCBC */
1273 12, /* MD5 */
1274 0, /* NULL_HASH */
1275 16, /* AES_GMAC */
1276 0, /* CUSTOM HASH */
1277 0, /* AES_CCM */
1278 16, /* AES_CMAC */
1279 };
1280
1281 switch (job->cipher_mode) {
1282 case CBC:
1283 if (job->cipher_direction == ENCRYPT &&
1284 job->aes_enc_key_expanded == NULL) {
1285 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1286 return 1;
1287 }
1288 if (job->cipher_direction == DECRYPT &&
1289 job->aes_dec_key_expanded == NULL) {
1290 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1291 return 1;
1292 }
1293 if (job->aes_key_len_in_bytes != UINT64_C(16) &&
1294 job->aes_key_len_in_bytes != UINT64_C(24) &&
1295 job->aes_key_len_in_bytes != UINT64_C(32)) {
1296 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1297 return 1;
1298 }
1299 if (job->msg_len_to_cipher_in_bytes == 0) {
1300 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1301 return 1;
1302 }
1303 if (job->msg_len_to_cipher_in_bytes & UINT64_C(15)) {
1304 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1305 return 1;
1306 }
1307 if (job->iv_len_in_bytes != UINT64_C(16)) {
1308 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1309 return 1;
1310 }
1311 break;
1312 case CNTR:
1313 if (job->aes_enc_key_expanded == NULL) {
1314 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1315 return 1;
1316 }
1317 if (job->aes_key_len_in_bytes != UINT64_C(16) &&
1318 job->aes_key_len_in_bytes != UINT64_C(24) &&
1319 job->aes_key_len_in_bytes != UINT64_C(32)) {
1320 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1321 return 1;
1322 }
1323 if (job->iv_len_in_bytes != UINT64_C(16) &&
1324 job->iv_len_in_bytes != UINT64_C(12)) {
1325 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1326 return 1;
1327 }
1328 if (job->msg_len_to_cipher_in_bytes == 0) {
1329 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1330 return 1;
1331 }
1332 break;
1333 case NULL_CIPHER:
1334 /* NULL_CIPHER only allowed in HASH_CIPHER */
1335 if (job->chain_order != HASH_CIPHER)
1336 return 1;
1337 /* XXX: not copy src to dst */
1338 break;
1339 case DOCSIS_SEC_BPI:
1340 if (job->aes_enc_key_expanded == NULL) {
1341 /* it has to be set regardless of direction (AES-CFB) */
1342 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1343 return 1;
1344 }
1345 if (job->cipher_direction == DECRYPT &&
1346 job->aes_dec_key_expanded == NULL) {
1347 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1348 return 1;
1349 }
1350 if (job->aes_key_len_in_bytes != UINT64_C(16)) {
1351 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1352 return 1;
1353 }
1354 if (job->iv_len_in_bytes != UINT64_C(16)) {
1355 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1356 return 1;
1357 }
1358 if (job->msg_len_to_cipher_in_bytes == 0) {
1359 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1360 return 1;
1361 }
1362 break;
1363 #ifndef NO_GCM
1364 case GCM:
1365 /* Same key structure used for encrypt and decrypt */
1366 if (job->cipher_direction == ENCRYPT &&
1367 job->aes_enc_key_expanded == NULL) {
1368 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1369 return 1;
1370 }
1371 if (job->cipher_direction == DECRYPT &&
1372 job->aes_dec_key_expanded == NULL) {
1373 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1374 return 1;
1375 }
1376 if (job->aes_key_len_in_bytes != UINT64_C(16) &&
1377 job->aes_key_len_in_bytes != UINT64_C(24) &&
1378 job->aes_key_len_in_bytes != UINT64_C(32)) {
1379 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1380 return 1;
1381 }
1382 if (job->iv_len_in_bytes != UINT64_C(12)) {
1383 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1384 return 1;
1385 }
1386 if (job->hash_alg != AES_GMAC) {
1387 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1388 return 1;
1389 }
1390 if (job->msg_len_to_cipher_in_bytes == 0) {
1391 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1392 return 1;
1393 }
1394 break;
1395 #endif /* !NO_GCM */
1396 case CUSTOM_CIPHER:
1397 /* no checks here */
1398 if (job->cipher_func == NULL) {
1399 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1400 return 1;
1401 }
1402 break;
1403 case DES:
1404 if (job->cipher_direction == ENCRYPT &&
1405 job->aes_enc_key_expanded == NULL) {
1406 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1407 return 1;
1408 }
1409 if (job->cipher_direction == DECRYPT &&
1410 job->aes_dec_key_expanded == NULL) {
1411 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1412 return 1;
1413 }
1414 if (job->aes_key_len_in_bytes != UINT64_C(8)) {
1415 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1416 return 1;
1417 }
1418 if (job->msg_len_to_cipher_in_bytes == 0) {
1419 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1420 return 1;
1421 }
1422 if (job->msg_len_to_cipher_in_bytes & UINT64_C(7)) {
1423 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1424 return 1;
1425 }
1426 if (job->iv_len_in_bytes != UINT64_C(8)) {
1427 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1428 return 1;
1429 }
1430 break;
1431 case DOCSIS_DES:
1432 if (job->cipher_direction == ENCRYPT &&
1433 job->aes_enc_key_expanded == NULL) {
1434 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1435 return 1;
1436 }
1437 if (job->cipher_direction == DECRYPT &&
1438 job->aes_dec_key_expanded == NULL) {
1439 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1440 return 1;
1441 }
1442 if (job->aes_key_len_in_bytes != UINT64_C(8)) {
1443 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1444 return 1;
1445 }
1446 if (job->msg_len_to_cipher_in_bytes == 0) {
1447 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1448 return 1;
1449 }
1450 if (job->iv_len_in_bytes != UINT64_C(8)) {
1451 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1452 return 1;
1453 }
1454 break;
1455 case CCM:
1456 if (job->aes_enc_key_expanded == NULL) {
1457 /* AES-CTR and CBC-MAC use only encryption keys */
1458 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1459 return 1;
1460 }
1461 /* currently only AES-CCM-128 is only supported */
1462 if (job->aes_key_len_in_bytes != UINT64_C(16)) {
1463 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1464 return 1;
1465 }
1466 /*
1467 * From RFC3610:
1468 * Nonce length = 15 - L
1469 * Valid L values are: 2 to 8
1470 * Then valid nonce lengths 13 to 7 (inclusive).
1471 */
1472 if (job->iv_len_in_bytes > UINT64_C(13) ||
1473 job->iv_len_in_bytes < UINT64_C(7)) {
1474 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1475 return 1;
1476 }
1477 if (job->msg_len_to_cipher_in_bytes == 0) {
1478 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1479 return 1;
1480 }
1481 if (job->hash_alg != AES_CCM) {
1482 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1483 return 1;
1484 }
1485 break;
1486 case DES3:
1487 if (job->aes_key_len_in_bytes != UINT64_C(8)) {
1488 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1489 return 1;
1490 }
1491 if (job->msg_len_to_cipher_in_bytes == 0) {
1492 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1493 return 1;
1494 }
1495 if (job->msg_len_to_cipher_in_bytes & UINT64_C(7)) {
1496 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1497 return 1;
1498 }
1499 if (job->iv_len_in_bytes != UINT64_C(8)) {
1500 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1501 return 1;
1502 }
1503 if (job->cipher_direction == ENCRYPT) {
1504 const void * const *ks_ptr =
1505 (const void * const *)job->aes_enc_key_expanded;
1506
1507 if (ks_ptr == NULL) {
1508 INVALID_PRN("cipher_mode:%d\n",
1509 job->cipher_mode);
1510 return 1;
1511 }
1512 if (ks_ptr[0] == NULL || ks_ptr[1] == NULL ||
1513 ks_ptr[2] == NULL) {
1514 INVALID_PRN("cipher_mode:%d\n",
1515 job->cipher_mode);
1516 return 1;
1517 }
1518 } else {
1519 const void * const *ks_ptr =
1520 (const void * const *)job->aes_dec_key_expanded;
1521
1522 if (ks_ptr == NULL) {
1523 INVALID_PRN("cipher_mode:%d\n",
1524 job->cipher_mode);
1525 return 1;
1526 }
1527 if (ks_ptr[0] == NULL || ks_ptr[1] == NULL ||
1528 ks_ptr[2] == NULL) {
1529 INVALID_PRN("cipher_mode:%d\n",
1530 job->cipher_mode);
1531 return 1;
1532 }
1533 }
1534 break;
1535 default:
1536 INVALID_PRN("cipher_mode:%d\n", job->cipher_mode);
1537 return 1;
1538 }
1539
1540 switch (job->hash_alg) {
1541 case SHA1:
1542 case AES_XCBC:
1543 case MD5:
1544 case SHA_224:
1545 case SHA_256:
1546 case SHA_384:
1547 case SHA_512:
1548 if (job->auth_tag_output_len_in_bytes !=
1549 auth_tag_len_max[job->hash_alg]) {
1550 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1551 return 1;
1552 }
1553 if (job->msg_len_to_hash_in_bytes == 0) {
1554 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1555 return 1;
1556 }
1557 if (job->auth_tag_output == NULL) {
1558 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1559 return 1;
1560 }
1561 break;
1562 case NULL_HASH:
1563 break;
1564 #ifndef NO_GCM
1565 case AES_GMAC:
1566 if (job->auth_tag_output_len_in_bytes != UINT64_C(8) &&
1567 job->auth_tag_output_len_in_bytes != UINT64_C(12) &&
1568 job->auth_tag_output_len_in_bytes != UINT64_C(16)) {
1569 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1570 return 1;
1571 }
1572 if (job->cipher_mode != GCM) {
1573 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1574 return 1;
1575 }
1576 if (job->auth_tag_output == NULL) {
1577 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1578 return 1;
1579 }
1580 /*
1581 * msg_len_to_hash_in_bytes not checked against zero.
1582 * It is not used for AES-GCM & GMAC - see
1583 * SUBMIT_JOB_AES_GCM_ENC and SUBMIT_JOB_AES_GCM_DEC functions.
1584 */
1585 break;
1586 #endif /* !NO_GCM */
1587 case CUSTOM_HASH:
1588 if (job->hash_func == NULL) {
1589 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1590 return 1;
1591 }
1592 break;
1593 case AES_CCM:
1594 if (job->u.CCM.aad_len_in_bytes > 46) {
1595 /* 3 x AES_BLOCK - 2 bytes for AAD len */
1596 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1597 return 1;
1598 }
1599 if ((job->u.CCM.aad_len_in_bytes > 0) &&
1600 (job->u.CCM.aad == NULL)) {
1601 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1602 return 1;
1603 }
1604 /* M can be any even number from 4 to 16 */
1605 if (job->auth_tag_output_len_in_bytes < UINT64_C(4) ||
1606 job->auth_tag_output_len_in_bytes > UINT64_C(16) ||
1607 ((job->auth_tag_output_len_in_bytes & 1) != 0)) {
1608 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1609 return 1;
1610 }
1611 if (job->cipher_mode != CCM) {
1612 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1613 return 1;
1614 }
1615 /*
1616 * AES-CCM allows for only one message for
1617 * cipher and uthentication.
1618 * AAD can be used to extend authentication over
1619 * clear text fields.
1620 */
1621 if (job->msg_len_to_cipher_in_bytes !=
1622 job->msg_len_to_hash_in_bytes) {
1623 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1624 return 1;
1625 }
1626 if (job->cipher_start_src_offset_in_bytes !=
1627 job->hash_start_src_offset_in_bytes) {
1628 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1629 return 1;
1630 }
1631 break;
1632 case AES_CMAC:
1633 if ((job->u.CMAC._key_expanded == NULL) ||
1634 (job->u.CMAC._skey1 == NULL) ||
1635 (job->u.CMAC._skey2 == NULL)) {
1636 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1637 return 1;
1638 }
1639 /*
1640 * T is 128 bits but 96 bits is also allowed due to
1641 * IPsec use case (RFC 4494)
1642 */
1643 if (job->auth_tag_output_len_in_bytes != UINT64_C(16) &&
1644 job->auth_tag_output_len_in_bytes != UINT64_C(12)) {
1645 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1646 return 1;
1647 }
1648 if (job->auth_tag_output == NULL) {
1649 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1650 return 1;
1651 }
1652 break;
1653 default:
1654 INVALID_PRN("hash_alg:%d\n", job->hash_alg);
1655 return 1;
1656 }
1657
1658 switch (job->chain_order) {
1659 case CIPHER_HASH:
1660 if (job->cipher_direction != ENCRYPT) {
1661 INVALID_PRN("chain_order:%d\n", job->chain_order);
1662 return 1;
1663 }
1664 break;
1665 case HASH_CIPHER:
1666 if (job->cipher_mode != NULL_CIPHER) {
1667 if (job->cipher_direction != DECRYPT) {
1668 INVALID_PRN("chain_order:%d\n",
1669 job->chain_order);
1670 return 1;
1671 }
1672 }
1673 break;
1674 default:
1675 INVALID_PRN("chain_order:%d\n", job->chain_order);
1676 return 1;
1677 }
1678
1679 return 0;
1680 }
1681
1682 __forceinline
1683 JOB_AES_HMAC *submit_new_job(MB_MGR *state, JOB_AES_HMAC *job)
1684 {
1685 if (job->chain_order == CIPHER_HASH) {
1686 /* assume job->cipher_direction == ENCRYPT */
1687 job = SUBMIT_JOB_AES_ENC(state, job);
1688 if (job) {
1689 job = SUBMIT_JOB_HASH(state, job);
1690 if (job && (job->chain_order == HASH_CIPHER))
1691 SUBMIT_JOB_AES_DEC(state, job);
1692 } /* end if job */
1693 } else { /* job->chain_order == HASH_CIPHER */
1694 /* assume job->cipher_direction == DECRYPT */
1695 job = SUBMIT_JOB_HASH(state, job);
1696 if (job && (job->chain_order == HASH_CIPHER))
1697 SUBMIT_JOB_AES_DEC(state, job);
1698 }
1699 return job;
1700 }
1701
1702 __forceinline
1703 void complete_job(MB_MGR *state, JOB_AES_HMAC *job)
1704 {
1705 JOB_AES_HMAC *tmp = NULL;
1706
1707 while (job->status < STS_COMPLETED) {
1708 if (job->chain_order == CIPHER_HASH) {
1709 /* assume job->cipher_direction == ENCRYPT */
1710 tmp = FLUSH_JOB_AES_ENC(state, job);
1711 if (tmp)
1712 tmp = SUBMIT_JOB_HASH(state, tmp);
1713 else
1714 tmp = FLUSH_JOB_HASH(state, job);
1715 if (tmp && (tmp->chain_order == HASH_CIPHER))
1716 SUBMIT_JOB_AES_DEC(state, tmp);
1717 } else { /* job->chain_order == HASH_CIPHER */
1718 /* assume job->cipher_direction == DECRYPT */
1719 tmp = FLUSH_JOB_HASH(state, job);
1720 if (tmp == NULL)
1721 tmp = FLUSH_JOB_AES_DEC(state, job);
1722 else
1723 if (tmp->chain_order == HASH_CIPHER)
1724 SUBMIT_JOB_AES_DEC(state, tmp);
1725 }
1726 }
1727 }
1728
1729 __forceinline
1730 JOB_AES_HMAC *
1731 submit_job_and_check(MB_MGR *state, const int run_check)
1732 {
1733 JOB_AES_HMAC *job = NULL;
1734 #ifndef LINUX
1735 DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
1736
1737 SAVE_XMMS(xmm_save);
1738 #endif
1739
1740 job = JOBS(state, state->next_job);
1741
1742 if (run_check) {
1743 if (is_job_invalid(job)) {
1744 job->status = STS_INVALID_ARGS;
1745 } else {
1746 job->status = STS_BEING_PROCESSED;
1747 job = submit_new_job(state, job);
1748 }
1749 } else {
1750 job->status = STS_BEING_PROCESSED;
1751 job = submit_new_job(state, job);
1752 }
1753
1754 if (state->earliest_job < 0) {
1755 /* state was previously empty */
1756 state->earliest_job = state->next_job;
1757 ADV_JOBS(&state->next_job);
1758 #ifndef LINUX
1759 RESTORE_XMMS(xmm_save);
1760 #endif
1761 return NULL; /* if we were empty, nothing to return */
1762 }
1763
1764 ADV_JOBS(&state->next_job);
1765
1766 if (state->earliest_job == state->next_job) {
1767 /* Full */
1768 job = JOBS(state, state->earliest_job);
1769 complete_job(state, job);
1770 ADV_JOBS(&state->earliest_job);
1771 #ifndef LINUX
1772 RESTORE_XMMS(xmm_save);
1773 #endif
1774 return job;
1775 }
1776
1777 /* not full */
1778 #ifndef LINUX
1779 RESTORE_XMMS(xmm_save);
1780 #endif
1781 job = JOBS(state, state->earliest_job);
1782 if (job->status < STS_COMPLETED)
1783 return NULL;
1784
1785 ADV_JOBS(&state->earliest_job);
1786 return job;
1787 }
1788
1789 IMB_DLL_EXPORT
1790 JOB_AES_HMAC *
1791 SUBMIT_JOB(MB_MGR *state)
1792 {
1793 return submit_job_and_check(state, 1);
1794 }
1795
1796 IMB_DLL_EXPORT
1797 JOB_AES_HMAC *
1798 SUBMIT_JOB_NOCHECK(MB_MGR *state)
1799 {
1800 return submit_job_and_check(state, 0);
1801 }
1802
1803 IMB_DLL_EXPORT
1804 JOB_AES_HMAC *
1805 FLUSH_JOB(MB_MGR *state)
1806 {
1807 JOB_AES_HMAC *job;
1808 #ifndef LINUX
1809 DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
1810 #endif
1811
1812 if (state->earliest_job < 0)
1813 return NULL; /* empty */
1814
1815 #ifndef LINUX
1816 SAVE_XMMS(xmm_save);
1817 #endif
1818 job = JOBS(state, state->earliest_job);
1819 complete_job(state, job);
1820
1821 ADV_JOBS(&state->earliest_job);
1822
1823 if (state->earliest_job == state->next_job)
1824 state->earliest_job = -1; /* becomes empty */
1825
1826 #ifndef LINUX
1827 RESTORE_XMMS(xmm_save);
1828 #endif
1829 return job;
1830 }
1831
1832 /* ========================================================================= */
1833 /* ========================================================================= */
1834
1835 IMB_DLL_EXPORT
1836 uint32_t
1837 QUEUE_SIZE(MB_MGR *state)
1838 {
1839 int a, b;
1840
1841 if (state->earliest_job < 0)
1842 return 0;
1843 a = state->next_job / sizeof(JOB_AES_HMAC);
1844 b = state->earliest_job / sizeof(JOB_AES_HMAC);
1845 return ((a-b) & (MAX_JOBS-1));
1846 }
Ceph