]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/intel-ipsec-mb/no-aesni/mb_mgr_sse_no_aesni.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
import 15.2.0 Octopus source
[ceph.git] / ceph / src / spdk / intel-ipsec-mb / no-aesni / mb_mgr_sse_no_aesni.c
1 /*******************************************************************************
2 Copyright (c) 2018, Intel Corporation
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions are met:
6
7 * Redistributions of source code must retain the above copyright notice,
8 this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in the
11 documentation and/or other materials provided with the distribution.
12 * Neither the name of Intel Corporation nor the names of its contributors
13 may be used to endorse or promote products derived from this software
14 without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *******************************************************************************/
27
28
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32
33 #include "intel-ipsec-mb.h"
34 #include "save_xmms.h"
35 #include "asm.h"
36 #include "des.h"
37 #include "gcm.h"
38 #include "noaesni.h"
39
40 /* ====================================================================== */
41
42 JOB_AES_HMAC *submit_job_aes128_enc_sse_no_aesni(MB_MGR_AES_OOO *state,
43 JOB_AES_HMAC *job);
44 JOB_AES_HMAC *flush_job_aes128_enc_sse_no_aesni(MB_MGR_AES_OOO *state);
45
46 JOB_AES_HMAC *submit_job_aes192_enc_sse_no_aesni(MB_MGR_AES_OOO *state,
47 JOB_AES_HMAC *job);
48 JOB_AES_HMAC *flush_job_aes192_enc_sse_no_aesni(MB_MGR_AES_OOO *state);
49
50 JOB_AES_HMAC *submit_job_aes256_enc_sse_no_aesni(MB_MGR_AES_OOO *state,
51 JOB_AES_HMAC *job);
52 JOB_AES_HMAC *flush_job_aes256_enc_sse_no_aesni(MB_MGR_AES_OOO *state);
53
54 JOB_AES_HMAC *submit_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state,
55 JOB_AES_HMAC *job);
56 JOB_AES_HMAC *flush_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state);
57
58 JOB_AES_HMAC *submit_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state,
59 JOB_AES_HMAC *job);
60 JOB_AES_HMAC *flush_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state);
61
62 JOB_AES_HMAC *submit_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state,
63 JOB_AES_HMAC *job);
64 JOB_AES_HMAC *flush_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state);
65
66 JOB_AES_HMAC *submit_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state,
67 JOB_AES_HMAC *job);
68 JOB_AES_HMAC *flush_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state);
69
70 JOB_AES_HMAC *submit_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state,
71 JOB_AES_HMAC *job);
72 JOB_AES_HMAC *flush_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state);
73
74 JOB_AES_HMAC *submit_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state,
75 JOB_AES_HMAC *job);
76 JOB_AES_HMAC *flush_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state);
77
78 JOB_AES_HMAC *submit_job_aes_xcbc_sse_no_aesni(MB_MGR_AES_XCBC_OOO *state,
79 JOB_AES_HMAC *job);
80 JOB_AES_HMAC *flush_job_aes_xcbc_sse_no_aesni(MB_MGR_AES_XCBC_OOO *state);
81
82 JOB_AES_HMAC *submit_job_aes_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state,
83 JOB_AES_HMAC *job);
84 JOB_AES_HMAC *flush_job_aes_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state);
85
86
87 #define SAVE_XMMS save_xmms
88 #define RESTORE_XMMS restore_xmms
89 #define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_sse_no_aesni
90 #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse_no_aesni
91 #define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_sse_no_aesni
92 #define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_sse_no_aesni
93 #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse_no_aesni
94 #define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_sse_no_aesni
95 #define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_sse_no_aesni
96 #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse_no_aesni
97 #define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_sse_no_aesni
98 #define SUBMIT_JOB_HMAC submit_job_hmac_sse
99 #define FLUSH_JOB_HMAC flush_job_hmac_sse
100 #define SUBMIT_JOB_HMAC_NI submit_job_hmac_sse
101 #define FLUSH_JOB_HMAC_NI flush_job_hmac_sse
102 #define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_sse
103 #define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_sse
104 #define SUBMIT_JOB_HMAC_SHA_224_NI submit_job_hmac_sha_224_sse
105 #define FLUSH_JOB_HMAC_SHA_224_NI flush_job_hmac_sha_224_sse
106 #define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_sse
107 #define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_sse
108 #define SUBMIT_JOB_HMAC_SHA_256_NI submit_job_hmac_sha_256_sse
109 #define FLUSH_JOB_HMAC_SHA_256_NI flush_job_hmac_sha_256_sse
110 #define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_sse
111 #define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_sse
112 #define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_sse
113 #define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_sse
114 #define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_sse
115 #define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_sse
116 #define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse_no_aesni
117 #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse_no_aesni
118
119 #define SUBMIT_JOB_AES128_CNTR submit_job_aes128_cntr_sse
120 #define SUBMIT_JOB_AES192_CNTR submit_job_aes192_cntr_sse
121 #define SUBMIT_JOB_AES256_CNTR submit_job_aes256_cntr_sse
122
123 #define AES_CBC_DEC_128 aes_cbc_dec_128_sse_no_aesni
124 #define AES_CBC_DEC_192 aes_cbc_dec_192_sse_no_aesni
125 #define AES_CBC_DEC_256 aes_cbc_dec_256_sse_no_aesni
126
127 #define AES_CNTR_128 aes_cntr_128_sse_no_aesni
128 #define AES_CNTR_192 aes_cntr_192_sse_no_aesni
129 #define AES_CNTR_256 aes_cntr_256_sse_no_aesni
130
131 #ifndef NO_GCM
132 #define AES_GCM_DEC_128 aes_gcm_dec_128_sse_no_aesni
133 #define AES_GCM_ENC_128 aes_gcm_enc_128_sse_no_aesni
134 #define AES_GCM_DEC_192 aes_gcm_dec_192_sse_no_aesni
135 #define AES_GCM_ENC_192 aes_gcm_enc_192_sse_no_aesni
136 #define AES_GCM_DEC_256 aes_gcm_dec_256_sse_no_aesni
137 #define AES_GCM_ENC_256 aes_gcm_enc_256_sse_no_aesni
138
139 #define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse_no_aesni
140 #define FLUSH_JOB_AES_GCM_DEC flush_job_aes_gcm_dec_sse_no_aesni
141 #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse_no_aesni
142 #define FLUSH_JOB_AES_GCM_ENC flush_job_aes_gcm_enc_sse_no_aesni
143 #endif /* NO_GCM */
144
145 /* ====================================================================== */
146
147 #define SUBMIT_JOB submit_job_sse_no_aesni
148 #define FLUSH_JOB flush_job_sse_no_aesni
149 #define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse_no_aesni
150 #define GET_NEXT_JOB get_next_job_sse_no_aesni
151 #define GET_COMPLETED_JOB get_completed_job_sse_no_aesni
152
153 #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse_no_aesni
154 #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse_no_aesni
155 #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse_no_aesni
156 #define QUEUE_SIZE queue_size_sse_no_aesni
157
158 /* ====================================================================== */
159
160 #define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE
161 #define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE
162 #define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE
163 #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE
164 #define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE
165
166 /* ====================================================================== */
167
168 #define AES_CFB_128_ONE aes_cfb_128_one_sse_no_aesni
169
170 void aes128_cbc_mac_x4_no_aesni(AES_ARGS_x8 *args, uint64_t len);
171
172 #define AES128_CBC_MAC aes128_cbc_mac_x4_no_aesni
173
174 #define FLUSH_JOB_AES_CCM_AUTH flush_job_aes_ccm_auth_arch
175 #define SUBMIT_JOB_AES_CCM_AUTH submit_job_aes_ccm_auth_arch
176 #define AES_CCM_MAX_JOBS 4
177
178 #define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes_cmac_auth_sse_no_aesni
179 #define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes_cmac_auth_sse_no_aesni
180
181
182 /* ====================================================================== */
183
184 /*
185 * GCM submit / flush API for SSE arch without AESNI
186 */
187 #ifndef NO_GCM
188 static JOB_AES_HMAC *
189 submit_job_aes_gcm_dec_sse_no_aesni(MB_MGR *state, JOB_AES_HMAC *job)
190 {
191 DECLARE_ALIGNED(struct gcm_context_data ctx, 16);
192 (void) state;
193
194 if (16 == job->aes_key_len_in_bytes)
195 AES_GCM_DEC_128(job->aes_dec_key_expanded, &ctx, job->dst,
196 job->src +
197 job->cipher_start_src_offset_in_bytes,
198 job->msg_len_to_cipher_in_bytes,
199 job->iv,
200 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
201 job->auth_tag_output,
202 job->auth_tag_output_len_in_bytes);
203 else if (24 == job->aes_key_len_in_bytes)
204 AES_GCM_DEC_192(job->aes_dec_key_expanded, &ctx, job->dst,
205 job->src +
206 job->cipher_start_src_offset_in_bytes,
207 job->msg_len_to_cipher_in_bytes,
208 job->iv,
209 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
210 job->auth_tag_output,
211 job->auth_tag_output_len_in_bytes);
212 else /* assume 32 bytes */
213 AES_GCM_DEC_256(job->aes_dec_key_expanded, &ctx, job->dst,
214 job->src +
215 job->cipher_start_src_offset_in_bytes,
216 job->msg_len_to_cipher_in_bytes,
217 job->iv,
218 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
219 job->auth_tag_output,
220 job->auth_tag_output_len_in_bytes);
221
222 job->status = STS_COMPLETED;
223 return job;
224 }
225
226 static JOB_AES_HMAC *
227 flush_job_aes_gcm_dec_sse_no_aesni(MB_MGR *state, JOB_AES_HMAC *job)
228 {
229 (void) state;
230 (void) job;
231 return NULL;
232 }
233
234 static JOB_AES_HMAC *
235 submit_job_aes_gcm_enc_sse_no_aesni(MB_MGR *state, JOB_AES_HMAC *job)
236 {
237 DECLARE_ALIGNED(struct gcm_context_data ctx, 16);
238 (void) state;
239
240 if (16 == job->aes_key_len_in_bytes)
241 AES_GCM_ENC_128(job->aes_enc_key_expanded, &ctx, job->dst,
242 job->src +
243 job->cipher_start_src_offset_in_bytes,
244 job->msg_len_to_cipher_in_bytes, job->iv,
245 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
246 job->auth_tag_output,
247 job->auth_tag_output_len_in_bytes);
248 else if (24 == job->aes_key_len_in_bytes)
249 AES_GCM_ENC_192(job->aes_enc_key_expanded, &ctx, job->dst,
250 job->src +
251 job->cipher_start_src_offset_in_bytes,
252 job->msg_len_to_cipher_in_bytes, job->iv,
253 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
254 job->auth_tag_output,
255 job->auth_tag_output_len_in_bytes);
256 else /* assume 32 bytes */
257 AES_GCM_ENC_256(job->aes_enc_key_expanded, &ctx, job->dst,
258 job->src +
259 job->cipher_start_src_offset_in_bytes,
260 job->msg_len_to_cipher_in_bytes, job->iv,
261 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
262 job->auth_tag_output,
263 job->auth_tag_output_len_in_bytes);
264
265 job->status = STS_COMPLETED;
266 return job;
267 }
268
269 static JOB_AES_HMAC *
270 flush_job_aes_gcm_enc_sse_no_aesni(MB_MGR *state, JOB_AES_HMAC *job)
271 {
272 (void) state;
273 (void) job;
274 return NULL;
275 }
276 #endif /* NO_GCM */
277
278 /* ====================================================================== */
279
280 void
281 init_mb_mgr_sse_no_aesni(MB_MGR *state)
282 {
283 unsigned int j;
284 uint8_t *p;
285
286 /* Init AES out-of-order fields */
287 state->aes128_ooo.lens[0] = 0;
288 state->aes128_ooo.lens[1] = 0;
289 state->aes128_ooo.lens[2] = 0;
290 state->aes128_ooo.lens[3] = 0;
291 state->aes128_ooo.lens[4] = 0xFFFF;
292 state->aes128_ooo.lens[5] = 0xFFFF;
293 state->aes128_ooo.lens[6] = 0xFFFF;
294 state->aes128_ooo.lens[7] = 0xFFFF;
295 state->aes128_ooo.unused_lanes = 0xFF03020100;
296 state->aes128_ooo.job_in_lane[0] = NULL;
297 state->aes128_ooo.job_in_lane[1] = NULL;
298 state->aes128_ooo.job_in_lane[2] = NULL;
299 state->aes128_ooo.job_in_lane[3] = NULL;
300
301 state->aes192_ooo.lens[0] = 0;
302 state->aes192_ooo.lens[1] = 0;
303 state->aes192_ooo.lens[2] = 0;
304 state->aes192_ooo.lens[3] = 0;
305 state->aes192_ooo.lens[4] = 0xFFFF;
306 state->aes192_ooo.lens[5] = 0xFFFF;
307 state->aes192_ooo.lens[6] = 0xFFFF;
308 state->aes192_ooo.lens[7] = 0xFFFF;
309 state->aes192_ooo.unused_lanes = 0xFF03020100;
310 state->aes192_ooo.job_in_lane[0] = NULL;
311 state->aes192_ooo.job_in_lane[1] = NULL;
312 state->aes192_ooo.job_in_lane[2] = NULL;
313 state->aes192_ooo.job_in_lane[3] = NULL;
314
315 state->aes256_ooo.lens[0] = 0;
316 state->aes256_ooo.lens[1] = 0;
317 state->aes256_ooo.lens[2] = 0;
318 state->aes256_ooo.lens[3] = 0;
319 state->aes256_ooo.lens[4] = 0xFFFF;
320 state->aes256_ooo.lens[5] = 0xFFFF;
321 state->aes256_ooo.lens[6] = 0xFFFF;
322 state->aes256_ooo.lens[7] = 0xFFFF;
323 state->aes256_ooo.unused_lanes = 0xFF03020100;
324 state->aes256_ooo.job_in_lane[0] = NULL;
325 state->aes256_ooo.job_in_lane[1] = NULL;
326 state->aes256_ooo.job_in_lane[2] = NULL;
327 state->aes256_ooo.job_in_lane[3] = NULL;
328
329 /* DOCSIS SEC BPI uses same settings as AES128 CBC */
330 state->docsis_sec_ooo.lens[0] = 0;
331 state->docsis_sec_ooo.lens[1] = 0;
332 state->docsis_sec_ooo.lens[2] = 0;
333 state->docsis_sec_ooo.lens[3] = 0;
334 state->docsis_sec_ooo.lens[4] = 0xFFFF;
335 state->docsis_sec_ooo.lens[5] = 0xFFFF;
336 state->docsis_sec_ooo.lens[6] = 0xFFFF;
337 state->docsis_sec_ooo.lens[7] = 0xFFFF;
338 state->docsis_sec_ooo.unused_lanes = 0xFF03020100;
339 state->docsis_sec_ooo.job_in_lane[0] = NULL;
340 state->docsis_sec_ooo.job_in_lane[1] = NULL;
341 state->docsis_sec_ooo.job_in_lane[2] = NULL;
342 state->docsis_sec_ooo.job_in_lane[3] = NULL;
343
344 /* Init HMAC/SHA1 out-of-order fields */
345 state->hmac_sha_1_ooo.lens[0] = 0;
346 state->hmac_sha_1_ooo.lens[1] = 0;
347 state->hmac_sha_1_ooo.lens[2] = 0;
348 state->hmac_sha_1_ooo.lens[3] = 0;
349 state->hmac_sha_1_ooo.lens[4] = 0xFFFF;
350 state->hmac_sha_1_ooo.lens[5] = 0xFFFF;
351 state->hmac_sha_1_ooo.lens[6] = 0xFFFF;
352 state->hmac_sha_1_ooo.lens[7] = 0xFFFF;
353 state->hmac_sha_1_ooo.unused_lanes = 0xFF03020100;
354 for (j = 0; j < SSE_NUM_SHA1_LANES; j++) {
355 state->hmac_sha_1_ooo.ldata[j].job_in_lane = NULL;
356 state->hmac_sha_1_ooo.ldata[j].extra_block[64] = 0x80;
357 memset(state->hmac_sha_1_ooo.ldata[j].extra_block + 65,
358 0x00,
359 64+7);
360 p = state->hmac_sha_1_ooo.ldata[j].outer_block;
361 memset(p + 5*4 + 1,
362 0x00,
363 64 - 5*4 - 1 - 2);
364 p[5*4] = 0x80;
365 p[64-2] = 0x02;
366 p[64-1] = 0xA0;
367 }
368
369 /* Init HMAC/SHA224 out-of-order fields */
370 state->hmac_sha_224_ooo.lens[0] = 0;
371 state->hmac_sha_224_ooo.lens[1] = 0;
372 state->hmac_sha_224_ooo.lens[2] = 0;
373 state->hmac_sha_224_ooo.lens[3] = 0;
374 state->hmac_sha_224_ooo.lens[4] = 0xFFFF;
375 state->hmac_sha_224_ooo.lens[5] = 0xFFFF;
376 state->hmac_sha_224_ooo.lens[6] = 0xFFFF;
377 state->hmac_sha_224_ooo.lens[7] = 0xFFFF;
378 state->hmac_sha_224_ooo.unused_lanes = 0xFF03020100;
379 for (j = 0; j < SSE_NUM_SHA256_LANES; j++) {
380 state->hmac_sha_224_ooo.ldata[j].job_in_lane = NULL;
381 state->hmac_sha_224_ooo.ldata[j].extra_block[64] = 0x80;
382 memset(state->hmac_sha_224_ooo.ldata[j].extra_block + 65,
383 0x00,
384 64+7);
385 p = state->hmac_sha_224_ooo.ldata[j].outer_block;
386 memset(p + 8*4 + 1,
387 0x00,
388 64 - 8*4 - 1 - 2);
389 p[7*4] = 0x80; /* digest 7 words long */
390 p[64-2] = 0x02; /* length in little endian = 0x02E0 */
391 p[64-1] = 0xE0;
392 }
393
394 /* Init HMAC/SHA_256 out-of-order fields */
395 state->hmac_sha_256_ooo.lens[0] = 0;
396 state->hmac_sha_256_ooo.lens[1] = 0;
397 state->hmac_sha_256_ooo.lens[2] = 0;
398 state->hmac_sha_256_ooo.lens[3] = 0;
399 state->hmac_sha_256_ooo.lens[4] = 0xFFFF;
400 state->hmac_sha_256_ooo.lens[5] = 0xFFFF;
401 state->hmac_sha_256_ooo.lens[6] = 0xFFFF;
402 state->hmac_sha_256_ooo.lens[7] = 0xFFFF;
403 state->hmac_sha_256_ooo.unused_lanes = 0xFF03020100;
404 for (j = 0; j < SSE_NUM_SHA256_LANES; j++) {
405 state->hmac_sha_256_ooo.ldata[j].job_in_lane = NULL;
406 state->hmac_sha_256_ooo.ldata[j].extra_block[64] = 0x80;
407 memset(state->hmac_sha_256_ooo.ldata[j].extra_block + 65,
408 0x00,
409 64+7);
410 p = state->hmac_sha_256_ooo.ldata[j].outer_block;
411 memset(p + 8*4 + 1,
412 0x00,
413 64 - 8*4 - 1 - 2); /* digest is 8*4 bytes long */
414 p[8*4] = 0x80;
415 p[64-2] = 0x03; /* length of (opad (64*8) bits + 256 bits)
416 * in hex is 0x300 */
417 p[64-1] = 0x00;
418 }
419
420 /* Init HMAC/SHA384 out-of-order fields */
421 state->hmac_sha_384_ooo.lens[0] = 0;
422 state->hmac_sha_384_ooo.lens[1] = 0;
423 state->hmac_sha_384_ooo.lens[2] = 0xFFFF;
424 state->hmac_sha_384_ooo.lens[3] = 0xFFFF;
425 state->hmac_sha_384_ooo.lens[4] = 0xFFFF;
426 state->hmac_sha_384_ooo.lens[5] = 0xFFFF;
427 state->hmac_sha_384_ooo.lens[6] = 0xFFFF;
428 state->hmac_sha_384_ooo.lens[7] = 0xFFFF;
429 state->hmac_sha_384_ooo.unused_lanes = 0xFF0100;
430 for (j = 0; j < SSE_NUM_SHA512_LANES; j++) {
431 MB_MGR_HMAC_SHA_512_OOO *ctx = &state->hmac_sha_384_ooo;
432
433 ctx->ldata[j].job_in_lane = NULL;
434 ctx->ldata[j].extra_block[SHA_384_BLOCK_SIZE] = 0x80;
435 memset(ctx->ldata[j].extra_block + (SHA_384_BLOCK_SIZE + 1),
436 0x00, SHA_384_BLOCK_SIZE + 7);
437
438 p = ctx->ldata[j].outer_block;
439 memset(p + SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00,
440 /* special end point because this length is constant */
441 SHA_384_BLOCK_SIZE -
442 SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2);
443 p[SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */
444 /*
445 * hmac outer block length always of fixed size, it is OKey
446 * length, a whole message block length, 1024 bits, with padding
447 * plus the length of the inner digest, which is 384 bits
448 * 1408 bits == 0x0580. The input message block needs to be
449 * converted to big endian within the sha implementation
450 * before use.
451 */
452 p[SHA_384_BLOCK_SIZE - 2] = 0x05;
453 p[SHA_384_BLOCK_SIZE - 1] = 0x80;
454 }
455
456 /* Init HMAC/SHA512 out-of-order fields */
457 state->hmac_sha_512_ooo.lens[0] = 0;
458 state->hmac_sha_512_ooo.lens[1] = 0;
459 state->hmac_sha_512_ooo.lens[2] = 0xFFFF;
460 state->hmac_sha_512_ooo.lens[3] = 0xFFFF;
461 state->hmac_sha_512_ooo.lens[4] = 0xFFFF;
462 state->hmac_sha_512_ooo.lens[5] = 0xFFFF;
463 state->hmac_sha_512_ooo.lens[6] = 0xFFFF;
464 state->hmac_sha_512_ooo.lens[7] = 0xFFFF;
465 state->hmac_sha_512_ooo.unused_lanes = 0xFF0100;
466 for (j = 0; j < SSE_NUM_SHA512_LANES; j++) {
467 MB_MGR_HMAC_SHA_512_OOO *ctx = &state->hmac_sha_512_ooo;
468
469 ctx->ldata[j].job_in_lane = NULL;
470 ctx->ldata[j].extra_block[SHA_512_BLOCK_SIZE] = 0x80;
471 memset(ctx->ldata[j].extra_block + (SHA_512_BLOCK_SIZE + 1),
472 0x00, SHA_512_BLOCK_SIZE + 7);
473
474 p = ctx->ldata[j].outer_block;
475 memset(p + SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00,
476 /* special end point because this length is constant */
477 SHA_512_BLOCK_SIZE -
478 SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2);
479 p[SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */
480 /*
481 * hmac outer block length always of fixed size, it is OKey
482 * length, a whole message block length, 1024 bits, with padding
483 * plus the length of the inner digest, which is 512 bits
484 * 1536 bits == 0x600. The input message block needs to be
485 * converted to big endian within the sha implementation
486 * before use.
487 */
488 p[SHA_512_BLOCK_SIZE - 2] = 0x06;
489 p[SHA_512_BLOCK_SIZE - 1] = 0x00;
490 }
491
492 /* Init HMAC/MD5 out-of-order fields */
493 state->hmac_md5_ooo.lens[0] = 0;
494 state->hmac_md5_ooo.lens[1] = 0;
495 state->hmac_md5_ooo.lens[2] = 0;
496 state->hmac_md5_ooo.lens[3] = 0;
497 state->hmac_md5_ooo.lens[4] = 0;
498 state->hmac_md5_ooo.lens[5] = 0;
499 state->hmac_md5_ooo.lens[6] = 0;
500 state->hmac_md5_ooo.lens[7] = 0;
501 state->hmac_md5_ooo.lens[8] = 0xFFFF;
502 state->hmac_md5_ooo.lens[9] = 0xFFFF;
503 state->hmac_md5_ooo.lens[10] = 0xFFFF;
504 state->hmac_md5_ooo.lens[11] = 0xFFFF;
505 state->hmac_md5_ooo.lens[12] = 0xFFFF;
506 state->hmac_md5_ooo.lens[13] = 0xFFFF;
507 state->hmac_md5_ooo.lens[14] = 0xFFFF;
508 state->hmac_md5_ooo.lens[15] = 0xFFFF;
509 state->hmac_md5_ooo.unused_lanes = 0xF76543210;
510 for (j = 0; j < SSE_NUM_MD5_LANES; j++) {
511 state->hmac_md5_ooo.ldata[j].job_in_lane = NULL;
512 state->hmac_md5_ooo.ldata[j].extra_block[64] = 0x80;
513 memset(state->hmac_md5_ooo.ldata[j].extra_block + 65,
514 0x00,
515 64 + 7);
516 p = state->hmac_md5_ooo.ldata[j].outer_block;
517 memset(p + (5 * 4) + 1,
518 0x00,
519 64 - (5 * 4) - 1 - 2);
520 p[4*4] = 0x80;
521 p[64-7] = 0x02;
522 p[64-8] = 0x80;
523 }
524
525 /* Init AES/XCBC OOO fields */
526 state->aes_xcbc_ooo.lens[0] = 0;
527 state->aes_xcbc_ooo.lens[1] = 0;
528 state->aes_xcbc_ooo.lens[2] = 0;
529 state->aes_xcbc_ooo.lens[3] = 0;
530 state->aes_xcbc_ooo.lens[4] = 0xFFFF;
531 state->aes_xcbc_ooo.lens[5] = 0xFFFF;
532 state->aes_xcbc_ooo.lens[6] = 0xFFFF;
533 state->aes_xcbc_ooo.lens[7] = 0xFFFF;
534 state->aes_xcbc_ooo.unused_lanes = 0xFF03020100;
535 for (j = 0; j < 4; j++) {
536 state->aes_xcbc_ooo.ldata[j].job_in_lane = NULL;
537 state->aes_xcbc_ooo.ldata[j].final_block[16] = 0x80;
538 memset(state->aes_xcbc_ooo.ldata[j].final_block + 17, 0x00, 15);
539 }
540
541 /* Init AES-CCM auth out-of-order fields */
542 for (j = 0; j < 4; j++) {
543 state->aes_ccm_ooo.init_done[j] = 0;
544 state->aes_ccm_ooo.lens[j] = 0;
545 state->aes_ccm_ooo.job_in_lane[j] = NULL;
546 }
547 state->aes_ccm_ooo.unused_lanes = 0xF3210;
548
549 /* Init AES-CMAC auth out-of-order fields */
550 state->aes_cmac_ooo.lens[0] = 0;
551 state->aes_cmac_ooo.lens[1] = 0;
552 state->aes_cmac_ooo.lens[2] = 0;
553 state->aes_cmac_ooo.lens[3] = 0;
554 state->aes_cmac_ooo.lens[4] = 0xFFFF;
555 state->aes_cmac_ooo.lens[5] = 0xFFFF;
556 state->aes_cmac_ooo.lens[6] = 0xFFFF;
557 state->aes_cmac_ooo.lens[7] = 0xFFFF;
558 for (j = 0; j < 4; j++) {
559 state->aes_cmac_ooo.init_done[j] = 0;
560 state->aes_cmac_ooo.job_in_lane[j] = NULL;
561 }
562 state->aes_cmac_ooo.unused_lanes = 0xF3210;
563
564 /* Init "in order" components */
565 state->next_job = 0;
566 state->earliest_job = -1;
567
568 /* set SSE NO AESNI handlers */
569 state->get_next_job = get_next_job_sse_no_aesni;
570 state->submit_job = submit_job_sse_no_aesni;
571 state->submit_job_nocheck = submit_job_nocheck_sse_no_aesni;
572 state->get_completed_job = get_completed_job_sse_no_aesni;
573 state->flush_job = flush_job_sse_no_aesni;
574 state->queue_size = queue_size_sse_no_aesni;
575 state->keyexp_128 = aes_keyexp_128_sse_no_aesni;
576 state->keyexp_192 = aes_keyexp_192_sse_no_aesni;
577 state->keyexp_256 = aes_keyexp_256_sse_no_aesni;
578 state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_sse_no_aesni;
579 state->xcbc_keyexp = aes_xcbc_expand_key_sse_no_aesni;
580 state->des_key_sched = des_key_schedule;
581 state->sha1_one_block = sha1_one_block_sse;
582 state->sha1 = sha1_sse;
583 state->sha224_one_block = sha224_one_block_sse;
584 state->sha224 = sha224_sse;
585 state->sha256_one_block = sha256_one_block_sse;
586 state->sha256 = sha256_sse;
587 state->sha384_one_block = sha384_one_block_sse;
588 state->sha384 = sha384_sse;
589 state->sha512_one_block = sha512_one_block_sse;
590 state->sha512 = sha512_sse;
591 state->md5_one_block = md5_one_block_sse;
592 state->aes128_cfb_one = aes_cfb_128_one_sse_no_aesni;
593 }
594
595 #include "mb_mgr_code.h"
Ceph