]> git.proxmox.com Git - ceph.git/blame - ceph/src/spdk/intel-ipsec-mb/sse/mb_mgr_sse.c
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update source to Ceph Pacific 16.2.2
[ceph.git] / ceph / src / spdk / intel-ipsec-mb / sse / mb_mgr_sse.c
CommitLineData
11fdf7f2
TL		 1/*******************************************************************************
2 Copyright (c) 2012-2018, Intel Corporation
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions are met:
6
7 * Redistributions of source code must retain the above copyright notice,
8 this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in the
11 documentation and/or other materials provided with the distribution.
12 * Neither the name of Intel Corporation nor the names of its contributors
13 may be used to endorse or promote products derived from this software
14 without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26*******************************************************************************/
27
28
29#include <stdio.h>
30#include <stdlib.h>
31#include <string.h>
32
f67539c2
TL		 33#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_sse
34
11fdf7f2 35#include "intel-ipsec-mb.h"
f67539c2
TL		 36#include "include/kasumi_internal.h"
37#include "include/zuc_internal.h"
38#include "include/snow3g.h"
39
11fdf7f2
TL		 40#include "save_xmms.h"
41#include "asm.h"
42#include "des.h"
9f95a23c
TL		 43#include "cpu_feature.h"
44#include "noaesni.h"
11fdf7f2
TL		 45
46JOB_AES_HMAC *submit_job_aes128_enc_sse(MB_MGR_AES_OOO *state,
47 JOB_AES_HMAC *job);
48JOB_AES_HMAC *flush_job_aes128_enc_sse(MB_MGR_AES_OOO *state);
49
50JOB_AES_HMAC *submit_job_aes192_enc_sse(MB_MGR_AES_OOO *state,
51 JOB_AES_HMAC *job);
52JOB_AES_HMAC *flush_job_aes192_enc_sse(MB_MGR_AES_OOO *state);
53
54JOB_AES_HMAC *submit_job_aes256_enc_sse(MB_MGR_AES_OOO *state,
55 JOB_AES_HMAC *job);
56JOB_AES_HMAC *flush_job_aes256_enc_sse(MB_MGR_AES_OOO *state);
57
58JOB_AES_HMAC *submit_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state,
59 JOB_AES_HMAC *job);
60JOB_AES_HMAC *flush_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state);
61
62JOB_AES_HMAC *submit_job_hmac_ni_sse(MB_MGR_HMAC_SHA_1_OOO *state,
63 JOB_AES_HMAC *job);
64JOB_AES_HMAC *flush_job_hmac_ni_sse(MB_MGR_HMAC_SHA_1_OOO *state);
65
66JOB_AES_HMAC *submit_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state,
67 JOB_AES_HMAC *job);
68JOB_AES_HMAC *flush_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state);
69
70JOB_AES_HMAC *submit_job_hmac_sha_224_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state,
71 JOB_AES_HMAC *job);
72JOB_AES_HMAC *flush_job_hmac_sha_224_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state);
73
74JOB_AES_HMAC *submit_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state,
75 JOB_AES_HMAC *job);
76JOB_AES_HMAC *flush_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state);
77
78JOB_AES_HMAC *submit_job_hmac_sha_256_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state,
79 JOB_AES_HMAC *job);
80JOB_AES_HMAC *flush_job_hmac_sha_256_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state);
81
82JOB_AES_HMAC *submit_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state,
83 JOB_AES_HMAC *job);
84JOB_AES_HMAC *flush_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state);
85
86JOB_AES_HMAC *submit_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state,
87 JOB_AES_HMAC *job);
88JOB_AES_HMAC *flush_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state);
89
90JOB_AES_HMAC *submit_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state,
91 JOB_AES_HMAC *job);
92JOB_AES_HMAC *flush_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state);
93
94
95JOB_AES_HMAC *submit_job_aes_xcbc_sse(MB_MGR_AES_XCBC_OOO *state,
96 JOB_AES_HMAC *job);
97JOB_AES_HMAC *flush_job_aes_xcbc_sse(MB_MGR_AES_XCBC_OOO *state);
98
9f95a23c
TL		 99JOB_AES_HMAC *submit_job_aes_cmac_auth_sse(MB_MGR_CMAC_OOO *state,
100 JOB_AES_HMAC *job);
101
102JOB_AES_HMAC *flush_job_aes_cmac_auth_sse(MB_MGR_CMAC_OOO *state);
103
f67539c2
TL		 104JOB_AES_HMAC *submit_job_aes_ccm_auth_sse(MB_MGR_CCM_OOO *state,
105 JOB_AES_HMAC *job);
106
107JOB_AES_HMAC *flush_job_aes_ccm_auth_sse(MB_MGR_CCM_OOO *state);
108
109JOB_AES_HMAC *submit_job_aes_cntr_sse(JOB_AES_HMAC *job);
110
111JOB_AES_HMAC *submit_job_aes_cntr_bit_sse(JOB_AES_HMAC *job);
112
113#define SAVE_XMMS save_xmms
114#define RESTORE_XMMS restore_xmms
9f95a23c 115
11fdf7f2
TL		 116#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_sse
117#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse
118#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_sse
119#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_sse
120#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse
121#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_sse
122#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_sse
123#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse
124#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_sse
f67539c2
TL		 125#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_sse
126#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_sse
127#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_sse
128#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_sse
129#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_sse
130#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_sse
11fdf7f2
TL		 131#define SUBMIT_JOB_HMAC submit_job_hmac_sse
132#define FLUSH_JOB_HMAC flush_job_hmac_sse
133#define SUBMIT_JOB_HMAC_NI submit_job_hmac_ni_sse
134#define FLUSH_JOB_HMAC_NI flush_job_hmac_ni_sse
135#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_sse
136#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_sse
137#define SUBMIT_JOB_HMAC_SHA_224_NI submit_job_hmac_sha_224_ni_sse
138#define FLUSH_JOB_HMAC_SHA_224_NI flush_job_hmac_sha_224_ni_sse
139#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_sse
140#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_sse
141#define SUBMIT_JOB_HMAC_SHA_256_NI submit_job_hmac_sha_256_ni_sse
142#define FLUSH_JOB_HMAC_SHA_256_NI flush_job_hmac_sha_256_ni_sse
143#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_sse
144#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_sse
145#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_sse
146#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_sse
147#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_sse
148#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_sse
149#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse
150#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse
151
f67539c2
TL		 152#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse
153#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse
11fdf7f2
TL		 154
155#define AES_CBC_DEC_128 aes_cbc_dec_128_sse
156#define AES_CBC_DEC_192 aes_cbc_dec_192_sse
157#define AES_CBC_DEC_256 aes_cbc_dec_256_sse
158
159#define AES_CNTR_128 aes_cntr_128_sse
160#define AES_CNTR_192 aes_cntr_192_sse
161#define AES_CNTR_256 aes_cntr_256_sse
162
f67539c2
TL		 163#define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse
164
165#define AES_ECB_ENC_128 aes_ecb_enc_128_sse
166#define AES_ECB_ENC_192 aes_ecb_enc_192_sse
167#define AES_ECB_ENC_256 aes_ecb_enc_256_sse
168#define AES_ECB_DEC_128 aes_ecb_dec_128_sse
169#define AES_ECB_DEC_192 aes_ecb_dec_192_sse
170#define AES_ECB_DEC_256 aes_ecb_dec_256_sse
171
172#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_sse
173#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_sse
174#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_sse
175#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_sse
176
11fdf7f2
TL		 177#ifndef NO_GCM
178#define AES_GCM_DEC_128 aes_gcm_dec_128_sse
179#define AES_GCM_ENC_128 aes_gcm_enc_128_sse
180#define AES_GCM_DEC_192 aes_gcm_dec_192_sse
181#define AES_GCM_ENC_192 aes_gcm_enc_192_sse
182#define AES_GCM_DEC_256 aes_gcm_dec_256_sse
183#define AES_GCM_ENC_256 aes_gcm_enc_256_sse
9f95a23c
TL		 184
185#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse
186#define FLUSH_JOB_AES_GCM_DEC flush_job_aes_gcm_dec_sse
187#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse
188#define FLUSH_JOB_AES_GCM_ENC flush_job_aes_gcm_enc_sse
11fdf7f2
TL		 189#endif /* NO_GCM */
190
191/* ====================================================================== */
192
193#define SUBMIT_JOB submit_job_sse
194#define FLUSH_JOB flush_job_sse
195#define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse
9f95a23c
TL		 196#define GET_NEXT_JOB get_next_job_sse
197#define GET_COMPLETED_JOB get_completed_job_sse
11fdf7f2
TL		 198
199#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse
200#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse
201#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse
202#define QUEUE_SIZE queue_size_sse
203
204/* ====================================================================== */
205
206#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE
207#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE
208#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE
209#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE
210#define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE
211
212/* ====================================================================== */
213
214#define AES_CFB_128_ONE aes_cfb_128_one_sse
215
f67539c2 216void aes128_cbc_mac_x4(AES_ARGS *args, uint64_t len);
11fdf7f2
TL		 217
218#define AES128_CBC_MAC aes128_cbc_mac_x4
219
f67539c2
TL		 220#define FLUSH_JOB_AES_CCM_AUTH flush_job_aes_ccm_auth_sse
221#define SUBMIT_JOB_AES_CCM_AUTH submit_job_aes_ccm_auth_sse
11fdf7f2 222
9f95a23c
TL		 223#define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes_cmac_auth_sse
224#define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes_cmac_auth_sse
11fdf7f2
TL		 225
226/* ====================================================================== */
227
228/*
229 * Used to decide if SHA1/SHA256 SIMD or SHA1NI OOO scheduler should be
230 * called.
231 */
232#define HASH_USE_SHAEXT 1
233
11fdf7f2 234
9f95a23c 235/* ====================================================================== */
11fdf7f2
TL		 236
237/*
9f95a23c 238 * GCM submit / flush API for SSE arch
11fdf7f2 239 */
9f95a23c
TL		 240#ifndef NO_GCM
241static JOB_AES_HMAC *
242submit_job_aes_gcm_dec_sse(MB_MGR *state, JOB_AES_HMAC *job)
11fdf7f2 243{
9f95a23c
TL		 244 DECLARE_ALIGNED(struct gcm_context_data ctx, 16);
245 (void) state;
246
247 if (16 == job->aes_key_len_in_bytes)
248 AES_GCM_DEC_128(job->aes_dec_key_expanded, &ctx, job->dst,
249 job->src +
250 job->cipher_start_src_offset_in_bytes,
251 job->msg_len_to_cipher_in_bytes,
252 job->iv,
253 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
254 job->auth_tag_output,
255 job->auth_tag_output_len_in_bytes);
256 else if (24 == job->aes_key_len_in_bytes)
257 AES_GCM_DEC_192(job->aes_dec_key_expanded, &ctx, job->dst,
258 job->src +
259 job->cipher_start_src_offset_in_bytes,
260 job->msg_len_to_cipher_in_bytes,
261 job->iv,
262 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
263 job->auth_tag_output,
264 job->auth_tag_output_len_in_bytes);
265 else /* assume 32 bytes */
266 AES_GCM_DEC_256(job->aes_dec_key_expanded, &ctx, job->dst,
267 job->src +
268 job->cipher_start_src_offset_in_bytes,
269 job->msg_len_to_cipher_in_bytes,
270 job->iv,
271 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
272 job->auth_tag_output,
273 job->auth_tag_output_len_in_bytes);
274
275 job->status = STS_COMPLETED;
276 return job;
11fdf7f2
TL		 277}
278
9f95a23c
TL		 279static JOB_AES_HMAC *
280flush_job_aes_gcm_dec_sse(MB_MGR *state, JOB_AES_HMAC *job)
11fdf7f2 281{
9f95a23c
TL		 282 (void) state;
283 (void) job;
284 return NULL;
285}
11fdf7f2 286
9f95a23c
TL		 287static JOB_AES_HMAC *
288submit_job_aes_gcm_enc_sse(MB_MGR *state, JOB_AES_HMAC *job)
289{
290 DECLARE_ALIGNED(struct gcm_context_data ctx, 16);
291 (void) state;
292
293 if (16 == job->aes_key_len_in_bytes)
294 AES_GCM_ENC_128(job->aes_enc_key_expanded, &ctx, job->dst,
295 job->src +
296 job->cipher_start_src_offset_in_bytes,
297 job->msg_len_to_cipher_in_bytes, job->iv,
298 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
299 job->auth_tag_output,
300 job->auth_tag_output_len_in_bytes);
301 else if (24 == job->aes_key_len_in_bytes)
302 AES_GCM_ENC_192(job->aes_enc_key_expanded, &ctx, job->dst,
303 job->src +
304 job->cipher_start_src_offset_in_bytes,
305 job->msg_len_to_cipher_in_bytes, job->iv,
306 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
307 job->auth_tag_output,
308 job->auth_tag_output_len_in_bytes);
309 else /* assume 32 bytes */
310 AES_GCM_ENC_256(job->aes_enc_key_expanded, &ctx, job->dst,
311 job->src +
312 job->cipher_start_src_offset_in_bytes,
313 job->msg_len_to_cipher_in_bytes, job->iv,
314 job->u.GCM.aad, job->u.GCM.aad_len_in_bytes,
315 job->auth_tag_output,
316 job->auth_tag_output_len_in_bytes);
317
318 job->status = STS_COMPLETED;
319 return job;
320}
11fdf7f2 321
9f95a23c
TL		 322static JOB_AES_HMAC *
323flush_job_aes_gcm_enc_sse(MB_MGR *state, JOB_AES_HMAC *job)
324{
325 (void) state;
326 (void) job;
327 return NULL;
11fdf7f2 328}
9f95a23c
TL		 329#endif /* NO_GCM */
330
f67539c2
TL		 331IMB_DLL_LOCAL JOB_AES_HMAC *
332submit_job_aes_cntr_sse(JOB_AES_HMAC *job)
333{
334 if (16 == job->aes_key_len_in_bytes)
335 AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes,
336 job->iv,
337 job->aes_enc_key_expanded,
338 job->dst,
339 job->msg_len_to_cipher_in_bytes,
340 job->iv_len_in_bytes);
341 else if (24 == job->aes_key_len_in_bytes)
342 AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes,
343 job->iv,
344 job->aes_enc_key_expanded,
345 job->dst,
346 job->msg_len_to_cipher_in_bytes,
347 job->iv_len_in_bytes);
348 else /* assume 32 bytes */
349 AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes,
350 job->iv,
351 job->aes_enc_key_expanded,
352 job->dst,
353 job->msg_len_to_cipher_in_bytes,
354 job->iv_len_in_bytes);
355
356 job->status |= STS_COMPLETED_AES;
357 return job;
358}
359
360IMB_DLL_LOCAL JOB_AES_HMAC *
361submit_job_aes_cntr_bit_sse(JOB_AES_HMAC *job)
362{
363 if (16 == job->aes_key_len_in_bytes)
364 aes_cntr_bit_128_sse(job->src +
365 job->cipher_start_src_offset_in_bytes,
366 job->iv,
367 job->aes_enc_key_expanded,
368 job->dst,
369 job->msg_len_to_cipher_in_bits,
370 job->iv_len_in_bytes);
371 else if (24 == job->aes_key_len_in_bytes)
372 aes_cntr_bit_192_sse(job->src +
373 job->cipher_start_src_offset_in_bytes,
374 job->iv,
375 job->aes_enc_key_expanded,
376 job->dst,
377 job->msg_len_to_cipher_in_bits,
378 job->iv_len_in_bytes);
379 else /* assume 32 bytes */
380 aes_cntr_bit_256_sse(job->src +
381 job->cipher_start_src_offset_in_bytes,
382 job->iv,
383 job->aes_enc_key_expanded,
384 job->dst,
385 job->msg_len_to_cipher_in_bits,
386 job->iv_len_in_bytes);
387
388 job->status |= STS_COMPLETED_AES;
389 return job;
390}
391
9f95a23c 392/* ====================================================================== */
11fdf7f2
TL		 393
394void
395init_mb_mgr_sse(MB_MGR *state)
396{
397 unsigned int j;
398 uint8_t *p;
f67539c2 399 size_t size;
11fdf7f2 400
9f95a23c
TL		 401 state->features = cpu_feature_adjust(state->flags,
402 cpu_feature_detect());
403
404 if (!(state->features & IMB_FEATURE_AESNI)) {
405 init_mb_mgr_sse_no_aesni(state);
406 return;
407 }
11fdf7f2
TL		 408
409 /* Init AES out-of-order fields */
f67539c2
TL		 410 memset(state->aes128_ooo.lens, 0xFF,
411 sizeof(state->aes128_ooo.lens));
412 memset(&state->aes128_ooo.lens[0], 0,
413 sizeof(state->aes128_ooo.lens[0]) * 4);
414 memset(state->aes128_ooo.job_in_lane, 0,
415 sizeof(state->aes128_ooo.job_in_lane));
11fdf7f2 416 state->aes128_ooo.unused_lanes = 0xFF03020100;
f67539c2
TL		 417 state->aes128_ooo.num_lanes_inuse = 0;
418
419
420 memset(state->aes192_ooo.lens, 0xFF,
421 sizeof(state->aes192_ooo.lens));
422 memset(&state->aes192_ooo.lens[0], 0,
423 sizeof(state->aes192_ooo.lens[0]) * 4);
424 memset(state->aes192_ooo.job_in_lane, 0,
425 sizeof(state->aes192_ooo.job_in_lane));
11fdf7f2 426 state->aes192_ooo.unused_lanes = 0xFF03020100;
f67539c2
TL		 427 state->aes192_ooo.num_lanes_inuse = 0;
428
429
430 memset(state->aes256_ooo.lens, 0xFF,
431 sizeof(state->aes256_ooo.lens));
432 memset(&state->aes256_ooo.lens[0], 0,
433 sizeof(state->aes256_ooo.lens[0]) * 4);
434 memset(state->aes256_ooo.job_in_lane, 0,
435 sizeof(state->aes256_ooo.job_in_lane));
11fdf7f2 436 state->aes256_ooo.unused_lanes = 0xFF03020100;
f67539c2
TL		 437 state->aes256_ooo.num_lanes_inuse = 0;
438
11fdf7f2
TL		 439
440 /* DOCSIS SEC BPI uses same settings as AES128 CBC */
f67539c2
TL		 441 memset(state->docsis_sec_ooo.lens, 0xFF,
442 sizeof(state->docsis_sec_ooo.lens));
443 memset(&state->docsis_sec_ooo.lens[0], 0,
444 sizeof(state->docsis_sec_ooo.lens[0]) * 4);
445 memset(state->docsis_sec_ooo.job_in_lane, 0,
446 sizeof(state->docsis_sec_ooo.job_in_lane));
11fdf7f2 447 state->docsis_sec_ooo.unused_lanes = 0xFF03020100;
f67539c2
TL		 448 state->docsis_sec_ooo.num_lanes_inuse = 0;
449
11fdf7f2
TL		 450
451 /* Init HMAC/SHA1 out-of-order fields */
452 state->hmac_sha_1_ooo.lens[0] = 0;
453 state->hmac_sha_1_ooo.lens[1] = 0;
454 state->hmac_sha_1_ooo.lens[2] = 0;
455 state->hmac_sha_1_ooo.lens[3] = 0;
456 state->hmac_sha_1_ooo.lens[4] = 0xFFFF;
457 state->hmac_sha_1_ooo.lens[5] = 0xFFFF;
458 state->hmac_sha_1_ooo.lens[6] = 0xFFFF;
459 state->hmac_sha_1_ooo.lens[7] = 0xFFFF;
460 state->hmac_sha_1_ooo.unused_lanes = 0xFF03020100;
461 for (j = 0; j < SSE_NUM_SHA1_LANES; j++) {
462 state->hmac_sha_1_ooo.ldata[j].job_in_lane = NULL;
463 state->hmac_sha_1_ooo.ldata[j].extra_block[64] = 0x80;
464 memset(state->hmac_sha_1_ooo.ldata[j].extra_block + 65,
465 0x00,
466 64+7);
467 p = state->hmac_sha_1_ooo.ldata[j].outer_block;
468 memset(p + 5*4 + 1,
469 0x00,
470 64 - 5*4 - 1 - 2);
471 p[5*4] = 0x80;
472 p[64-2] = 0x02;
473 p[64-1] = 0xA0;
474 }
475
476#ifdef HASH_USE_SHAEXT
477 if (state->features & IMB_FEATURE_SHANI) {
478 /* Init HMAC/SHA1 NI out-of-order fields */
479 state->hmac_sha_1_ooo.lens[0] = 0;
480 state->hmac_sha_1_ooo.lens[1] = 0;
481 state->hmac_sha_1_ooo.lens[2] = 0xFFFF;
482 state->hmac_sha_1_ooo.lens[3] = 0xFFFF;
483 state->hmac_sha_1_ooo.lens[4] = 0xFFFF;
484 state->hmac_sha_1_ooo.lens[5] = 0xFFFF;
485 state->hmac_sha_1_ooo.lens[6] = 0xFFFF;
486 state->hmac_sha_1_ooo.lens[7] = 0xFFFF;
487 state->hmac_sha_1_ooo.unused_lanes = 0xFF0100;
488 }
489#endif /* HASH_USE_SHAEXT */
490
491 /* Init HMAC/SHA224 out-of-order fields */
492 state->hmac_sha_224_ooo.lens[0] = 0;
493 state->hmac_sha_224_ooo.lens[1] = 0;
494 state->hmac_sha_224_ooo.lens[2] = 0;
495 state->hmac_sha_224_ooo.lens[3] = 0;
496 state->hmac_sha_224_ooo.lens[4] = 0xFFFF;
497 state->hmac_sha_224_ooo.lens[5] = 0xFFFF;
498 state->hmac_sha_224_ooo.lens[6] = 0xFFFF;
499 state->hmac_sha_224_ooo.lens[7] = 0xFFFF;
500 state->hmac_sha_224_ooo.unused_lanes = 0xFF03020100;
501 for (j = 0; j < SSE_NUM_SHA256_LANES; j++) {
502 state->hmac_sha_224_ooo.ldata[j].job_in_lane = NULL;
f67539c2
TL		 503
504 p = state->hmac_sha_224_ooo.ldata[j].extra_block;
505 size = sizeof(state->hmac_sha_224_ooo.ldata[j].extra_block);
506 memset (p, 0x00, size);
507 p[64] = 0x80;
508
11fdf7f2 509 p = state->hmac_sha_224_ooo.ldata[j].outer_block;
f67539c2
TL		 510 size = sizeof(state->hmac_sha_224_ooo.ldata[j].outer_block);
511 memset(p, 0x00, size);
11fdf7f2
TL		 512 p[7*4] = 0x80; /* digest 7 words long */
513 p[64-2] = 0x02; /* length in little endian = 0x02E0 */
514 p[64-1] = 0xE0;
515 }
516#ifdef HASH_USE_SHAEXT
517 if (state->features & IMB_FEATURE_SHANI) {
518 /* Init HMAC/SHA224 NI out-of-order fields */
519 state->hmac_sha_224_ooo.lens[0] = 0;
520 state->hmac_sha_224_ooo.lens[1] = 0;
521 state->hmac_sha_224_ooo.lens[2] = 0xFFFF;
522 state->hmac_sha_224_ooo.lens[3] = 0xFFFF;
523 state->hmac_sha_224_ooo.lens[4] = 0xFFFF;
524 state->hmac_sha_224_ooo.lens[5] = 0xFFFF;
525 state->hmac_sha_224_ooo.lens[6] = 0xFFFF;
526 state->hmac_sha_224_ooo.lens[7] = 0xFFFF;
527 state->hmac_sha_224_ooo.unused_lanes = 0xFF0100;
528 }
529#endif /* HASH_USE_SHAEXT */
530
531 /* Init HMAC/SHA_256 out-of-order fields */
532 state->hmac_sha_256_ooo.lens[0] = 0;
533 state->hmac_sha_256_ooo.lens[1] = 0;
534 state->hmac_sha_256_ooo.lens[2] = 0;
535 state->hmac_sha_256_ooo.lens[3] = 0;
536 state->hmac_sha_256_ooo.lens[4] = 0xFFFF;
537 state->hmac_sha_256_ooo.lens[5] = 0xFFFF;
538 state->hmac_sha_256_ooo.lens[6] = 0xFFFF;
539 state->hmac_sha_256_ooo.lens[7] = 0xFFFF;
540 state->hmac_sha_256_ooo.unused_lanes = 0xFF03020100;
541 for (j = 0; j < SSE_NUM_SHA256_LANES; j++) {
542 state->hmac_sha_256_ooo.ldata[j].job_in_lane = NULL;
543 state->hmac_sha_256_ooo.ldata[j].extra_block[64] = 0x80;
544 memset(state->hmac_sha_256_ooo.ldata[j].extra_block + 65,
545 0x00,
546 64+7);
547 p = state->hmac_sha_256_ooo.ldata[j].outer_block;
548 memset(p + 8*4 + 1,
549 0x00,
550 64 - 8*4 - 1 - 2); /* digest is 8*4 bytes long */
551 p[8*4] = 0x80;
552 p[64-2] = 0x03; /* length of (opad (64*8) bits + 256 bits)
553 * in hex is 0x300 */
554 p[64-1] = 0x00;
555 }
556#ifdef HASH_USE_SHAEXT
557 if (state->features & IMB_FEATURE_SHANI) {
558 /* Init HMAC/SHA256 NI out-of-order fields */
559 state->hmac_sha_256_ooo.lens[0] = 0;
560 state->hmac_sha_256_ooo.lens[1] = 0;
561 state->hmac_sha_256_ooo.lens[2] = 0xFFFF;
562 state->hmac_sha_256_ooo.lens[3] = 0xFFFF;
563 state->hmac_sha_256_ooo.lens[4] = 0xFFFF;
564 state->hmac_sha_256_ooo.lens[5] = 0xFFFF;
565 state->hmac_sha_256_ooo.lens[6] = 0xFFFF;
566 state->hmac_sha_256_ooo.lens[7] = 0xFFFF;
567 state->hmac_sha_256_ooo.unused_lanes = 0xFF0100;
568 }
569#endif /* HASH_USE_SHAEXT */
570
571 /* Init HMAC/SHA384 out-of-order fields */
572 state->hmac_sha_384_ooo.lens[0] = 0;
573 state->hmac_sha_384_ooo.lens[1] = 0;
574 state->hmac_sha_384_ooo.lens[2] = 0xFFFF;
575 state->hmac_sha_384_ooo.lens[3] = 0xFFFF;
576 state->hmac_sha_384_ooo.lens[4] = 0xFFFF;
577 state->hmac_sha_384_ooo.lens[5] = 0xFFFF;
578 state->hmac_sha_384_ooo.lens[6] = 0xFFFF;
579 state->hmac_sha_384_ooo.lens[7] = 0xFFFF;
580 state->hmac_sha_384_ooo.unused_lanes = 0xFF0100;
581 for (j = 0; j < SSE_NUM_SHA512_LANES; j++) {
582 MB_MGR_HMAC_SHA_512_OOO *ctx = &state->hmac_sha_384_ooo;
583
584 ctx->ldata[j].job_in_lane = NULL;
585 ctx->ldata[j].extra_block[SHA_384_BLOCK_SIZE] = 0x80;
586 memset(ctx->ldata[j].extra_block + (SHA_384_BLOCK_SIZE + 1),
587 0x00, SHA_384_BLOCK_SIZE + 7);
588
589 p = ctx->ldata[j].outer_block;
590 memset(p + SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00,
591 /* special end point because this length is constant */
592 SHA_384_BLOCK_SIZE -
593 SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2);
594 p[SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */
595 /*
596 * hmac outer block length always of fixed size, it is OKey
597 * length, a whole message block length, 1024 bits, with padding
598 * plus the length of the inner digest, which is 384 bits
599 * 1408 bits == 0x0580. The input message block needs to be
600 * converted to big endian within the sha implementation
601 * before use.
602 */
603 p[SHA_384_BLOCK_SIZE - 2] = 0x05;
604 p[SHA_384_BLOCK_SIZE - 1] = 0x80;
605 }
606
607 /* Init HMAC/SHA512 out-of-order fields */
608 state->hmac_sha_512_ooo.lens[0] = 0;
609 state->hmac_sha_512_ooo.lens[1] = 0;
610 state->hmac_sha_512_ooo.lens[2] = 0xFFFF;
611 state->hmac_sha_512_ooo.lens[3] = 0xFFFF;
612 state->hmac_sha_512_ooo.lens[4] = 0xFFFF;
613 state->hmac_sha_512_ooo.lens[5] = 0xFFFF;
614 state->hmac_sha_512_ooo.lens[6] = 0xFFFF;
615 state->hmac_sha_512_ooo.lens[7] = 0xFFFF;
616 state->hmac_sha_512_ooo.unused_lanes = 0xFF0100;
617 for (j = 0; j < SSE_NUM_SHA512_LANES; j++) {
618 MB_MGR_HMAC_SHA_512_OOO *ctx = &state->hmac_sha_512_ooo;
619
620 ctx->ldata[j].job_in_lane = NULL;
621 ctx->ldata[j].extra_block[SHA_512_BLOCK_SIZE] = 0x80;
622 memset(ctx->ldata[j].extra_block + (SHA_512_BLOCK_SIZE + 1),
623 0x00, SHA_512_BLOCK_SIZE + 7);
624
625 p = ctx->ldata[j].outer_block;
626 memset(p + SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00,
627 /* special end point because this length is constant */
628 SHA_512_BLOCK_SIZE -
629 SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2);
630 p[SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */
631 /*
632 * hmac outer block length always of fixed size, it is OKey
633 * length, a whole message block length, 1024 bits, with padding
634 * plus the length of the inner digest, which is 512 bits
635 * 1536 bits == 0x600. The input message block needs to be
636 * converted to big endian within the sha implementation
637 * before use.
638 */
639 p[SHA_512_BLOCK_SIZE - 2] = 0x06;
640 p[SHA_512_BLOCK_SIZE - 1] = 0x00;
641 }
642
643 /* Init HMAC/MD5 out-of-order fields */
644 state->hmac_md5_ooo.lens[0] = 0;
645 state->hmac_md5_ooo.lens[1] = 0;
646 state->hmac_md5_ooo.lens[2] = 0;
647 state->hmac_md5_ooo.lens[3] = 0;
648 state->hmac_md5_ooo.lens[4] = 0;
649 state->hmac_md5_ooo.lens[5] = 0;
650 state->hmac_md5_ooo.lens[6] = 0;
651 state->hmac_md5_ooo.lens[7] = 0;
652 state->hmac_md5_ooo.lens[8] = 0xFFFF;
653 state->hmac_md5_ooo.lens[9] = 0xFFFF;
654 state->hmac_md5_ooo.lens[10] = 0xFFFF;
655 state->hmac_md5_ooo.lens[11] = 0xFFFF;
656 state->hmac_md5_ooo.lens[12] = 0xFFFF;
657 state->hmac_md5_ooo.lens[13] = 0xFFFF;
658 state->hmac_md5_ooo.lens[14] = 0xFFFF;
659 state->hmac_md5_ooo.lens[15] = 0xFFFF;
660 state->hmac_md5_ooo.unused_lanes = 0xF76543210;
661 for (j = 0; j < SSE_NUM_MD5_LANES; j++) {
662 state->hmac_md5_ooo.ldata[j].job_in_lane = NULL;
f67539c2
TL		 663
664 p = state->hmac_md5_ooo.ldata[j].extra_block;
665 size = sizeof(state->hmac_md5_ooo.ldata[j].extra_block);
666 memset (p, 0x00, size);
667 p[64] = 0x80;
668
11fdf7f2 669 p = state->hmac_md5_ooo.ldata[j].outer_block;
f67539c2
TL		 670 size = sizeof(state->hmac_md5_ooo.ldata[j].outer_block);
671 memset(p, 0x00, size);
11fdf7f2
TL		 672 p[4*4] = 0x80;
673 p[64-7] = 0x02;
674 p[64-8] = 0x80;
675 }
676
677 /* Init AES/XCBC OOO fields */
678 state->aes_xcbc_ooo.lens[0] = 0;
679 state->aes_xcbc_ooo.lens[1] = 0;
680 state->aes_xcbc_ooo.lens[2] = 0;
681 state->aes_xcbc_ooo.lens[3] = 0;
682 state->aes_xcbc_ooo.lens[4] = 0xFFFF;
683 state->aes_xcbc_ooo.lens[5] = 0xFFFF;
684 state->aes_xcbc_ooo.lens[6] = 0xFFFF;
685 state->aes_xcbc_ooo.lens[7] = 0xFFFF;
686 state->aes_xcbc_ooo.unused_lanes = 0xFF03020100;
687 for (j = 0; j < 4; j++) {
688 state->aes_xcbc_ooo.ldata[j].job_in_lane = NULL;
689 state->aes_xcbc_ooo.ldata[j].final_block[16] = 0x80;
690 memset(state->aes_xcbc_ooo.ldata[j].final_block + 17, 0x00, 15);
691 }
692
693 /* Init AES-CCM auth out-of-order fields */
694 for (j = 0; j < 4; j++) {
695 state->aes_ccm_ooo.init_done[j] = 0;
696 state->aes_ccm_ooo.lens[j] = 0;
697 state->aes_ccm_ooo.job_in_lane[j] = NULL;
698 }
f67539c2
TL		 699 for (; j < 8; j++)
700 state->aes_ccm_ooo.lens[j] = 0xFFFF;
701
11fdf7f2
TL		 702 state->aes_ccm_ooo.unused_lanes = 0xF3210;
703
704 /* Init AES-CMAC auth out-of-order fields */
9f95a23c
TL		 705 state->aes_cmac_ooo.lens[0] = 0;
706 state->aes_cmac_ooo.lens[1] = 0;
707 state->aes_cmac_ooo.lens[2] = 0;
708 state->aes_cmac_ooo.lens[3] = 0;
709 state->aes_cmac_ooo.lens[4] = 0xFFFF;
710 state->aes_cmac_ooo.lens[5] = 0xFFFF;
711 state->aes_cmac_ooo.lens[6] = 0xFFFF;
712 state->aes_cmac_ooo.lens[7] = 0xFFFF;
11fdf7f2
TL		 713 for (j = 0; j < 4; j++) {
714 state->aes_cmac_ooo.init_done[j] = 0;
11fdf7f2
TL		 715 state->aes_cmac_ooo.job_in_lane[j] = NULL;
716 }
717 state->aes_cmac_ooo.unused_lanes = 0xF3210;
718
719 /* Init "in order" components */
720 state->next_job = 0;
721 state->earliest_job = -1;
722
723 /* set SSE handlers */
724 state->get_next_job = get_next_job_sse;
725 state->submit_job = submit_job_sse;
726 state->submit_job_nocheck = submit_job_nocheck_sse;
727 state->get_completed_job = get_completed_job_sse;
728 state->flush_job = flush_job_sse;
729 state->queue_size = queue_size_sse;
730 state->keyexp_128 = aes_keyexp_128_sse;
731 state->keyexp_192 = aes_keyexp_192_sse;
732 state->keyexp_256 = aes_keyexp_256_sse;
733 state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_sse;
734 state->xcbc_keyexp = aes_xcbc_expand_key_sse;
735 state->des_key_sched = des_key_schedule;
736 state->sha1_one_block = sha1_one_block_sse;
9f95a23c 737 state->sha1 = sha1_sse;
11fdf7f2 738 state->sha224_one_block = sha224_one_block_sse;
9f95a23c 739 state->sha224 = sha224_sse;
11fdf7f2 740 state->sha256_one_block = sha256_one_block_sse;
9f95a23c 741 state->sha256 = sha256_sse;
11fdf7f2 742 state->sha384_one_block = sha384_one_block_sse;
9f95a23c 743 state->sha384 = sha384_sse;
11fdf7f2 744 state->sha512_one_block = sha512_one_block_sse;
9f95a23c 745 state->sha512 = sha512_sse;
11fdf7f2 746 state->md5_one_block = md5_one_block_sse;
9f95a23c 747 state->aes128_cfb_one = aes_cfb_128_one_sse;
f67539c2
TL		 748
749 state->eea3_1_buffer = zuc_eea3_1_buffer_sse;
750 state->eea3_4_buffer = zuc_eea3_4_buffer_sse;
751 state->eea3_n_buffer = zuc_eea3_n_buffer_sse;
752 state->eia3_1_buffer = zuc_eia3_1_buffer_sse;
753
754 state->f8_1_buffer = kasumi_f8_1_buffer_sse;
755 state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_sse;
756 state->f8_2_buffer = kasumi_f8_2_buffer_sse;
757 state->f8_3_buffer = kasumi_f8_3_buffer_sse;
758 state->f8_4_buffer = kasumi_f8_4_buffer_sse;
759 state->f8_n_buffer = kasumi_f8_n_buffer_sse;
760 state->f9_1_buffer = kasumi_f9_1_buffer_sse;
761 state->f9_1_buffer_user = kasumi_f9_1_buffer_user_sse;
762 state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_sse;
763 state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_sse;
764 state->kasumi_key_sched_size = kasumi_key_sched_size_sse;
765
766 state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_sse;
767 state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_sse;
768 state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_sse;
769 state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_sse;
770 state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_sse;
771 state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_sse;
772 state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_sse;
773 state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_sse;
774 state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_sse;
775 state->snow3g_init_key_sched = snow3g_init_key_sched_sse;
776 state->snow3g_key_sched_size = snow3g_key_sched_size_sse;
777
9f95a23c
TL		 778#ifndef NO_GCM
779 state->gcm128_enc = aes_gcm_enc_128_sse;
780 state->gcm192_enc = aes_gcm_enc_192_sse;
781 state->gcm256_enc = aes_gcm_enc_256_sse;
782 state->gcm128_dec = aes_gcm_dec_128_sse;
783 state->gcm192_dec = aes_gcm_dec_192_sse;
784 state->gcm256_dec = aes_gcm_dec_256_sse;
785 state->gcm128_init = aes_gcm_init_128_sse;
786 state->gcm192_init = aes_gcm_init_192_sse;
787 state->gcm256_init = aes_gcm_init_256_sse;
788 state->gcm128_enc_update = aes_gcm_enc_128_update_sse;
789 state->gcm192_enc_update = aes_gcm_enc_192_update_sse;
790 state->gcm256_enc_update = aes_gcm_enc_256_update_sse;
791 state->gcm128_dec_update = aes_gcm_dec_128_update_sse;
792 state->gcm192_dec_update = aes_gcm_dec_192_update_sse;
793 state->gcm256_dec_update = aes_gcm_dec_256_update_sse;
794 state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_sse;
795 state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_sse;
796 state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_sse;
797 state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_sse;
798 state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_sse;
799 state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_sse;
800 state->gcm128_precomp = aes_gcm_precomp_128_sse;
801 state->gcm192_precomp = aes_gcm_precomp_192_sse;
802 state->gcm256_precomp = aes_gcm_precomp_256_sse;
803 state->gcm128_pre = aes_gcm_pre_128_sse;
804 state->gcm192_pre = aes_gcm_pre_192_sse;
805 state->gcm256_pre = aes_gcm_pre_256_sse;
806#endif
11fdf7f2
TL		 807}
808
809#include "mb_mgr_code.h"
Ceph