#include <stdlib.h>
#include <string.h>
+#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_sse
+
#include "intel-ipsec-mb.h"
+#include "include/kasumi_internal.h"
+#include "include/zuc_internal.h"
+#include "include/snow3g.h"
+
#include "save_xmms.h"
#include "asm.h"
#include "des.h"
JOB_AES_HMAC *job);
JOB_AES_HMAC *flush_job_aes_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state);
+JOB_AES_HMAC *submit_job_aes_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state,
+ JOB_AES_HMAC *job);
+
+JOB_AES_HMAC *flush_job_aes_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state);
+
+JOB_AES_HMAC *submit_job_aes_cntr_sse_no_aesni(JOB_AES_HMAC *job);
+
+JOB_AES_HMAC *submit_job_aes_cntr_bit_sse_no_aesni(JOB_AES_HMAC *job);
+
+#define SAVE_XMMS save_xmms
+#define RESTORE_XMMS restore_xmms
-#define SAVE_XMMS save_xmms
-#define RESTORE_XMMS restore_xmms
#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_sse_no_aesni
#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse_no_aesni
#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_sse_no_aesni
#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_sse_no_aesni
#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse_no_aesni
#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_sse_no_aesni
+#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_sse_no_aesni
+#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_sse_no_aesni
+#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_sse_no_aesni
+#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_sse_no_aesni
+#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_sse_no_aesni
+#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_sse_no_aesni
#define SUBMIT_JOB_HMAC submit_job_hmac_sse
#define FLUSH_JOB_HMAC flush_job_hmac_sse
#define SUBMIT_JOB_HMAC_NI submit_job_hmac_sse
#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse_no_aesni
#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse_no_aesni
-#define SUBMIT_JOB_AES128_CNTR submit_job_aes128_cntr_sse
-#define SUBMIT_JOB_AES192_CNTR submit_job_aes192_cntr_sse
-#define SUBMIT_JOB_AES256_CNTR submit_job_aes256_cntr_sse
+#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse_no_aesni
+#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse_no_aesni
#define AES_CBC_DEC_128 aes_cbc_dec_128_sse_no_aesni
#define AES_CBC_DEC_192 aes_cbc_dec_192_sse_no_aesni
#define AES_CNTR_192 aes_cntr_192_sse_no_aesni
#define AES_CNTR_256 aes_cntr_256_sse_no_aesni
+#define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse_no_aesni
+
+#define AES_ECB_ENC_128 aes_ecb_enc_128_sse_no_aesni
+#define AES_ECB_ENC_192 aes_ecb_enc_192_sse_no_aesni
+#define AES_ECB_ENC_256 aes_ecb_enc_256_sse_no_aesni
+#define AES_ECB_DEC_128 aes_ecb_dec_128_sse_no_aesni
+#define AES_ECB_DEC_192 aes_ecb_dec_192_sse_no_aesni
+#define AES_ECB_DEC_256 aes_ecb_dec_256_sse_no_aesni
+
+#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_sse_no_aesni
+#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_sse_no_aesni
+#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_sse_no_aesni
+#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_sse_no_aesni
+
#ifndef NO_GCM
#define AES_GCM_DEC_128 aes_gcm_dec_128_sse_no_aesni
#define AES_GCM_ENC_128 aes_gcm_enc_128_sse_no_aesni
#define AES_CFB_128_ONE aes_cfb_128_one_sse_no_aesni
-void aes128_cbc_mac_x4_no_aesni(AES_ARGS_x8 *args, uint64_t len);
+void aes128_cbc_mac_x4_no_aesni(AES_ARGS *args, uint64_t len);
#define AES128_CBC_MAC aes128_cbc_mac_x4_no_aesni
-#define FLUSH_JOB_AES_CCM_AUTH flush_job_aes_ccm_auth_arch
-#define SUBMIT_JOB_AES_CCM_AUTH submit_job_aes_ccm_auth_arch
-#define AES_CCM_MAX_JOBS 4
+#define FLUSH_JOB_AES_CCM_AUTH flush_job_aes_ccm_auth_sse_no_aesni
+#define SUBMIT_JOB_AES_CCM_AUTH submit_job_aes_ccm_auth_sse_no_aesni
#define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes_cmac_auth_sse_no_aesni
#define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes_cmac_auth_sse_no_aesni
}
#endif /* NO_GCM */
+IMB_DLL_LOCAL JOB_AES_HMAC *
+submit_job_aes_cntr_sse_no_aesni(JOB_AES_HMAC *job)
+{
+ if (16 == job->aes_key_len_in_bytes)
+ AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes,
+ job->iv,
+ job->aes_enc_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes,
+ job->iv_len_in_bytes);
+ else if (24 == job->aes_key_len_in_bytes)
+ AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes,
+ job->iv,
+ job->aes_enc_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes,
+ job->iv_len_in_bytes);
+ else /* assume 32 bytes */
+ AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes,
+ job->iv,
+ job->aes_enc_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bytes,
+ job->iv_len_in_bytes);
+
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
+IMB_DLL_LOCAL JOB_AES_HMAC *
+submit_job_aes_cntr_bit_sse_no_aesni(JOB_AES_HMAC *job)
+{
+ const uint64_t offset = job->cipher_start_src_offset_in_bytes;
+
+ if (16 == job->aes_key_len_in_bytes)
+ aes_cntr_bit_128_sse_no_aesni(job->src + offset,
+ job->iv,
+ job->aes_enc_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bits,
+ job->iv_len_in_bytes);
+ else if (24 == job->aes_key_len_in_bytes)
+ aes_cntr_bit_192_sse_no_aesni(job->src + offset,
+ job->iv,
+ job->aes_enc_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bits,
+ job->iv_len_in_bytes);
+ else /* assume 32 bytes */
+ aes_cntr_bit_256_sse_no_aesni(job->src + offset,
+ job->iv,
+ job->aes_enc_key_expanded,
+ job->dst,
+ job->msg_len_to_cipher_in_bits,
+ job->iv_len_in_bytes);
+
+ job->status |= STS_COMPLETED_AES;
+ return job;
+}
+
/* ====================================================================== */
void
{
unsigned int j;
uint8_t *p;
+ size_t size;
/* Init AES out-of-order fields */
- state->aes128_ooo.lens[0] = 0;
- state->aes128_ooo.lens[1] = 0;
- state->aes128_ooo.lens[2] = 0;
- state->aes128_ooo.lens[3] = 0;
- state->aes128_ooo.lens[4] = 0xFFFF;
- state->aes128_ooo.lens[5] = 0xFFFF;
- state->aes128_ooo.lens[6] = 0xFFFF;
- state->aes128_ooo.lens[7] = 0xFFFF;
+ memset(state->aes128_ooo.lens, 0xFF,
+ sizeof(state->aes128_ooo.lens));
+ memset(&state->aes128_ooo.lens[0], 0,
+ sizeof(state->aes128_ooo.lens[0]) * 4);
+ memset(state->aes128_ooo.job_in_lane, 0,
+ sizeof(state->aes128_ooo.job_in_lane));
state->aes128_ooo.unused_lanes = 0xFF03020100;
- state->aes128_ooo.job_in_lane[0] = NULL;
- state->aes128_ooo.job_in_lane[1] = NULL;
- state->aes128_ooo.job_in_lane[2] = NULL;
- state->aes128_ooo.job_in_lane[3] = NULL;
-
- state->aes192_ooo.lens[0] = 0;
- state->aes192_ooo.lens[1] = 0;
- state->aes192_ooo.lens[2] = 0;
- state->aes192_ooo.lens[3] = 0;
- state->aes192_ooo.lens[4] = 0xFFFF;
- state->aes192_ooo.lens[5] = 0xFFFF;
- state->aes192_ooo.lens[6] = 0xFFFF;
- state->aes192_ooo.lens[7] = 0xFFFF;
+ state->aes128_ooo.num_lanes_inuse = 0;
+
+
+ memset(state->aes192_ooo.lens, 0xFF,
+ sizeof(state->aes192_ooo.lens));
+ memset(&state->aes192_ooo.lens[0], 0,
+ sizeof(state->aes192_ooo.lens[0]) * 4);
+ memset(state->aes192_ooo.job_in_lane, 0,
+ sizeof(state->aes192_ooo.job_in_lane));
state->aes192_ooo.unused_lanes = 0xFF03020100;
- state->aes192_ooo.job_in_lane[0] = NULL;
- state->aes192_ooo.job_in_lane[1] = NULL;
- state->aes192_ooo.job_in_lane[2] = NULL;
- state->aes192_ooo.job_in_lane[3] = NULL;
-
- state->aes256_ooo.lens[0] = 0;
- state->aes256_ooo.lens[1] = 0;
- state->aes256_ooo.lens[2] = 0;
- state->aes256_ooo.lens[3] = 0;
- state->aes256_ooo.lens[4] = 0xFFFF;
- state->aes256_ooo.lens[5] = 0xFFFF;
- state->aes256_ooo.lens[6] = 0xFFFF;
- state->aes256_ooo.lens[7] = 0xFFFF;
+ state->aes192_ooo.num_lanes_inuse = 0;
+
+
+ memset(state->aes256_ooo.lens, 0xFF,
+ sizeof(state->aes256_ooo.lens));
+ memset(&state->aes256_ooo.lens[0], 0,
+ sizeof(state->aes256_ooo.lens[0]) * 4);
+ memset(state->aes256_ooo.job_in_lane, 0,
+ sizeof(state->aes256_ooo.job_in_lane));
state->aes256_ooo.unused_lanes = 0xFF03020100;
- state->aes256_ooo.job_in_lane[0] = NULL;
- state->aes256_ooo.job_in_lane[1] = NULL;
- state->aes256_ooo.job_in_lane[2] = NULL;
- state->aes256_ooo.job_in_lane[3] = NULL;
+ state->aes256_ooo.num_lanes_inuse = 0;
+
/* DOCSIS SEC BPI uses same settings as AES128 CBC */
- state->docsis_sec_ooo.lens[0] = 0;
- state->docsis_sec_ooo.lens[1] = 0;
- state->docsis_sec_ooo.lens[2] = 0;
- state->docsis_sec_ooo.lens[3] = 0;
- state->docsis_sec_ooo.lens[4] = 0xFFFF;
- state->docsis_sec_ooo.lens[5] = 0xFFFF;
- state->docsis_sec_ooo.lens[6] = 0xFFFF;
- state->docsis_sec_ooo.lens[7] = 0xFFFF;
+ memset(state->docsis_sec_ooo.lens, 0xFF,
+ sizeof(state->docsis_sec_ooo.lens));
+ memset(&state->docsis_sec_ooo.lens[0], 0,
+ sizeof(state->docsis_sec_ooo.lens[0]) * 4);
+ memset(state->docsis_sec_ooo.job_in_lane, 0,
+ sizeof(state->docsis_sec_ooo.job_in_lane));
state->docsis_sec_ooo.unused_lanes = 0xFF03020100;
- state->docsis_sec_ooo.job_in_lane[0] = NULL;
- state->docsis_sec_ooo.job_in_lane[1] = NULL;
- state->docsis_sec_ooo.job_in_lane[2] = NULL;
- state->docsis_sec_ooo.job_in_lane[3] = NULL;
+ state->docsis_sec_ooo.num_lanes_inuse = 0;
+
/* Init HMAC/SHA1 out-of-order fields */
state->hmac_sha_1_ooo.lens[0] = 0;
state->hmac_sha_224_ooo.unused_lanes = 0xFF03020100;
for (j = 0; j < SSE_NUM_SHA256_LANES; j++) {
state->hmac_sha_224_ooo.ldata[j].job_in_lane = NULL;
- state->hmac_sha_224_ooo.ldata[j].extra_block[64] = 0x80;
- memset(state->hmac_sha_224_ooo.ldata[j].extra_block + 65,
- 0x00,
- 64+7);
+
+ p = state->hmac_sha_224_ooo.ldata[j].extra_block;
+ size = sizeof(state->hmac_sha_224_ooo.ldata[j].extra_block);
+ memset (p, 0x00, size);
+ p[64] = 0x80;
+
p = state->hmac_sha_224_ooo.ldata[j].outer_block;
- memset(p + 8*4 + 1,
- 0x00,
- 64 - 8*4 - 1 - 2);
+ size = sizeof(state->hmac_sha_224_ooo.ldata[j].outer_block);
+ memset(p, 0x00, size);
p[7*4] = 0x80; /* digest 7 words long */
p[64-2] = 0x02; /* length in little endian = 0x02E0 */
p[64-1] = 0xE0;
state->hmac_md5_ooo.unused_lanes = 0xF76543210;
for (j = 0; j < SSE_NUM_MD5_LANES; j++) {
state->hmac_md5_ooo.ldata[j].job_in_lane = NULL;
- state->hmac_md5_ooo.ldata[j].extra_block[64] = 0x80;
- memset(state->hmac_md5_ooo.ldata[j].extra_block + 65,
- 0x00,
- 64 + 7);
+
+ p = state->hmac_md5_ooo.ldata[j].extra_block;
+ size = sizeof(state->hmac_md5_ooo.ldata[j].extra_block);
+ memset (p, 0x00, size);
+ p[64] = 0x80;
+
p = state->hmac_md5_ooo.ldata[j].outer_block;
- memset(p + (5 * 4) + 1,
- 0x00,
- 64 - (5 * 4) - 1 - 2);
+ size = sizeof(state->hmac_md5_ooo.ldata[j].outer_block);
+ memset(p, 0x00, size);
p[4*4] = 0x80;
p[64-7] = 0x02;
p[64-8] = 0x80;
}
/* Init AES-CCM auth out-of-order fields */
- for (j = 0; j < 4; j++) {
- state->aes_ccm_ooo.init_done[j] = 0;
- state->aes_ccm_ooo.lens[j] = 0;
- state->aes_ccm_ooo.job_in_lane[j] = NULL;
- }
+ memset(&state->aes_ccm_ooo, 0, sizeof(MB_MGR_CCM_OOO));
+ for (j = 4; j < 8; j++)
+ state->aes_ccm_ooo.lens[j] = 0xFFFF;
state->aes_ccm_ooo.unused_lanes = 0xF3210;
/* Init AES-CMAC auth out-of-order fields */
state->sha512 = sha512_sse;
state->md5_one_block = md5_one_block_sse;
state->aes128_cfb_one = aes_cfb_128_one_sse_no_aesni;
+
+ state->eea3_1_buffer = zuc_eea3_1_buffer_sse;
+ state->eea3_4_buffer = zuc_eea3_4_buffer_sse;
+ state->eea3_n_buffer = zuc_eea3_n_buffer_sse;
+ state->eia3_1_buffer = zuc_eia3_1_buffer_sse;
+
+ state->f8_1_buffer = kasumi_f8_1_buffer_sse;
+ state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_sse;
+ state->f8_2_buffer = kasumi_f8_2_buffer_sse;
+ state->f8_3_buffer = kasumi_f8_3_buffer_sse;
+ state->f8_4_buffer = kasumi_f8_4_buffer_sse;
+ state->f8_n_buffer = kasumi_f8_n_buffer_sse;
+ state->f9_1_buffer = kasumi_f9_1_buffer_sse;
+ state->f9_1_buffer_user = kasumi_f9_1_buffer_user_sse;
+ state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_sse;
+ state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_sse;
+ state->kasumi_key_sched_size = kasumi_key_sched_size_sse;
+
+ state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_sse_no_aesni;
+ state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_sse_no_aesni;
+ state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_sse_no_aesni;
+ state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_sse_no_aesni;
+ state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_sse_no_aesni;
+ state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_sse_no_aesni;
+ state->snow3g_f8_8_buffer_multikey =
+ snow3g_f8_8_buffer_multikey_sse_no_aesni;
+ state->snow3g_f8_n_buffer_multikey =
+ snow3g_f8_n_buffer_multikey_sse_no_aesni;
+ state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_sse_no_aesni;
+ state->snow3g_init_key_sched = snow3g_init_key_sched_sse_no_aesni;
+ state->snow3g_key_sched_size = snow3g_key_sched_size_sse_no_aesni;
+
+#ifndef NO_GCM
+ state->gcm128_enc = aes_gcm_enc_128_sse_no_aesni;
+ state->gcm192_enc = aes_gcm_enc_192_sse_no_aesni;
+ state->gcm256_enc = aes_gcm_enc_256_sse_no_aesni;
+ state->gcm128_dec = aes_gcm_dec_128_sse_no_aesni;
+ state->gcm192_dec = aes_gcm_dec_192_sse_no_aesni;
+ state->gcm256_dec = aes_gcm_dec_256_sse_no_aesni;
+ state->gcm128_init = aes_gcm_init_128_sse_no_aesni;
+ state->gcm192_init = aes_gcm_init_192_sse_no_aesni;
+ state->gcm256_init = aes_gcm_init_256_sse_no_aesni;
+ state->gcm128_enc_update = aes_gcm_enc_128_update_sse_no_aesni;
+ state->gcm192_enc_update = aes_gcm_enc_192_update_sse_no_aesni;
+ state->gcm256_enc_update = aes_gcm_enc_256_update_sse_no_aesni;
+ state->gcm128_dec_update = aes_gcm_dec_128_update_sse_no_aesni;
+ state->gcm192_dec_update = aes_gcm_dec_192_update_sse_no_aesni;
+ state->gcm256_dec_update = aes_gcm_dec_256_update_sse_no_aesni;
+ state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_sse_no_aesni;
+ state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_sse_no_aesni;
+ state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_sse_no_aesni;
+ state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_sse_no_aesni;
+ state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_sse_no_aesni;
+ state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_sse_no_aesni;
+ state->gcm128_precomp = aes_gcm_precomp_128_sse_no_aesni;
+ state->gcm192_precomp = aes_gcm_precomp_192_sse_no_aesni;
+ state->gcm256_precomp = aes_gcm_precomp_256_sse_no_aesni;
+ state->gcm128_pre = aes_gcm_pre_128_sse_no_aesni;
+ state->gcm192_pre = aes_gcm_pre_192_sse_no_aesni;
+ state->gcm256_pre = aes_gcm_pre_256_sse_no_aesni;
+#endif
}
#include "mb_mgr_code.h"
projects / ceph.git / blobdiff