ceph/src/spdk/intel-ipsec-mb/intel-ipsec-mb.h

   1 /*******************************************************************************
   2   Copyright (c) 2012-2018, Intel Corporation
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions are met:
   6
   7       * Redistributions of source code must retain the above copyright notice,
   8         this list of conditions and the following disclaimer.
   9       * Redistributions in binary form must reproduce the above copyright
  10         notice, this list of conditions and the following disclaimer in the
  11         documentation and/or other materials provided with the distribution.
  12       * Neither the name of Intel Corporation nor the names of its contributors
  13         may be used to endorse or promote products derived from this software
  14         without specific prior written permission.
  15
  16   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19   DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23   CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 *******************************************************************************/
  27
  28 #ifndef IMB_IPSEC_MB_H
  29 #define IMB_IPSEC_MB_H
  30
  31 #include <stdlib.h>
  32 #include <stdint.h>
  33
  34 #ifdef __cplusplus
  35 extern "C" {
  36 #endif
  37
  38 /* 128-bit data type that is not in sdtint.h */
  39 typedef struct {
  40         uint64_t low;
  41         uint64_t high;
  42 } uint128_t;
  43
  44 /*
  45  * Macros for aligning data structures and function inlines
  46  */
  47 #ifdef __linux__
  48 /* Linux */
  49 #define DECLARE_ALIGNED(decl, alignval) \
  50         decl __attribute__((aligned(alignval)))
  51 #define __forceinline \
  52         static inline __attribute__((always_inline))
  53
  54 #if __GNUC__ >= 4
  55 #define IMB_DLL_EXPORT __attribute__((visibility("default")))
  56 #define IMB_DLL_LOCAL  __attribute__((visibility("hidden")))
  57 #else /* GNU C 4.0 and later */
  58 #define IMB_DLL_EXPORT
  59 #define IMB_DLL_LOCAL
  60 #endif /* different C compiler */
  61
  62 #else
  63 /* Windows */
  64 #define DECLARE_ALIGNED(decl, alignval) \
  65         __declspec(align(alignval)) decl
  66 #define __forceinline \
  67         static __forceinline
  68
  69 /* Windows DLL export is done via DEF file */
  70 #define IMB_DLL_EXPORT
  71 #define IMB_DLL_LOCAL
  72 #endif
  73
  74 /* Library version */
  75 #define IMB_VERSION_STR "0.52.0"
  76 #define IMB_VERSION_NUM 0x3400
  77
  78 /* Macro to translate version number */
  79 #define IMB_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))
  80
  81 /*
  82  * Custom ASSERT and DIM macros
  83  */
  84 #ifdef DEBUG
  85 #include <assert.h>
  86 #define IMB_ASSERT(x) assert(x)
  87 #else
  88 #define IMB_ASSERT(x)
  89 #endif
  90
  91 #ifndef IMB_DIM
  92 #define IMB_DIM(x) (sizeof(x) / sizeof(x[0]))
  93 #endif
  94
  95 /*
  96  * Algorithm constants
  97  */
  98
  99 #define DES_KEY_SCHED_SIZE (16 * 8) /* 16 rounds x 8 bytes */
 100 #define DES_BLOCK_SIZE 8
 101
 102 #define AES_BLOCK_SIZE 16
 103
 104 #define NUM_MD5_DIGEST_WORDS     4
 105 #define NUM_SHA_DIGEST_WORDS     5
 106 #define NUM_SHA_256_DIGEST_WORDS 8
 107 #define NUM_SHA_224_DIGEST_WORDS 7
 108 #define NUM_SHA_512_DIGEST_WORDS 8
 109 #define NUM_SHA_384_DIGEST_WORDS 6
 110
 111 #define SHA_DIGEST_WORD_SIZE      4
 112 #define SHA224_DIGEST_WORD_SIZE   4
 113 #define SHA256_DIGEST_WORD_SIZE   4
 114 #define SHA384_DIGEST_WORD_SIZE   8
 115 #define SHA512_DIGEST_WORD_SIZE   8
 116
 117 #define SHA1_DIGEST_SIZE_IN_BYTES \
 118         (NUM_SHA_DIGEST_WORDS * SHA_DIGEST_WORD_SIZE)
 119 #define SHA224_DIGEST_SIZE_IN_BYTES \
 120         (NUM_SHA_224_DIGEST_WORDS * SHA224_DIGEST_WORD_SIZE)
 121 #define SHA256_DIGEST_SIZE_IN_BYTES \
 122         (NUM_SHA_256_DIGEST_WORDS * SHA256_DIGEST_WORD_SIZE)
 123 #define SHA384_DIGEST_SIZE_IN_BYTES \
 124         (NUM_SHA_384_DIGEST_WORDS * SHA384_DIGEST_WORD_SIZE)
 125 #define SHA512_DIGEST_SIZE_IN_BYTES \
 126         (NUM_SHA_512_DIGEST_WORDS * SHA512_DIGEST_WORD_SIZE)
 127
 128 #define SHA1_BLOCK_SIZE 64    /* 512 bits is 64 byte blocks */
 129 #define SHA_256_BLOCK_SIZE 64 /* 512 bits is 64 byte blocks */
 130 #define SHA_384_BLOCK_SIZE 128
 131 #define SHA_512_BLOCK_SIZE 128
 132
 133 /* Number of lanes AVX512, AVX2, AVX and SSE */
 134 #define AVX512_NUM_SHA1_LANES   16
 135 #define AVX512_NUM_SHA256_LANES 16
 136 #define AVX512_NUM_SHA512_LANES 8
 137 #define AVX512_NUM_MD5_LANES    32
 138 #define AVX512_NUM_DES_LANES    16
 139
 140 #define AVX2_NUM_SHA1_LANES     8
 141 #define AVX2_NUM_SHA256_LANES   8
 142 #define AVX2_NUM_SHA512_LANES   4
 143 #define AVX2_NUM_MD5_LANES      16
 144
 145 #define AVX_NUM_SHA1_LANES      4
 146 #define AVX_NUM_SHA256_LANES    4
 147 #define AVX_NUM_SHA512_LANES    2
 148 #define AVX_NUM_MD5_LANES       8
 149
 150 #define SSE_NUM_SHA1_LANES   AVX_NUM_SHA1_LANES
 151 #define SSE_NUM_SHA256_LANES AVX_NUM_SHA256_LANES
 152 #define SSE_NUM_SHA512_LANES AVX_NUM_SHA512_LANES
 153 #define SSE_NUM_MD5_LANES    AVX_NUM_MD5_LANES
 154
 155 /*
 156  *  Each row is sized to hold enough lanes for AVX2, AVX1 and SSE use a subset
 157  * of each row. Thus one row is not adjacent in memory to its neighboring rows
 158  * in the case of SSE and AVX1.
 159  */
 160 #define MD5_DIGEST_SZ    (NUM_MD5_DIGEST_WORDS * AVX512_NUM_MD5_LANES)
 161 #define SHA1_DIGEST_SZ   (NUM_SHA_DIGEST_WORDS * AVX512_NUM_SHA1_LANES)
 162 #define SHA256_DIGEST_SZ (NUM_SHA_256_DIGEST_WORDS * AVX512_NUM_SHA256_LANES)
 163 #define SHA512_DIGEST_SZ (NUM_SHA_512_DIGEST_WORDS * AVX512_NUM_SHA512_LANES)
 164
 165 /*
 166  * Job structure definitions
 167  */
 168
 169 typedef enum {
 170         STS_BEING_PROCESSED = 0,
 171         STS_COMPLETED_AES =   1,
 172         STS_COMPLETED_HMAC =  2,
 173         STS_COMPLETED =       3, /* COMPLETED_AES | COMPLETED_HMAC */
 174         STS_INVALID_ARGS =    4,
 175         STS_INTERNAL_ERROR,
 176         STS_ERROR
 177 } JOB_STS;
 178
 179 typedef enum {
 180         CBC = 1,
 181         CNTR,
 182         NULL_CIPHER,
 183         DOCSIS_SEC_BPI,
 184 #ifndef NO_GCM
 185         GCM,
 186 #endif /* !NO_GCM */
 187         CUSTOM_CIPHER,
 188         DES,
 189         DOCSIS_DES,
 190         CCM,
 191         DES3
 192 } JOB_CIPHER_MODE;
 193
 194 typedef enum {
 195         ENCRYPT = 1,
 196         DECRYPT
 197 } JOB_CIPHER_DIRECTION;
 198
 199 typedef enum {
 200         SHA1 = 1,  /* HMAC-SHA1 */
 201         SHA_224,   /* HMAC-SHA224 */
 202         SHA_256,   /* HMAC-SHA256 */
 203         SHA_384,   /* HMAC-SHA384 */
 204         SHA_512,   /* HMAC-SHA512 */
 205         AES_XCBC,
 206         MD5,       /* HMAC-MD5 */
 207         NULL_HASH,
 208 #ifndef NO_GCM
 209         AES_GMAC,
 210 #endif /* !NO_GCM */
 211         CUSTOM_HASH,
 212         AES_CCM,         /* AES128-CCM */
 213         AES_CMAC,        /* AES128-CMAC */
 214         PLAIN_SHA1,      /* SHA1 */
 215         PLAIN_SHA_224,   /* SHA224 */
 216         PLAIN_SHA_256,   /* SHA256 */
 217         PLAIN_SHA_384,   /* SHA384 */
 218         PLAIN_SHA_512,   /* SHA512 */
 219 } JOB_HASH_ALG;
 220
 221 typedef enum {
 222         CIPHER_HASH = 1,
 223         HASH_CIPHER
 224 } JOB_CHAIN_ORDER;
 225
 226 typedef enum {
 227         AES_128_BYTES = 16,
 228         AES_192_BYTES = 24,
 229         AES_256_BYTES = 32
 230 } AES_KEY_SIZE_BYTES;
 231
 232 typedef struct JOB_AES_HMAC {
 233         /*
 234          * For AES, aes_enc_key_expanded and aes_dec_key_expanded are
 235          * expected to point to expanded keys structure.
 236          * - AES-CTR and AES-CCM, only aes_enc_key_expanded is used
 237          * - DOCSIS (AES-CBC + AES-CFB), both pointers are used
 238          *   aes_enc_key_expanded has to be set always for the partial block
 239          *
 240          * For DES, aes_enc_key_expanded and aes_dec_key_expanded are
 241          * expected to point to DES key schedule.
 242          * - same key schedule used for enc and dec operations
 243          *
 244          * For 3DES, aes_enc_key_expanded and aes_dec_key_expanded are
 245          * expected to point to an array of 3 pointers for
 246          * the corresponding 3 key schedules.
 247          * - same key schedule used for enc and dec operations
 248          */
 249         const void *aes_enc_key_expanded;  /* 16-byte aligned pointer. */
 250         const void *aes_dec_key_expanded;
 251         uint64_t aes_key_len_in_bytes; /* 16, 24 and 32 byte (128, 192 and
 252                                         * 256-bit) keys supported */
 253         const uint8_t *src; /* Input. May be cipher text or plaintext.
 254                              * In-place ciphering allowed. */
 255         uint8_t *dst; /*Output. May be cipher text or plaintext.
 256                        * In-place ciphering allowed, i.e. dst = src. */
 257         uint64_t cipher_start_src_offset_in_bytes;
 258         uint64_t msg_len_to_cipher_in_bytes; /* Max len = 65472 bytes.
 259                                               * IPSec case, the maximum cipher
 260                                               * length would be:
 261                                               * 65535 -
 262                                               * 20 (outer IP header) -
 263                                               * 24 (ESP header + IV) -
 264                                               * 12 (supported ICV length) */
 265         uint64_t hash_start_src_offset_in_bytes;
 266         uint64_t msg_len_to_hash_in_bytes; /* Max len = 65496 bytes.
 267                                             * (Max cipher len +
 268                                             * 24 bytes ESP header) */
 269         const uint8_t *iv; /* AES IV. */
 270         uint64_t iv_len_in_bytes; /* AES IV length in bytes. */
 271         uint8_t *auth_tag_output; /* HMAC Tag output. This may point to
 272                                    * a location in the src buffer
 273                                    * (for in place)*/
 274         uint64_t auth_tag_output_len_in_bytes; /* Authentication (i.e. HMAC) tag
 275                                                 * output length in bytes
 276                                                 * (may be a truncated value) */
 277
 278         /* Start algorithm-specific fields */
 279         union {
 280                 struct _HMAC_specific_fields {
 281                         /* Hashed result of HMAC key xor'd with ipad (0x36). */
 282                         const uint8_t *_hashed_auth_key_xor_ipad;
 283                         /* Hashed result of HMAC key xor'd with opad (0x5c). */
 284                         const uint8_t *_hashed_auth_key_xor_opad;
 285                 } HMAC;
 286                 struct _AES_XCBC_specific_fields {
 287                         /* 16-byte aligned pointers */
 288                         const uint32_t *_k1_expanded;
 289                         const uint8_t *_k2;
 290                         const uint8_t *_k3;
 291                 } XCBC;
 292                 struct _AES_CCM_specific_fields {
 293                         /* Additional Authentication Data (AAD) */
 294                         const void *aad;
 295                         uint64_t aad_len_in_bytes; /* Length of AAD */
 296                 } CCM;
 297                 struct _AES_CMAC_specific_fields {
 298                         const void *_key_expanded; /* 16-byte aligned */
 299                         const void *_skey1;
 300                         const void *_skey2;
 301                 } CMAC;
 302 #ifndef NO_GCM
 303                 struct _AES_GCM_specific_fields {
 304                         /* Additional Authentication Data (AAD) */
 305                         const void *aad;
 306                         uint64_t aad_len_in_bytes;    /* Length of AAD */
 307                 } GCM;
 308 #endif /* !NO_GCM */
 309         } u;
 310
 311         JOB_STS status;
 312         JOB_CIPHER_MODE cipher_mode; /* CBC, CNTR, DES, GCM etc. */
 313         JOB_CIPHER_DIRECTION cipher_direction; /* Encrypt/decrypt */
 314         /* Ignored as the direction is implied by the chain _order field. */
 315         JOB_HASH_ALG hash_alg; /* SHA-1 or others... */
 316         JOB_CHAIN_ORDER chain_order; /* CIPHER_HASH or HASH_CIPHER */
 317
 318         void *user_data;
 319         void *user_data2;
 320
 321         /*
 322          * stateless custom cipher and hash
 323          *   Return:
 324          *     success: 0
 325          *     fail:    other
 326          */
 327         int (*cipher_func)(struct JOB_AES_HMAC *);
 328         int (*hash_func)(struct JOB_AES_HMAC *);
 329 } JOB_AES_HMAC;
 330
 331 /*
 332  * Argument structures for various algorithms
 333  */
 334 typedef struct {
 335         const uint8_t *in[8];
 336         uint8_t *out[8];
 337         const uint32_t *keys[8];
 338         DECLARE_ALIGNED(uint128_t IV[8], 32);
 339 } AES_ARGS_x8;
 340
 341 typedef struct {
 342         DECLARE_ALIGNED(uint32_t digest[SHA1_DIGEST_SZ], 32);
 343         uint8_t *data_ptr[AVX512_NUM_SHA1_LANES];
 344 } SHA1_ARGS;
 345
 346 typedef struct {
 347         DECLARE_ALIGNED(uint32_t digest[SHA256_DIGEST_SZ], 32);
 348         uint8_t *data_ptr[AVX512_NUM_SHA256_LANES];
 349 } SHA256_ARGS;
 350
 351 typedef struct {
 352         DECLARE_ALIGNED(uint64_t digest[SHA512_DIGEST_SZ], 32);
 353         uint8_t *data_ptr[AVX512_NUM_SHA512_LANES];
 354 }  SHA512_ARGS;
 355
 356 typedef struct {
 357         DECLARE_ALIGNED(uint32_t digest[MD5_DIGEST_SZ], 32);
 358         uint8_t *data_ptr[AVX512_NUM_MD5_LANES];
 359 } MD5_ARGS;
 360
 361 typedef struct {
 362         const uint8_t *in[8];
 363         const uint32_t *keys[8];
 364         DECLARE_ALIGNED(uint128_t ICV[8], 32);
 365 } AES_XCBC_ARGS_x8;
 366
 367 typedef struct {
 368         const uint8_t *in[AVX512_NUM_DES_LANES];
 369         uint8_t *out[AVX512_NUM_DES_LANES];
 370         const uint8_t *keys[AVX512_NUM_DES_LANES];
 371         uint32_t IV[AVX512_NUM_DES_LANES * 2]; /* uint32_t is more handy here */
 372         uint32_t partial_len[AVX512_NUM_DES_LANES];
 373         uint32_t block_len[AVX512_NUM_DES_LANES];
 374         const uint8_t *last_in[AVX512_NUM_DES_LANES];
 375         uint8_t *last_out[AVX512_NUM_DES_LANES];
 376 } DES_ARGS_x16;
 377
 378 /* AES out-of-order scheduler fields */
 379 typedef struct {
 380         AES_ARGS_x8 args;
 381         DECLARE_ALIGNED(uint16_t lens[8], 16);
 382         /* each nibble is index (0...7) of an unused lane,
 383          * the last nibble is set to F as a flag
 384          */
 385         uint64_t unused_lanes;
 386         JOB_AES_HMAC *job_in_lane[8];
 387 } MB_MGR_AES_OOO;
 388
 389 /* AES XCBC out-of-order scheduler fields */
 390 typedef struct {
 391         DECLARE_ALIGNED(uint8_t final_block[2 * 16], 32);
 392         JOB_AES_HMAC *job_in_lane;
 393         uint64_t final_done;
 394 } XCBC_LANE_DATA;
 395
 396 typedef struct {
 397         AES_XCBC_ARGS_x8 args;
 398         DECLARE_ALIGNED(uint16_t lens[8], 16);
 399         /* each byte is index (0...3) of unused lanes
 400          * byte 4 is set to FF as a flag
 401          */
 402         uint64_t unused_lanes;
 403         XCBC_LANE_DATA ldata[8];
 404 } MB_MGR_AES_XCBC_OOO;
 405
 406 /* AES-CCM out-of-order scheduler structure */
 407 typedef struct {
 408         AES_ARGS_x8 args; /* need to re-use AES arguments */
 409         DECLARE_ALIGNED(uint16_t lens[8], 16);
 410         DECLARE_ALIGNED(uint16_t init_done[8], 16);
 411         /* each byte is index (0...3) of unused lanes
 412          * byte 4 is set to FF as a flag
 413          */
 414         uint64_t unused_lanes;
 415         JOB_AES_HMAC *job_in_lane[8];
 416         DECLARE_ALIGNED(uint8_t init_blocks[8 * (4 * 16)], 32);
 417 } MB_MGR_CCM_OOO;
 418
 419
 420 /* AES-CMAC out-of-order scheduler structure */
 421 typedef struct {
 422         AES_ARGS_x8 args; /* need to re-use AES arguments */
 423         DECLARE_ALIGNED(uint16_t lens[8], 16);
 424         DECLARE_ALIGNED(uint16_t init_done[8], 16);
 425         /* each byte is index (0...3) of unused lanes
 426          * byte 4 is set to FF as a flag
 427          */
 428         uint64_t unused_lanes;
 429         JOB_AES_HMAC *job_in_lane[8];
 430         DECLARE_ALIGNED(uint8_t scratch[8 * 16], 32);
 431 } MB_MGR_CMAC_OOO;
 432
 433
 434 /* DES out-of-order scheduler fields */
 435 typedef struct {
 436         DES_ARGS_x16 args;
 437         DECLARE_ALIGNED(uint16_t lens[16], 16);
 438         /* each nibble is index (0...7) of unused lanes
 439          * nibble 8 is set to F as a flag
 440          */
 441         uint64_t unused_lanes;
 442         JOB_AES_HMAC *job_in_lane[16];
 443         uint32_t num_lanes_inuse;
 444 } MB_MGR_DES_OOO;
 445
 446
 447 /* HMAC-SHA1 and HMAC-SHA256/224 */
 448 typedef struct {
 449         /* YMM aligned access to extra_block */
 450         DECLARE_ALIGNED(uint8_t extra_block[2 * SHA1_BLOCK_SIZE+8], 32);
 451         JOB_AES_HMAC *job_in_lane;
 452         uint8_t outer_block[64];
 453         uint32_t outer_done;
 454         uint32_t extra_blocks; /* num extra blocks (1 or 2) */
 455         uint32_t size_offset;  /* offset in extra_block to start of
 456                                 * size field */
 457         uint32_t start_offset; /* offset to start of data */
 458 } HMAC_SHA1_LANE_DATA;
 459
 460 /* HMAC-SHA512/384 */
 461 typedef struct {
 462         DECLARE_ALIGNED(uint8_t extra_block[2 * SHA_512_BLOCK_SIZE + 16], 32);
 463         uint8_t outer_block[SHA_512_BLOCK_SIZE];
 464         JOB_AES_HMAC *job_in_lane;
 465         uint32_t outer_done;
 466         uint32_t extra_blocks; /* num extra blocks (1 or 2) */
 467         uint32_t size_offset;  /* offset in extra_block to start of
 468                                 * size field */
 469         uint32_t start_offset; /* offset to start of data */
 470 } HMAC_SHA512_LANE_DATA;
 471
 472 /*
 473  * unused_lanes contains a list of unused lanes stored as bytes or as
 474  * nibbles depending on the arch. The end of list is either FF or F.
 475  */
 476 typedef struct {
 477         SHA1_ARGS args;
 478         DECLARE_ALIGNED(uint16_t lens[16], 32);
 479         uint64_t unused_lanes;
 480         HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_SHA1_LANES];
 481         uint32_t num_lanes_inuse;
 482 } MB_MGR_HMAC_SHA_1_OOO;
 483
 484 typedef struct {
 485         SHA256_ARGS args;
 486         DECLARE_ALIGNED(uint16_t lens[16], 16);
 487         uint64_t unused_lanes;
 488         HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_SHA256_LANES];
 489         uint32_t num_lanes_inuse;
 490 } MB_MGR_HMAC_SHA_256_OOO;
 491
 492 typedef struct {
 493         SHA512_ARGS args;
 494         DECLARE_ALIGNED(uint16_t lens[8], 16);
 495         uint64_t unused_lanes;
 496         HMAC_SHA512_LANE_DATA ldata[AVX512_NUM_SHA512_LANES];
 497 } MB_MGR_HMAC_SHA_512_OOO;
 498
 499 /* MD5-HMAC out-of-order scheduler fields */
 500 typedef struct {
 501         MD5_ARGS args;
 502         DECLARE_ALIGNED(uint16_t lens[AVX512_NUM_MD5_LANES], 16);
 503         /*
 504          * In the avx2 case, all 16 nibbles of unused lanes are used.
 505          * In that case num_lanes_inuse is used to detect the end of the list
 506          */
 507         uint64_t unused_lanes;
 508         HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_MD5_LANES];
 509         uint32_t num_lanes_inuse;
 510 } MB_MGR_HMAC_MD5_OOO;
 511
 512
 513 /* GCM data structures */
 514 #define GCM_BLOCK_LEN   16
 515
 516 /**
 517  * @brief holds GCM operation context
 518  */
 519 struct gcm_context_data {
 520         /* init, update and finalize context data */
 521         uint8_t  aad_hash[GCM_BLOCK_LEN];
 522         uint64_t aad_length;
 523         uint64_t in_length;
 524         uint8_t  partial_block_enc_key[GCM_BLOCK_LEN];
 525         uint8_t  orig_IV[GCM_BLOCK_LEN];
 526         uint8_t  current_counter[GCM_BLOCK_LEN];
 527         uint64_t partial_block_length;
 528 };
 529
 530 /**
 531  * @brief GCM argument data per lane
 532  */
 533 struct GCM_ARGS {
 534         struct gcm_context_data *ctx[4];
 535         const void *keys[4];
 536         uint8_t *out[4];
 537         const uint8_t *in[4];
 538         void *tag[4];
 539         uint64_t tag_len[4];
 540 };
 541
 542 /**
 543  * @brief GCM multi-buffer manager structure
 544  */
 545 typedef struct {
 546         struct GCM_ARGS args;
 547         struct gcm_context_data ctxs[4];
 548         uint64_t lens[4];
 549         JOB_AES_HMAC *job_in_lane[4];
 550         uint64_t unused_lanes;
 551 } MB_MGR_GCM_OOO;
 552
 553 /* Authenticated Tag Length in bytes.
 554  * Valid values are 16 (most likely), 12 or 8. */
 555 #define MAX_TAG_LEN (16)
 556
 557 /*
 558  * IV data is limited to 16 bytes as follows:
 559  * 12 bytes is provided by an application -
 560  *    pre-counter block j0: 4 byte salt (from Security Association)
 561  *    concatenated with 8 byte Initialization Vector (from IPSec ESP
 562  *    Payload).
 563  * 4 byte value 0x00000001 is padded automatically by the library -
 564  *    there is no need to add these 4 bytes on application side anymore.
 565  */
 566 #define GCM_IV_DATA_LEN (12)
 567
 568 #define LONGEST_TESTED_AAD_LENGTH (2 * 1024)
 569
 570 /* Key lengths of 128 and 256 supported */
 571 #define GCM_128_KEY_LEN (16)
 572 #define GCM_192_KEY_LEN (24)
 573 #define GCM_256_KEY_LEN (32)
 574
 575 /* #define GCM_BLOCK_LEN   16 */
 576 #define GCM_ENC_KEY_LEN 16
 577 #define GCM_KEY_SETS    (15) /*exp key + 14 exp round keys*/
 578
 579 /**
 580  * @brief holds intermediate key data needed to improve performance
 581  *
 582  * gcm_key_data hold internal key information used by gcm128, gcm192 and gcm256.
 583  */
 584 #ifdef __WIN32
 585 __declspec(align(64))
 586 #endif /* WIN32 */
 587 struct gcm_key_data {
 588         uint8_t expanded_keys[GCM_ENC_KEY_LEN * GCM_KEY_SETS];
 589         uint8_t padding[GCM_ENC_KEY_LEN];        /* To align HashKey to 64 */
 590         /* storage for HashKey mod poly */
 591         uint8_t shifted_hkey_8[GCM_ENC_KEY_LEN]; /* HashKey^8<<1 mod poly */
 592         uint8_t shifted_hkey_7[GCM_ENC_KEY_LEN]; /* HashKey^7<<1 mod poly */
 593         uint8_t shifted_hkey_6[GCM_ENC_KEY_LEN]; /* HashKey^6<<1 mod poly */
 594         uint8_t shifted_hkey_5[GCM_ENC_KEY_LEN]; /* HashKey^5<<1 mod poly */
 595         uint8_t shifted_hkey_4[GCM_ENC_KEY_LEN]; /* HashKey^4<<1 mod poly */
 596         uint8_t shifted_hkey_3[GCM_ENC_KEY_LEN]; /* HashKey^3<<1 mod poly */
 597         uint8_t shifted_hkey_2[GCM_ENC_KEY_LEN]; /* HashKey^2<<1 mod poly */
 598         uint8_t shifted_hkey_1[GCM_ENC_KEY_LEN]; /* HashKey<<1 mod poly */
 599         /*
 600          * Storage for XOR of High 64 bits and low 64 bits of HashKey mod poly.
 601          * This is needed for Karatsuba purposes.
 602          */
 603         uint8_t shifted_hkey_1_k[GCM_ENC_KEY_LEN]; /* HashKey<<1 mod poly */
 604         uint8_t shifted_hkey_2_k[GCM_ENC_KEY_LEN]; /* HashKey^2<<1 mod poly */
 605         uint8_t shifted_hkey_3_k[GCM_ENC_KEY_LEN]; /* HashKey^3<<1 mod poly */
 606         uint8_t shifted_hkey_4_k[GCM_ENC_KEY_LEN]; /* HashKey^4<<1 mod poly */
 607         uint8_t shifted_hkey_5_k[GCM_ENC_KEY_LEN]; /* HashKey^5<<1 mod poly */
 608         uint8_t shifted_hkey_6_k[GCM_ENC_KEY_LEN]; /* HashKey^6<<1 mod poly */
 609         uint8_t shifted_hkey_7_k[GCM_ENC_KEY_LEN]; /* HashKey^7<<1 mod poly */
 610         uint8_t shifted_hkey_8_k[GCM_ENC_KEY_LEN]; /* HashKey^8<<1 mod poly */
 611 }
 612 #ifdef LINUX
 613 __attribute__((aligned(64)));
 614 #else
 615 ;
 616 #endif
 617
 618 /* ========================================================================== */
 619 /* API data type definitions */
 620 struct MB_MGR;
 621
 622 typedef void (*init_mb_mgr_t)(struct MB_MGR *);
 623 typedef JOB_AES_HMAC *(*get_next_job_t)(struct MB_MGR *);
 624 typedef JOB_AES_HMAC *(*submit_job_t)(struct MB_MGR *);
 625 typedef JOB_AES_HMAC *(*get_completed_job_t)(struct MB_MGR *);
 626 typedef JOB_AES_HMAC *(*flush_job_t)(struct MB_MGR *);
 627 typedef uint32_t (*queue_size_t)(struct MB_MGR *);
 628 typedef void (*keyexp_t)(const void *, void *, void *);
 629 typedef void (*cmac_subkey_gen_t)(const void *, void *, void *);
 630 typedef void (*hash_one_block_t)(const void *, void *);
 631 typedef void (*hash_fn_t)(const void *, const uint64_t, void *);
 632 typedef void (*xcbc_keyexp_t)(const void *, void *, void *, void *);
 633 typedef int (*des_keysched_t)(uint64_t *, const void *);
 634 typedef void (*aes128_cfb_t)(void *, const void *, const void *, const void *,
 635                              uint64_t);
 636 typedef void (*aes_gcm_enc_dec_t)(const struct gcm_key_data *,
 637                                   struct gcm_context_data *,
 638                                   uint8_t *, uint8_t const *, uint64_t,
 639                                   const uint8_t *, uint8_t const *, uint64_t,
 640                                   uint8_t *, uint64_t);
 641 typedef void (*aes_gcm_init_t)(const struct gcm_key_data *,
 642                                struct gcm_context_data *,
 643                                const uint8_t *, uint8_t const *, uint64_t);
 644 typedef void (*aes_gcm_enc_dec_update_t)(const struct gcm_key_data *,
 645                                          struct gcm_context_data *,
 646                                          uint8_t *, const uint8_t *, uint64_t);
 647 typedef void (*aes_gcm_enc_dec_finalize_t)(const struct gcm_key_data *,
 648                                            struct gcm_context_data *,
 649                                            uint8_t *, uint64_t);
 650 typedef void (*aes_gcm_precomp_t)(struct gcm_key_data *);
 651 typedef void (*aes_gcm_pre_t)(const void *, struct gcm_key_data *);
 652
 653 /* ========================================================================== */
 654 /* Multi-buffer manager flags passed to alloc_mb_mgr() */
 655
 656 #define IMB_FLAG_SHANI_OFF (1ULL << 0) /* disable use of SHANI extension */
 657 #define IMB_FLAG_AESNI_OFF (1ULL << 1) /* disable use of AESNI extension */
 658
 659 /* ========================================================================== */
 660 /* Multi-buffer manager detected features
 661  * - if bit is set then hardware supports given extension
 662  * - valid after call to init_mb_mgr() or alloc_mb_mgr()
 663  * - some HW supported features can be disabled via IMB_FLAG_xxx (see above)
 664  */
 665
 666 #define IMB_FEATURE_SHANI      (1ULL << 0)
 667 #define IMB_FEATURE_AESNI      (1ULL << 1)
 668 #define IMB_FEATURE_PCLMULQDQ  (1ULL << 2)
 669 #define IMB_FEATURE_CMOV       (1ULL << 3)
 670 #define IMB_FEATURE_SSE4_2     (1ULL << 4)
 671 #define IMB_FEATURE_AVX        (1ULL << 5)
 672 #define IMB_FEATURE_AVX2       (1ULL << 6)
 673 #define IMB_FEATURE_AVX512F    (1ULL << 7)
 674 #define IMB_FEATURE_AVX512DQ   (1ULL << 8)
 675 #define IMB_FEATURE_AVX512CD   (1ULL << 9)
 676 #define IMB_FEATURE_AVX512BW   (1ULL << 10)
 677 #define IMB_FEATURE_AVX512VL   (1ULL << 11)
 678 #define IMB_FEATURE_AVX512_SKX (IMB_FEATURE_AVX512F | IMB_FEATURE_AVX512DQ | \
 679                                 IMB_FEATURE_AVX512CD | IMB_FEATURE_AVX512BW | \
 680                                 IMB_FEATURE_AVX512VL)
 681 #define IMB_FEATURE_VAES       (1ULL << 12)
 682 #define IMB_FEATURE_VPCLMULQDQ (1ULL << 13)
 683
 684 /* ========================================================================== */
 685 /* TOP LEVEL (MB_MGR) Data structure fields */
 686
 687 #define MAX_JOBS 128
 688
 689 typedef struct MB_MGR {
 690         /*
 691          * flags - passed to alloc_mb_mgr()
 692          * features - reflects features of multi-buffer instance
 693          */
 694         uint64_t flags;
 695         uint64_t features;
 696
 697         /*
 698          * Reserved for the future
 699          */
 700         uint64_t reserved[6];
 701
 702         /*
 703          * ARCH handlers / API
 704          * Careful as changes here can break ABI compatibility
 705          */
 706         get_next_job_t          get_next_job;
 707         submit_job_t            submit_job;
 708         submit_job_t            submit_job_nocheck;
 709         get_completed_job_t     get_completed_job;
 710         flush_job_t             flush_job;
 711         queue_size_t            queue_size;
 712         keyexp_t                keyexp_128;
 713         keyexp_t                keyexp_192;
 714         keyexp_t                keyexp_256;
 715         cmac_subkey_gen_t       cmac_subkey_gen_128;
 716         xcbc_keyexp_t           xcbc_keyexp;
 717         des_keysched_t          des_key_sched;
 718         hash_one_block_t        sha1_one_block;
 719         hash_one_block_t        sha224_one_block;
 720         hash_one_block_t        sha256_one_block;
 721         hash_one_block_t        sha384_one_block;
 722         hash_one_block_t        sha512_one_block;
 723         hash_one_block_t        md5_one_block;
 724         hash_fn_t               sha1;
 725         hash_fn_t               sha224;
 726         hash_fn_t               sha256;
 727         hash_fn_t               sha384;
 728         hash_fn_t               sha512;
 729         aes128_cfb_t            aes128_cfb_one;
 730
 731         aes_gcm_enc_dec_t       gcm128_enc;
 732         aes_gcm_enc_dec_t       gcm192_enc;
 733         aes_gcm_enc_dec_t       gcm256_enc;
 734         aes_gcm_enc_dec_t       gcm128_dec;
 735         aes_gcm_enc_dec_t       gcm192_dec;
 736         aes_gcm_enc_dec_t       gcm256_dec;
 737         aes_gcm_init_t          gcm128_init;
 738         aes_gcm_init_t          gcm192_init;
 739         aes_gcm_init_t          gcm256_init;
 740         aes_gcm_enc_dec_update_t gcm128_enc_update;
 741         aes_gcm_enc_dec_update_t gcm192_enc_update;
 742         aes_gcm_enc_dec_update_t gcm256_enc_update;
 743         aes_gcm_enc_dec_update_t gcm128_dec_update;
 744         aes_gcm_enc_dec_update_t gcm192_dec_update;
 745         aes_gcm_enc_dec_update_t gcm256_dec_update;
 746         aes_gcm_enc_dec_finalize_t gcm128_enc_finalize;
 747         aes_gcm_enc_dec_finalize_t gcm192_enc_finalize;
 748         aes_gcm_enc_dec_finalize_t gcm256_enc_finalize;
 749         aes_gcm_enc_dec_finalize_t gcm128_dec_finalize;
 750         aes_gcm_enc_dec_finalize_t gcm192_dec_finalize;
 751         aes_gcm_enc_dec_finalize_t gcm256_dec_finalize;
 752         aes_gcm_precomp_t       gcm128_precomp;
 753         aes_gcm_precomp_t       gcm192_precomp;
 754         aes_gcm_precomp_t       gcm256_precomp;
 755         aes_gcm_pre_t           gcm128_pre;
 756         aes_gcm_pre_t           gcm192_pre;
 757         aes_gcm_pre_t           gcm256_pre;
 758
 759         /* in-order scheduler fields */
 760         int              earliest_job; /* byte offset, -1 if none */
 761         int              next_job;     /* byte offset */
 762         JOB_AES_HMAC     jobs[MAX_JOBS];
 763
 764         /* out of order managers */
 765         DECLARE_ALIGNED(MB_MGR_AES_OOO aes128_ooo, 64);
 766         DECLARE_ALIGNED(MB_MGR_AES_OOO aes192_ooo, 64);
 767         DECLARE_ALIGNED(MB_MGR_AES_OOO aes256_ooo, 64);
 768         DECLARE_ALIGNED(MB_MGR_AES_OOO docsis_sec_ooo, 64);
 769         DECLARE_ALIGNED(MB_MGR_DES_OOO des_enc_ooo, 64);
 770         DECLARE_ALIGNED(MB_MGR_DES_OOO des_dec_ooo, 64);
 771         DECLARE_ALIGNED(MB_MGR_DES_OOO des3_enc_ooo, 64);
 772         DECLARE_ALIGNED(MB_MGR_DES_OOO des3_dec_ooo, 64);
 773         DECLARE_ALIGNED(MB_MGR_DES_OOO docsis_des_enc_ooo, 64);
 774         DECLARE_ALIGNED(MB_MGR_DES_OOO docsis_des_dec_ooo, 64);
 775
 776         DECLARE_ALIGNED(MB_MGR_HMAC_SHA_1_OOO hmac_sha_1_ooo, 64);
 777         DECLARE_ALIGNED(MB_MGR_HMAC_SHA_256_OOO hmac_sha_224_ooo, 64);
 778         DECLARE_ALIGNED(MB_MGR_HMAC_SHA_256_OOO hmac_sha_256_ooo, 64);
 779         DECLARE_ALIGNED(MB_MGR_HMAC_SHA_512_OOO hmac_sha_384_ooo, 64);
 780         DECLARE_ALIGNED(MB_MGR_HMAC_SHA_512_OOO hmac_sha_512_ooo, 64);
 781         DECLARE_ALIGNED(MB_MGR_HMAC_MD5_OOO hmac_md5_ooo, 64);
 782         DECLARE_ALIGNED(MB_MGR_AES_XCBC_OOO aes_xcbc_ooo, 64);
 783         DECLARE_ALIGNED(MB_MGR_CCM_OOO aes_ccm_ooo, 64);
 784         DECLARE_ALIGNED(MB_MGR_CMAC_OOO aes_cmac_ooo, 64);
 785
 786         DECLARE_ALIGNED(MB_MGR_GCM_OOO gcm128_enc_ooo, 64);
 787         DECLARE_ALIGNED(MB_MGR_GCM_OOO gcm192_enc_ooo, 64);
 788         DECLARE_ALIGNED(MB_MGR_GCM_OOO gcm256_enc_ooo, 64);
 789         DECLARE_ALIGNED(MB_MGR_GCM_OOO gcm128_dec_ooo, 64);
 790         DECLARE_ALIGNED(MB_MGR_GCM_OOO gcm192_dec_ooo, 64);
 791         DECLARE_ALIGNED(MB_MGR_GCM_OOO gcm256_dec_ooo, 64);
 792 } MB_MGR;
 793
 794 /* ========================================================================== */
 795 /* API definitions */
 796
 797 /**
 798  * @brief Get library version in string format
 799  *
 800  * @return library version string
 801  */
 802 IMB_DLL_EXPORT const char *imb_get_version_str(void);
 803
 804 /**
 805  * @brief Get library version in numerical format
 806  *
 807  * Use IMB_VERSION() macro to compare this
 808  * numerical version against known library version.
 809  *
 810  * @return library version number
 811  */
 812 IMB_DLL_EXPORT unsigned imb_get_version(void);
 813
 814 /*
 815  * get_next_job returns a job object. This must be filled in and returned
 816  * via submit_job before get_next_job is called again.
 817  * After submit_job is called, one should call get_completed_job() at least
 818  * once (and preferably until it returns NULL).
 819  * get_completed_job and flush_job returns a job object. This job object ceases
 820  * to be usable at the next call to get_next_job
 821  */
 822 IMB_DLL_EXPORT MB_MGR *alloc_mb_mgr(uint64_t flags);
 823 IMB_DLL_EXPORT void free_mb_mgr(MB_MGR *state);
 824
 825 IMB_DLL_EXPORT void init_mb_mgr_avx(MB_MGR *state);
 826 IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_avx(MB_MGR *state);
 827 IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_nocheck_avx(MB_MGR *state);
 828 IMB_DLL_EXPORT JOB_AES_HMAC *flush_job_avx(MB_MGR *state);
 829 IMB_DLL_EXPORT uint32_t queue_size_avx(MB_MGR *state);
 830 IMB_DLL_EXPORT JOB_AES_HMAC *get_completed_job_avx(MB_MGR *state);
 831 IMB_DLL_EXPORT JOB_AES_HMAC *get_next_job_avx(MB_MGR *state);
 832
 833 IMB_DLL_EXPORT void init_mb_mgr_avx2(MB_MGR *state);
 834 IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_avx2(MB_MGR *state);
 835 IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_nocheck_avx2(MB_MGR *state);
 836 IMB_DLL_EXPORT JOB_AES_HMAC *flush_job_avx2(MB_MGR *state);
 837 IMB_DLL_EXPORT uint32_t queue_size_avx2(MB_MGR *state);
 838 IMB_DLL_EXPORT JOB_AES_HMAC *get_completed_job_avx2(MB_MGR *state);
 839 IMB_DLL_EXPORT JOB_AES_HMAC *get_next_job_avx2(MB_MGR *state);
 840
 841 IMB_DLL_EXPORT void init_mb_mgr_avx512(MB_MGR *state);
 842 IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_avx512(MB_MGR *state);
 843 IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_nocheck_avx512(MB_MGR *state);
 844 IMB_DLL_EXPORT JOB_AES_HMAC *flush_job_avx512(MB_MGR *state);
 845 IMB_DLL_EXPORT uint32_t queue_size_avx512(MB_MGR *state);
 846 IMB_DLL_EXPORT JOB_AES_HMAC *get_completed_job_avx512(MB_MGR *state);
 847 IMB_DLL_EXPORT JOB_AES_HMAC *get_next_job_avx512(MB_MGR *state);
 848
 849 IMB_DLL_EXPORT void init_mb_mgr_sse(MB_MGR *state);
 850 IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_sse(MB_MGR *state);
 851 IMB_DLL_EXPORT JOB_AES_HMAC *submit_job_nocheck_sse(MB_MGR *state);
 852 IMB_DLL_EXPORT JOB_AES_HMAC *flush_job_sse(MB_MGR *state);
 853 IMB_DLL_EXPORT uint32_t queue_size_sse(MB_MGR *state);
 854 IMB_DLL_EXPORT JOB_AES_HMAC *get_completed_job_sse(MB_MGR *state);
 855 IMB_DLL_EXPORT JOB_AES_HMAC *get_next_job_sse(MB_MGR *state);
 856
 857 /*
 858  * Wrapper macros to call arch API's set up
 859  * at init phase of multi-buffer manager.
 860  *
 861  * For example, after calling init_mb_mgr_sse(&mgr)
 862  * The 'mgr' structure be set up so that:
 863  *   mgr.get_next_job will point to get_next_job_sse(),
 864  *   mgr.submit_job will point to submit_job_sse(),
 865  *   mgr.submit_job_nocheck will point to submit_job_nocheck_sse(),
 866  *   mgr.get_completed_job will point to get_completed_job_sse(),
 867  *   mgr.flush_job will point to flush_job_sse(),
 868  *   mgr.queue_size will point to queue_size_sse()
 869  *   mgr.keyexp_128 will point to aes_keyexp_128_sse()
 870  *   mgr.keyexp_192 will point to aes_keyexp_192_sse()
 871  *   mgr.keyexp_256 will point to aes_keyexp_256_sse()
 872  *   etc.
 873  *
 874  * Direct use of arch API's may result in better performance.
 875  * Using below indirect interface may produce slightly worse performance but
 876  * it can simplify application implementation.
 877  * LibTestApp provides example of using the indirect interface.
 878  */
 879 #define IMB_GET_NEXT_JOB(_mgr)       ((_mgr)->get_next_job((_mgr)))
 880 #define IMB_SUBMIT_JOB(_mgr)         ((_mgr)->submit_job((_mgr)))
 881 #define IMB_SUBMIT_JOB_NOCHECK(_mgr) ((_mgr)->submit_job_nocheck((_mgr)))
 882 #define IMB_GET_COMPLETED_JOB(_mgr)  ((_mgr)->get_completed_job((_mgr)))
 883 #define IMB_FLUSH_JOB(_mgr)          ((_mgr)->flush_job((_mgr)))
 884 #define IMB_QUEUE_SIZE(_mgr)         ((_mgr)->queue_size((_mgr)))
 885
 886 /* Key expansion and generation API's */
 887 #define IMB_AES_KEYEXP_128(_mgr, _raw, _enc, _dec)      \
 888         ((_mgr)->keyexp_128((_raw), (_enc), (_dec)))
 889 #define IMB_AES_KEYEXP_192(_mgr, _raw, _enc, _dec)      \
 890         ((_mgr)->keyexp_192((_raw), (_enc), (_dec)))
 891 #define IMB_AES_KEYEXP_256(_mgr, _raw, _enc, _dec)      \
 892         ((_mgr)->keyexp_256((_raw), (_enc), (_dec)))
 893
 894 #define IMB_AES_CMAC_SUBKEY_GEN_128(_mgr, _key_exp, _k1, _k2)   \
 895         ((_mgr)->cmac_subkey_gen_128((_key_exp), (_k1), (_k2)))
 896
 897 #define IMB_AES_XCBC_KEYEXP(_mgr, _key, _k1_exp, _k2, _k3)      \
 898         ((_mgr)->xcbc_keyexp((_key), (_k1_exp), (_k2), (_k3)))
 899
 900 #define IMB_DES_KEYSCHED(_mgr, _ks, _key)       \
 901         ((_mgr)->des_key_sched((_ks), (_key)))
 902
 903 /* Hash API's */
 904 #define IMB_SHA1_ONE_BLOCK(_mgr, _data, _digest)        \
 905         ((_mgr)->sha1_one_block((_data), (_digest)))
 906 #define IMB_SHA1(_mgr, _data, _length, _digest)         \
 907         ((_mgr)->sha1((_data), (_length), (_digest)))
 908 #define IMB_SHA224_ONE_BLOCK(_mgr, _data, _digest)      \
 909         ((_mgr)->sha224_one_block((_data), (_digest)))
 910 #define IMB_SHA224(_mgr, _data, _length, _digest)       \
 911         ((_mgr)->sha224((_data), (_length), (_digest)))
 912 #define IMB_SHA256_ONE_BLOCK(_mgr, _data, _digest)      \
 913         ((_mgr)->sha256_one_block((_data), (_digest)))
 914 #define IMB_SHA256(_mgr, _data, _length, _digest)       \
 915         ((_mgr)->sha256((_data), (_length), (_digest)))
 916 #define IMB_SHA384_ONE_BLOCK(_mgr, _data, _digest)      \
 917         ((_mgr)->sha384_one_block((_data), (_digest)))
 918 #define IMB_SHA384(_mgr, _data, _length, _digest)       \
 919         ((_mgr)->sha384((_data), (_length), (_digest)))
 920 #define IMB_SHA512_ONE_BLOCK(_mgr, _data, _digest)      \
 921         ((_mgr)->sha512_one_block((_data), (_digest)))
 922 #define IMB_SHA512(_mgr, _data, _length, _digest)       \
 923         ((_mgr)->sha512((_data), (_length), (_digest)))
 924 #define IMB_MD5_ONE_BLOCK(_mgr, _data, _digest)         \
 925         ((_mgr)->md5_one_block((_data), (_digest)))
 926
 927 /* AES-CFB API */
 928 #define IMB_AES128_CFB_ONE(_mgr, _out, _in, _iv, _enc, _len)            \
 929         ((_mgr)->aes128_cfb_one((_out), (_in), (_iv), (_enc), (_len)))
 930
 931 /* AES-GCM API's */
 932 #define IMB_AES128_GCM_ENC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\
 933                            _tag, _tagl)                                 \
 934         ((_mgr)->gcm128_enc((_key), (_ctx), (_out), (_in), (_len), (_iv), \
 935                             (_aad), (_aadl), (_tag), (_tagl)))
 936 #define IMB_AES192_GCM_ENC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\
 937                            _tag, _tagl)                                 \
 938         ((_mgr)->gcm192_enc((_key), (_ctx), (_out), (_in), (_len), (_iv), \
 939                             (_aad), (_aadl), (_tag), (_tagl)))
 940 #define IMB_AES256_GCM_ENC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\
 941                            _tag, _tagl)                                 \
 942         ((_mgr)->gcm256_enc((_key), (_ctx), (_out), (_in), (_len), (_iv), \
 943                             (_aad), (_aadl), (_tag), (_tagl)))
 944
 945 #define IMB_AES128_GCM_DEC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\
 946                            _tag, _tagl)                                 \
 947         ((_mgr)->gcm128_dec((_key), (_ctx), (_out), (_in), (_len), (_iv), \
 948                             (_aad), (_aadl), (_tag), (_tagl)))
 949 #define IMB_AES192_GCM_DEC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\
 950                            _tag, _tagl)                                 \
 951         ((_mgr)->gcm192_dec((_key), (_ctx), (_out), (_in), (_len), (_iv), \
 952                             (_aad), (_aadl), (_tag), (_tagl)))
 953 #define IMB_AES256_GCM_DEC(_mgr, _key, _ctx, _out, _in, _len, _iv, _aad, _aadl,\
 954                            _tag, _tagl)                                 \
 955         ((_mgr)->gcm256_dec((_key), (_ctx), (_out), (_in), (_len), (_iv), \
 956                             (_aad), (_aadl), (_tag), (_tagl)))
 957
 958 #define IMB_AES128_GCM_INIT(_mgr, _key, _ctx, _iv, _aad, _aadl)        \
 959         ((_mgr)->gcm128_init((_key), (_ctx), (_iv), (_aad), (_aadl)))
 960 #define IMB_AES192_GCM_INIT(_mgr, _key, _ctx, _iv, _aad, _aadl)        \
 961         ((_mgr)->gcm192_init((_key), (_ctx), (_iv), (_aad), (_aadl)))
 962 #define IMB_AES256_GCM_INIT(_mgr, _key, _ctx, _iv, _aad, _aadl)        \
 963         ((_mgr)->gcm256_init((_key), (_ctx), (_iv), (_aad), (_aadl)))
 964
 965 #define IMB_AES128_GCM_ENC_UPDATE(_mgr, _key, _ctx, _out, _in, _len)    \
 966         ((_mgr)->gcm128_enc_update((_key), (_ctx), (_out), (_in), (_len)))
 967 #define IMB_AES192_GCM_ENC_UPDATE(_mgr, _key, _ctx, _out, _in, _len)    \
 968         ((_mgr)->gcm192_enc_update((_key), (_ctx), (_out), (_in), (_len)))
 969 #define IMB_AES256_GCM_ENC_UPDATE(_mgr, _key, _ctx, _out, _in, _len)    \
 970         ((_mgr)->gcm256_enc_update((_key), (_ctx), (_out), (_in), (_len)))
 971
 972 #define IMB_AES128_GCM_DEC_UPDATE(_mgr, _key, _ctx, _out, _in, _len)    \
 973         ((_mgr)->gcm128_dec_update((_key), (_ctx), (_out), (_in), (_len)))
 974 #define IMB_AES192_GCM_DEC_UPDATE(_mgr, _key, _ctx, _out, _in, _len)    \
 975         ((_mgr)->gcm192_dec_update((_key), (_ctx), (_out), (_in), (_len)))
 976 #define IMB_AES256_GCM_DEC_UPDATE(_mgr, _key, _ctx, _out, _in, _len)    \
 977         ((_mgr)->gcm256_dec_update((_key), (_ctx), (_out), (_in), (_len)))
 978
 979 #define IMB_AES128_GCM_ENC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl)      \
 980         ((_mgr)->gcm128_enc_finalize((_key), (_ctx), (_tag), (_tagl)))
 981 #define IMB_AES192_GCM_ENC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl)      \
 982         ((_mgr)->gcm192_enc_finalize((_key), (_ctx), (_tag), (_tagl)))
 983 #define IMB_AES256_GCM_ENC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl)      \
 984         ((_mgr)->gcm256_enc_finalize((_key), (_ctx), (_tag), (_tagl)))
 985
 986 #define IMB_AES128_GCM_DEC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl)      \
 987         ((_mgr)->gcm128_dec_finalize((_key), (_ctx), (_tag), (_tagl)))
 988 #define IMB_AES192_GCM_DEC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl)      \
 989         ((_mgr)->gcm192_dec_finalize((_key), (_ctx), (_tag), (_tagl)))
 990 #define IMB_AES256_GCM_DEC_FINALIZE(_mgr, _key, _ctx, _tag, _tagl)      \
 991         ((_mgr)->gcm256_dec_finalize((_key), (_ctx), (_tag), (_tagl)))
 992
 993 #define IMB_AES128_GCM_PRECOMP(_mgr, _key) \
 994         ((_mgr)->gcm128_precomp((_key)))
 995 #define IMB_AES192_GCM_PRECOMP(_mgr, _key) \
 996         ((_mgr)->gcm192_precomp((_key)))
 997 #define IMB_AES256_GCM_PRECOMP(_mgr, _key) \
 998         ((_mgr)->gcm256_precomp((_key)))
 999
1000 #define IMB_AES128_GCM_PRE(_mgr, _key_in, _key_exp)     \
1001         ((_mgr)->gcm128_pre((_key_in), (_key_exp)))
1002 #define IMB_AES192_GCM_PRE(_mgr, _key_in, _key_exp)     \
1003         ((_mgr)->gcm192_pre((_key_in), (_key_exp)))
1004 #define IMB_AES256_GCM_PRE(_mgr, _key_in, _key_exp)     \
1005         ((_mgr)->gcm256_pre((_key_in), (_key_exp)))
1006
1007 /* Auxiliary functions */
1008
1009 /**
1010  * @brief DES key schedule set up
1011  *
1012  * \a ks buffer needs to accomodate \a DES_KEY_SCHED_SIZE (128) bytes of data.
1013  *
1014  * @param ks destination buffer to accomodate DES key schedule
1015  * @param key a pointer to an 8 byte DES key
1016  *
1017  * @return Operation status
1018  * @retval 0 success
1019  * @retval !0 error
1020  */
1021 IMB_DLL_EXPORT int
1022 des_key_schedule(uint64_t *ks, const void *key);
1023
1024 /* SSE */
1025 IMB_DLL_EXPORT void sha1_sse(const void *data, const uint64_t length,
1026                              void *digest);
1027 IMB_DLL_EXPORT void sha1_one_block_sse(const void *data, void *digest);
1028 IMB_DLL_EXPORT void sha224_sse(const void *data, const uint64_t length,
1029                                void *digest);
1030 IMB_DLL_EXPORT void sha224_one_block_sse(const void *data, void *digest);
1031 IMB_DLL_EXPORT void sha256_sse(const void *data, const uint64_t length,
1032                                void *digest);
1033 IMB_DLL_EXPORT void sha256_one_block_sse(const void *data, void *digest);
1034 IMB_DLL_EXPORT void sha384_sse(const void *data, const uint64_t length,
1035                                void *digest);
1036 IMB_DLL_EXPORT void sha384_one_block_sse(const void *data, void *digest);
1037 IMB_DLL_EXPORT void sha512_sse(const void *data, const uint64_t length,
1038                                void *digest);
1039 IMB_DLL_EXPORT void sha512_one_block_sse(const void *data, void *digest);
1040 IMB_DLL_EXPORT void md5_one_block_sse(const void *data, void *digest);
1041 IMB_DLL_EXPORT void aes_keyexp_128_sse(const void *key, void *enc_exp_keys,
1042                                        void *dec_exp_keys);
1043 IMB_DLL_EXPORT void aes_keyexp_192_sse(const void *key, void *enc_exp_keys,
1044                                        void *dec_exp_keys);
1045 IMB_DLL_EXPORT void aes_keyexp_256_sse(const void *key, void *enc_exp_keys,
1046                                        void *dec_exp_keys);
1047 IMB_DLL_EXPORT void aes_xcbc_expand_key_sse(const void *key, void *k1_exp,
1048                                             void *k2, void *k3);
1049 IMB_DLL_EXPORT void aes_keyexp_128_enc_sse(const void *key,
1050                                            void *enc_exp_keys);
1051 IMB_DLL_EXPORT void aes_keyexp_192_enc_sse(const void *key,
1052                                            void *enc_exp_keys);
1053 IMB_DLL_EXPORT void aes_keyexp_256_enc_sse(const void *key,
1054                                            void *enc_exp_keys);
1055 IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse(const void *key_exp, void *key1,
1056                                             void *key2);
1057 IMB_DLL_EXPORT void aes_cfb_128_one_sse(void *out, const void *in,
1058                                         const void *iv, const void *keys,
1059                                         uint64_t len);
1060
1061 /* AVX */
1062 IMB_DLL_EXPORT void sha1_avx(const void *data, const uint64_t length,
1063                              void *digest);
1064 IMB_DLL_EXPORT void sha1_one_block_avx(const void *data, void *digest);
1065 IMB_DLL_EXPORT void sha224_avx(const void *data, const uint64_t length,
1066                                void *digest);
1067 IMB_DLL_EXPORT void sha224_one_block_avx(const void *data, void *digest);
1068 IMB_DLL_EXPORT void sha256_avx(const void *data, const uint64_t length,
1069                                void *digest);
1070 IMB_DLL_EXPORT void sha256_one_block_avx(const void *data, void *digest);
1071 IMB_DLL_EXPORT void sha384_avx(const void *data, const uint64_t length,
1072                                void *digest);
1073 IMB_DLL_EXPORT void sha384_one_block_avx(const void *data, void *digest);
1074 IMB_DLL_EXPORT void sha512_avx(const void *data, const uint64_t length,
1075                                void *digest);
1076 IMB_DLL_EXPORT void sha512_one_block_avx(const void *data, void *digest);
1077 IMB_DLL_EXPORT void md5_one_block_avx(const void *data, void *digest);
1078 IMB_DLL_EXPORT void aes_keyexp_128_avx(const void *key, void *enc_exp_keys,
1079                                        void *dec_exp_keys);
1080 IMB_DLL_EXPORT void aes_keyexp_192_avx(const void *key, void *enc_exp_keys,
1081                                        void *dec_exp_keys);
1082 IMB_DLL_EXPORT void aes_keyexp_256_avx(const void *key, void *enc_exp_keys,
1083                                        void *dec_exp_keys);
1084 IMB_DLL_EXPORT void aes_xcbc_expand_key_avx(const void *key, void *k1_exp,
1085                                             void *k2, void *k3);
1086 IMB_DLL_EXPORT void aes_keyexp_128_enc_avx(const void *key,
1087                                            void *enc_exp_keys);
1088 IMB_DLL_EXPORT void aes_keyexp_192_enc_avx(const void *key,
1089                                            void *enc_exp_keys);
1090 IMB_DLL_EXPORT void aes_keyexp_256_enc_avx(const void *key,
1091                                            void *enc_exp_keys);
1092 IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx(const void *key_exp, void *key1,
1093                                             void *key2);
1094 IMB_DLL_EXPORT void aes_cfb_128_one_avx(void *out, const void *in,
1095                                         const void *iv, const void *keys,
1096                                         uint64_t len);
1097
1098 /* AVX2 */
1099 IMB_DLL_EXPORT void sha1_avx2(const void *data, const uint64_t length,
1100                               void *digest);
1101 IMB_DLL_EXPORT void sha1_one_block_avx2(const void *data, void *digest);
1102 IMB_DLL_EXPORT void sha224_avx2(const void *data, const uint64_t length,
1103                                 void *digest);
1104 IMB_DLL_EXPORT void sha224_one_block_avx2(const void *data, void *digest);
1105 IMB_DLL_EXPORT void sha256_avx2(const void *data, const uint64_t length,
1106                                 void *digest);
1107 IMB_DLL_EXPORT void sha256_one_block_avx2(const void *data, void *digest);
1108 IMB_DLL_EXPORT void sha384_avx2(const void *data, const uint64_t length,
1109                                 void *digest);
1110 IMB_DLL_EXPORT void sha384_one_block_avx2(const void *data, void *digest);
1111 IMB_DLL_EXPORT void sha512_avx2(const void *data, const uint64_t length,
1112                                 void *digest);
1113 IMB_DLL_EXPORT void sha512_one_block_avx2(const void *data, void *digest);
1114 IMB_DLL_EXPORT void md5_one_block_avx2(const void *data, void *digest);
1115 IMB_DLL_EXPORT void aes_keyexp_128_avx2(const void *key, void *enc_exp_keys,
1116                                         void *dec_exp_keys);
1117 IMB_DLL_EXPORT void aes_keyexp_192_avx2(const void *key, void *enc_exp_keys,
1118                                         void *dec_exp_keys);
1119 IMB_DLL_EXPORT void aes_keyexp_256_avx2(const void *key, void *enc_exp_keys,
1120                                         void *dec_exp_keys);
1121 IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2(const void *key, void *k1_exp,
1122                                              void *k2, void *k3);
1123 IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2(const void *key,
1124                                             void *enc_exp_keys);
1125 IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2(const void *key,
1126                                             void *enc_exp_keys);
1127 IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2(const void *key,
1128                                             void *enc_exp_keys);
1129 IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2(const void *key_exp, void *key1,
1130                                              void *key2);
1131 IMB_DLL_EXPORT void aes_cfb_128_one_avx2(void *out, const void *in,
1132                                          const void *iv, const void *keys,
1133                                          uint64_t len);
1134
1135 /* AVX512 */
1136 IMB_DLL_EXPORT void sha1_avx512(const void *data, const uint64_t length,
1137                                  void *digest);
1138 IMB_DLL_EXPORT void sha1_one_block_avx512(const void *data, void *digest);
1139 IMB_DLL_EXPORT void sha224_avx512(const void *data, const uint64_t length,
1140                                   void *digest);
1141 IMB_DLL_EXPORT void sha224_one_block_avx512(const void *data, void *digest);
1142 IMB_DLL_EXPORT void sha256_avx512(const void *data, const uint64_t length,
1143                                   void *digest);
1144 IMB_DLL_EXPORT void sha256_one_block_avx512(const void *data, void *digest);
1145 IMB_DLL_EXPORT void sha384_avx512(const void *data, const uint64_t length,
1146                                   void *digest);
1147 IMB_DLL_EXPORT void sha384_one_block_avx512(const void *data, void *digest);
1148 IMB_DLL_EXPORT void sha512_avx512(const void *data, const uint64_t length,
1149                                   void *digest);
1150 IMB_DLL_EXPORT void sha512_one_block_avx512(const void *data, void *digest);
1151 IMB_DLL_EXPORT void md5_one_block_avx512(const void *data, void *digest);
1152 IMB_DLL_EXPORT void aes_keyexp_128_avx512(const void *key, void *enc_exp_keys,
1153                                           void *dec_exp_keys);
1154 IMB_DLL_EXPORT void aes_keyexp_192_avx512(const void *key, void *enc_exp_keys,
1155                                           void *dec_exp_keys);
1156 IMB_DLL_EXPORT void aes_keyexp_256_avx512(const void *key, void *enc_exp_keys,
1157                                           void *dec_exp_keys);
1158 IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512(const void *key, void *k1_exp,
1159                                                void *k2, void *k3);
1160 IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512(const void *key,
1161                                               void *enc_exp_keys);
1162 IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512(const void *key,
1163                                               void *enc_exp_keys);
1164 IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512(const void *key,
1165                                               void *enc_exp_keys);
1166 IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512(const void *key_exp, void *key1,
1167                                                void *key2);
1168 IMB_DLL_EXPORT void aes_cfb_128_one_avx512(void *out, const void *in,
1169                                            const void *iv, const void *keys,
1170                                            uint64_t len);
1171
1172 /*
1173  * Direct GCM API.
1174  * Note that GCM is also availabe through job API.
1175  */
1176 #ifndef NO_GCM
1177 /**
1178  * @brief GCM-AES Encryption
1179  *
1180  * @param key_data GCM expanded key data
1181  * @param context_data GCM operation context data
1182  * @param out Ciphertext output. Encrypt in-place is allowed.
1183  * @param in Plaintext input.
1184  * @param len Length of data in Bytes for encryption.
1185  * @param iv pointer to 12 byte IV structure. Internally, library
1186  *        concates 0x00000001 value to it.
1187  * @param aad Additional Authentication Data (AAD).
1188  * @param aad_len Length of AAD.
1189  * @param auth_tag Authenticated Tag output.
1190  * @param auth_tag_len Authenticated Tag Length in bytes (must be
1191  *                     a multiple of 4 bytes). Valid values are
1192  *                     16 (most likely), 12 or 8.
1193  */
1194 IMB_DLL_EXPORT void
1195 aes_gcm_enc_128_sse(const struct gcm_key_data *key_data,
1196                     struct gcm_context_data *context_data,
1197                     uint8_t *out, uint8_t const *in, uint64_t len,
1198                     const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
1199                     uint8_t *auth_tag, uint64_t auth_tag_len);
1200 IMB_DLL_EXPORT void
1201 aes_gcm_enc_128_avx_gen2(const struct gcm_key_data *key_data,
1202                          struct gcm_context_data *context_data,
1203                          uint8_t *out, uint8_t const *in, uint64_t len,
1204                          const uint8_t *iv,
1205                          uint8_t const *aad, uint64_t aad_len,
1206                          uint8_t *auth_tag, uint64_t auth_tag_len);
1207 IMB_DLL_EXPORT void
1208 aes_gcm_enc_128_avx_gen4(const struct gcm_key_data *key_data,
1209                          struct gcm_context_data *context_data,
1210                          uint8_t *out, uint8_t const *in, uint64_t len,
1211                          const uint8_t *iv,
1212                          uint8_t const *aad, uint64_t aad_len,
1213                          uint8_t *auth_tag, uint64_t auth_tag_len);
1214
1215 IMB_DLL_EXPORT void
1216 aes_gcm_enc_192_sse(const struct gcm_key_data *key_data,
1217                     struct gcm_context_data *context_data,
1218                     uint8_t *out, uint8_t const *in, uint64_t len,
1219                     const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
1220                     uint8_t *auth_tag, uint64_t auth_tag_len);
1221 IMB_DLL_EXPORT void
1222 aes_gcm_enc_192_avx_gen2(const struct gcm_key_data *key_data,
1223                          struct gcm_context_data *context_data,
1224                          uint8_t *out, uint8_t const *in, uint64_t len,
1225                          const uint8_t *iv,
1226                          uint8_t const *aad, uint64_t aad_len,
1227                          uint8_t *auth_tag, uint64_t auth_tag_len);
1228 IMB_DLL_EXPORT void
1229 aes_gcm_enc_192_avx_gen4(const struct gcm_key_data *key_data,
1230                          struct gcm_context_data *context_data,
1231                          uint8_t *out, uint8_t const *in, uint64_t len,
1232                          const uint8_t *iv,
1233                          uint8_t const *aad, uint64_t aad_len,
1234                          uint8_t *auth_tag, uint64_t auth_tag_len);
1235
1236 IMB_DLL_EXPORT void
1237 aes_gcm_enc_256_sse(const struct gcm_key_data *key_data,
1238                     struct gcm_context_data *context_data,
1239                     uint8_t *out, uint8_t const *in, uint64_t len,
1240                     const uint8_t *iv,
1241                     uint8_t const *aad, uint64_t aad_len,
1242                     uint8_t *auth_tag, uint64_t auth_tag_len);
1243 IMB_DLL_EXPORT void
1244 aes_gcm_enc_256_avx_gen2(const struct gcm_key_data *key_data,
1245                          struct gcm_context_data *context_data,
1246                          uint8_t *out, uint8_t const *in, uint64_t len,
1247                          const uint8_t *iv,
1248                          uint8_t const *aad, uint64_t aad_len,
1249                          uint8_t *auth_tag, uint64_t auth_tag_len);
1250 IMB_DLL_EXPORT void
1251 aes_gcm_enc_256_avx_gen4(const struct gcm_key_data *key_data,
1252                          struct gcm_context_data *context_data,
1253                          uint8_t *out, uint8_t const *in, uint64_t len,
1254                          const uint8_t *iv,
1255                          uint8_t const *aad, uint64_t aad_len,
1256                          uint8_t *auth_tag, uint64_t auth_tag_len);
1257
1258 /**
1259  * @brief GCM-AES Decryption
1260  *
1261  * @param key_data GCM expanded keys data
1262  * @param context_data GCM operation context data
1263  * @param out Plaintext output. Decrypt in-place is allowed.
1264  * @param in Ciphertext input.
1265  * @param len Length of data in Bytes for decryption.
1266  * @param iv pointer to 12 byte IV structure. Internally, library
1267  *        concates 0x00000001 value to it.
1268  * @param aad Additional Authentication Data (AAD).
1269  * @param aad_len Length of AAD.
1270  * @param auth_tag Authenticated Tag output.
1271  * @param auth_tag_len Authenticated Tag Length in bytes (must be
1272  *                     a multiple of 4 bytes). Valid values are
1273  *                     16 (most likely), 12 or 8.
1274  */
1275 IMB_DLL_EXPORT void
1276 aes_gcm_dec_128_sse(const struct gcm_key_data *key_data,
1277                     struct gcm_context_data *context_data,
1278                     uint8_t *out, uint8_t const *in, uint64_t len,
1279                     const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
1280                     uint8_t *auth_tag, uint64_t auth_tag_len);
1281 IMB_DLL_EXPORT void
1282 aes_gcm_dec_128_avx_gen2(const struct gcm_key_data *key_data,
1283                          struct gcm_context_data *context_data,
1284                          uint8_t *out, uint8_t const *in, uint64_t len,
1285                          const uint8_t *iv,
1286                          uint8_t const *aad, uint64_t aad_len,
1287                          uint8_t *auth_tag, uint64_t auth_tag_len);
1288 IMB_DLL_EXPORT void
1289 aes_gcm_dec_128_avx_gen4(const struct gcm_key_data *key_data,
1290                          struct gcm_context_data *context_data,
1291                          uint8_t *out, uint8_t const *in, uint64_t len,
1292                          const uint8_t *iv,
1293                          uint8_t const *aad, uint64_t aad_len,
1294                          uint8_t *auth_tag, uint64_t auth_tag_len);
1295
1296 IMB_DLL_EXPORT void
1297 aes_gcm_dec_192_sse(const struct gcm_key_data *key_data,
1298                     struct gcm_context_data *context_data,
1299                     uint8_t *out, uint8_t const *in, uint64_t len,
1300                     const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
1301                     uint8_t *auth_tag, uint64_t auth_tag_len);
1302 IMB_DLL_EXPORT void
1303 aes_gcm_dec_192_avx_gen2(const struct gcm_key_data *key_data,
1304                          struct gcm_context_data *context_data,
1305                          uint8_t *out, uint8_t const *in, uint64_t len,
1306                          const uint8_t *iv,
1307                          uint8_t const *aad, uint64_t aad_len,
1308                          uint8_t *auth_tag, uint64_t auth_tag_len);
1309 IMB_DLL_EXPORT void
1310 aes_gcm_dec_192_avx_gen4(const struct gcm_key_data *key_data,
1311                          struct gcm_context_data *context_data,
1312                          uint8_t *out, uint8_t const *in, uint64_t len,
1313                          const uint8_t *iv,
1314                          uint8_t const *aad, uint64_t aad_len,
1315                          uint8_t *auth_tag, uint64_t auth_tag_len);
1316
1317 IMB_DLL_EXPORT void
1318 aes_gcm_dec_256_sse(const struct gcm_key_data *key_data,
1319                     struct gcm_context_data *context_data,
1320                     uint8_t *out, uint8_t const *in, uint64_t len,
1321                     const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
1322                     uint8_t *auth_tag, uint64_t auth_tag_len);
1323 IMB_DLL_EXPORT void
1324 aes_gcm_dec_256_avx_gen2(const struct gcm_key_data *key_data,
1325                          struct gcm_context_data *context_data,
1326                          uint8_t *out, uint8_t const *in, uint64_t len,
1327                          const uint8_t *iv,
1328                          uint8_t const *aad, uint64_t aad_len,
1329                          uint8_t *auth_tag, uint64_t auth_tag_len);
1330 IMB_DLL_EXPORT void
1331 aes_gcm_dec_256_avx_gen4(const struct gcm_key_data *key_data,
1332                          struct gcm_context_data *context_data,
1333                          uint8_t *out, uint8_t const *in, uint64_t len,
1334                          const uint8_t *iv,
1335                          uint8_t const *aad, uint64_t aad_len,
1336                          uint8_t *auth_tag, uint64_t auth_tag_len);
1337
1338 /**
1339  * @brief Start a AES-GCM Encryption message
1340  *
1341  * @param key_data GCM expanded key data
1342  * @param context_data GCM operation context data
1343  * @param iv pointer to 12 byte IV structure. Internally, library
1344  *        concates 0x00000001 value to it.
1345  * @param aad Additional Authentication Data (AAD).
1346  * @param aad_len Length of AAD.
1347  *
1348  */
1349 IMB_DLL_EXPORT void
1350 aes_gcm_init_128_sse(const struct gcm_key_data *key_data,
1351                      struct gcm_context_data *context_data,
1352                      const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
1353 IMB_DLL_EXPORT void
1354 aes_gcm_init_128_avx_gen2(const struct gcm_key_data *key_data,
1355                           struct gcm_context_data *context_data,
1356                           const uint8_t *iv,
1357                           uint8_t const *aad, uint64_t aad_len);
1358 IMB_DLL_EXPORT void
1359 aes_gcm_init_128_avx_gen4(const struct gcm_key_data *key_data,
1360                           struct gcm_context_data *context_data,
1361                           const uint8_t *iv,
1362                           uint8_t const *aad, uint64_t aad_len);
1363
1364 IMB_DLL_EXPORT void
1365 aes_gcm_init_192_sse(const struct gcm_key_data *key_data,
1366                      struct gcm_context_data *context_data,
1367                      const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
1368 IMB_DLL_EXPORT void
1369 aes_gcm_init_192_avx_gen2(const struct gcm_key_data *key_data,
1370                           struct gcm_context_data *context_data,
1371                           const uint8_t *iv,
1372                           uint8_t const *aad, uint64_t aad_len);
1373 IMB_DLL_EXPORT void
1374 aes_gcm_init_192_avx_gen4(const struct gcm_key_data *key_data,
1375                           struct gcm_context_data *context_data,
1376                           const uint8_t *iv,
1377                           uint8_t const *aad, uint64_t aad_len);
1378
1379 IMB_DLL_EXPORT void
1380 aes_gcm_init_256_sse(const struct gcm_key_data *key_data,
1381                      struct gcm_context_data *context_data,
1382                      const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
1383 IMB_DLL_EXPORT void
1384 aes_gcm_init_256_avx_gen2(const struct gcm_key_data *key_data,
1385                           struct gcm_context_data *context_data,
1386                           const uint8_t *iv,
1387                           uint8_t const *aad, uint64_t aad_len);
1388 IMB_DLL_EXPORT void
1389 aes_gcm_init_256_avx_gen4(const struct gcm_key_data *key_data,
1390                           struct gcm_context_data *context_data,
1391                           const uint8_t *iv,
1392                           uint8_t const *aad, uint64_t aad_len);
1393
1394 /**
1395  * @brief encrypt a block of a AES-GCM Encryption message
1396  *
1397  * @param key_data GCM expanded key data
1398  * @param context_data GCM operation context data
1399  * @param out Ciphertext output. Encrypt in-place is allowed.
1400  * @param in Plaintext input.
1401  * @param len Length of data in Bytes for decryption.
1402  */
1403 IMB_DLL_EXPORT void
1404 aes_gcm_enc_128_update_sse(const struct gcm_key_data *key_data,
1405                            struct gcm_context_data *context_data,
1406                            uint8_t *out, const uint8_t *in, uint64_t len);
1407 IMB_DLL_EXPORT void
1408 aes_gcm_enc_128_update_avx_gen2(const struct gcm_key_data *key_data,
1409                                 struct gcm_context_data *context_data,
1410                                 uint8_t *out, const uint8_t *in, uint64_t len);
1411 IMB_DLL_EXPORT void
1412 aes_gcm_enc_128_update_avx_gen4(const struct gcm_key_data *key_data,
1413                                 struct gcm_context_data *context_data,
1414                                 uint8_t *out, const uint8_t *in, uint64_t len);
1415
1416 IMB_DLL_EXPORT void
1417 aes_gcm_enc_192_update_sse(const struct gcm_key_data *key_data,
1418                            struct gcm_context_data *context_data,
1419                            uint8_t *out, const uint8_t *in, uint64_t len);
1420 IMB_DLL_EXPORT void
1421 aes_gcm_enc_192_update_avx_gen2(const struct gcm_key_data *key_data,
1422                                 struct gcm_context_data *context_data,
1423                                 uint8_t *out, const uint8_t *in, uint64_t len);
1424 IMB_DLL_EXPORT void
1425 aes_gcm_enc_192_update_avx_gen4(const struct gcm_key_data *key_data,
1426                                 struct gcm_context_data *context_data,
1427                                 uint8_t *out, const uint8_t *in, uint64_t len);
1428
1429 IMB_DLL_EXPORT void
1430 aes_gcm_enc_256_update_sse(const struct gcm_key_data *key_data,
1431                            struct gcm_context_data *context_data,
1432                            uint8_t *out, const uint8_t *in, uint64_t len);
1433 IMB_DLL_EXPORT void
1434 aes_gcm_enc_256_update_avx_gen2(const struct gcm_key_data *key_data,
1435                                 struct gcm_context_data *context_data,
1436                                 uint8_t *out, const uint8_t *in, uint64_t len);
1437 IMB_DLL_EXPORT void
1438 aes_gcm_enc_256_update_avx_gen4(const struct gcm_key_data *key_data,
1439                                 struct gcm_context_data *context_data,
1440                                 uint8_t *out, const uint8_t *in, uint64_t len);
1441
1442 /**
1443  * @brief decrypt a block of a AES-GCM Encryption message
1444  *
1445  * @param key_data GCM expanded key data
1446  * @param context_data GCM operation context data
1447  * @param out Plaintext output. Decrypt in-place is allowed.
1448  * @param in Ciphertext input.
1449  * @param len Length of data in Bytes for decryption.
1450  */
1451 IMB_DLL_EXPORT void
1452 aes_gcm_dec_128_update_sse(const struct gcm_key_data *key_data,
1453                            struct gcm_context_data *context_data,
1454                            uint8_t *out, const uint8_t *in, uint64_t len);
1455 IMB_DLL_EXPORT void
1456 aes_gcm_dec_128_update_avx_gen2(const struct gcm_key_data *key_data,
1457                                 struct gcm_context_data *context_data,
1458                                 uint8_t *out, const uint8_t *in, uint64_t len);
1459 IMB_DLL_EXPORT void
1460 aes_gcm_dec_128_update_avx_gen4(const struct gcm_key_data *key_data,
1461                                 struct gcm_context_data *context_data,
1462                                 uint8_t *out, const uint8_t *in, uint64_t len);
1463
1464 IMB_DLL_EXPORT void
1465 aes_gcm_dec_192_update_sse(const struct gcm_key_data *key_data,
1466                            struct gcm_context_data *context_data,
1467                            uint8_t *out, const uint8_t *in, uint64_t len);
1468 IMB_DLL_EXPORT void
1469 aes_gcm_dec_192_update_avx_gen2(const struct gcm_key_data *key_data,
1470                                 struct gcm_context_data *context_data,
1471                                 uint8_t *out, const uint8_t *in, uint64_t len);
1472 IMB_DLL_EXPORT void
1473 aes_gcm_dec_192_update_avx_gen4(const struct gcm_key_data *key_data,
1474                                 struct gcm_context_data *context_data,
1475                                 uint8_t *out, const uint8_t *in, uint64_t len);
1476
1477 IMB_DLL_EXPORT void
1478 aes_gcm_dec_256_update_sse(const struct gcm_key_data *key_data,
1479                            struct gcm_context_data *context_data,
1480                            uint8_t *out, const uint8_t *in, uint64_t len);
1481 IMB_DLL_EXPORT void
1482 aes_gcm_dec_256_update_avx_gen2(const struct gcm_key_data *key_data,
1483                                 struct gcm_context_data *context_data,
1484                                 uint8_t *out, const uint8_t *in, uint64_t len);
1485 IMB_DLL_EXPORT void
1486 aes_gcm_dec_256_update_avx_gen4(const struct gcm_key_data *key_data,
1487                                 struct gcm_context_data *context_data,
1488                                 uint8_t *out, const uint8_t *in, uint64_t len);
1489
1490 /**
1491  * @brief End encryption of a AES-GCM Encryption message
1492  *
1493  * @param key_data GCM expanded key data
1494  * @param context_data GCM operation context data
1495  * @param auth_tag Authenticated Tag output.
1496  * @param auth_tag_len Authenticated Tag Length in bytes (must be
1497  *                     a multiple of 4 bytes). Valid values are
1498  *                     16 (most likely), 12 or 8.
1499  */
1500 IMB_DLL_EXPORT void
1501 aes_gcm_enc_128_finalize_sse(const struct gcm_key_data *key_data,
1502                              struct gcm_context_data *context_data,
1503                              uint8_t *auth_tag, uint64_t auth_tag_len);
1504 IMB_DLL_EXPORT void
1505 aes_gcm_enc_128_finalize_avx_gen2(const struct gcm_key_data *key_data,
1506                                   struct gcm_context_data *context_data,
1507                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1508 IMB_DLL_EXPORT void
1509 aes_gcm_enc_128_finalize_avx_gen4(const struct gcm_key_data *key_data,
1510                                   struct gcm_context_data *context_data,
1511                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1512
1513 IMB_DLL_EXPORT void
1514 aes_gcm_enc_192_finalize_sse(const struct gcm_key_data *key_data,
1515                              struct gcm_context_data *context_data,
1516                              uint8_t *auth_tag, uint64_t auth_tag_len);
1517 IMB_DLL_EXPORT void
1518 aes_gcm_enc_192_finalize_avx_gen2(const struct gcm_key_data *key_data,
1519                                   struct gcm_context_data *context_data,
1520                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1521 IMB_DLL_EXPORT void
1522 aes_gcm_enc_192_finalize_avx_gen4(const struct gcm_key_data *key_data,
1523                                   struct gcm_context_data *context_data,
1524                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1525
1526 IMB_DLL_EXPORT void
1527 aes_gcm_enc_256_finalize_sse(const struct gcm_key_data *key_data,
1528                              struct gcm_context_data *context_data,
1529                              uint8_t *auth_tag, uint64_t auth_tag_len);
1530 IMB_DLL_EXPORT void
1531 aes_gcm_enc_256_finalize_avx_gen2(const struct gcm_key_data *key_data,
1532                                   struct gcm_context_data *context_data,
1533                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1534 IMB_DLL_EXPORT void
1535 aes_gcm_enc_256_finalize_avx_gen4(const struct gcm_key_data *key_data,
1536                                   struct gcm_context_data *context_data,
1537                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1538
1539 /**
1540  * @brief End decryption of a AES-GCM Encryption message
1541  *
1542  * @param key_data GCM expanded key data
1543  * @param context_data GCM operation context data
1544  * @param auth_tag Authenticated Tag output.
1545  * @param auth_tag_len Authenticated Tag Length in bytes (must be
1546  *                     a multiple of 4 bytes). Valid values are
1547  *                     16 (most likely), 12 or 8.
1548  */
1549 IMB_DLL_EXPORT void
1550 aes_gcm_dec_128_finalize_sse(const struct gcm_key_data *key_data,
1551                              struct gcm_context_data *context_data,
1552                              uint8_t *auth_tag, uint64_t auth_tag_len);
1553 IMB_DLL_EXPORT void
1554 aes_gcm_dec_128_finalize_avx_gen2(const struct gcm_key_data *key_data,
1555                                   struct gcm_context_data *context_data,
1556                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1557 IMB_DLL_EXPORT void
1558 aes_gcm_dec_128_finalize_avx_gen4(const struct gcm_key_data *key_data,
1559                                   struct gcm_context_data *context_data,
1560                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1561
1562 IMB_DLL_EXPORT void
1563 aes_gcm_dec_192_finalize_sse(const struct gcm_key_data *key_data,
1564                              struct gcm_context_data *context_data,
1565                              uint8_t *auth_tag, uint64_t auth_tag_len);
1566 IMB_DLL_EXPORT void
1567 aes_gcm_dec_192_finalize_avx_gen2(const struct gcm_key_data *key_data,
1568                                   struct gcm_context_data *context_data,
1569                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1570 IMB_DLL_EXPORT void
1571 aes_gcm_dec_192_finalize_avx_gen4(const struct gcm_key_data *key_data,
1572                                   struct gcm_context_data *context_data,
1573                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1574
1575 IMB_DLL_EXPORT void
1576 aes_gcm_dec_256_finalize_sse(const struct gcm_key_data *key_data,
1577                              struct gcm_context_data *context_data,
1578                              uint8_t *auth_tag, uint64_t auth_tag_len);
1579 IMB_DLL_EXPORT void
1580 aes_gcm_dec_256_finalize_avx_gen2(const struct gcm_key_data *key_data,
1581                                   struct gcm_context_data *context_data,
1582                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1583 IMB_DLL_EXPORT void
1584 aes_gcm_dec_256_finalize_avx_gen4(const struct gcm_key_data *key_data,
1585                                   struct gcm_context_data *context_data,
1586                                   uint8_t *auth_tag, uint64_t auth_tag_len);
1587
1588 /**
1589  * @brief Precomputation of HashKey constants
1590  *
1591  * Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and
1592  * shifted_hkey_X_k).
1593  *
1594  * @param gdata GCM context data
1595  */
1596 IMB_DLL_EXPORT void aes_gcm_precomp_128_sse(struct gcm_key_data *key_data);
1597 IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2(struct gcm_key_data *key_data);
1598 IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4(struct gcm_key_data *key_data);
1599
1600 IMB_DLL_EXPORT void aes_gcm_precomp_192_sse(struct gcm_key_data *key_data);
1601 IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2(struct gcm_key_data *key_data);
1602 IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4(struct gcm_key_data *key_data);
1603
1604 IMB_DLL_EXPORT void aes_gcm_precomp_256_sse(struct gcm_key_data *key_data);
1605 IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2(struct gcm_key_data *key_data);
1606 IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4(struct gcm_key_data *key_data);
1607
1608 /**
1609  * @brief Pre-processes GCM key data
1610  *
1611  * Prefills the gcm key data with key values for each round and
1612  * the initial sub hash key for tag encoding
1613  *
1614  * @param key pointer to key data
1615  * @param key_data GCM expanded key data
1616  *
1617  */
1618 IMB_DLL_EXPORT void aes_gcm_pre_128_sse(const void *key,
1619                                         struct gcm_key_data *key_data);
1620 IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2(const void *key,
1621                                              struct gcm_key_data *key_data);
1622 IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4(const void *key,
1623                                              struct gcm_key_data *key_data);
1624 IMB_DLL_EXPORT void aes_gcm_pre_192_sse(const void *key,
1625                                         struct gcm_key_data *key_data);
1626 IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2(const void *key,
1627                                              struct gcm_key_data *key_data);
1628 IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4(const void *key,
1629                                              struct gcm_key_data *key_data);
1630 IMB_DLL_EXPORT void aes_gcm_pre_256_sse(const void *key,
1631                                         struct gcm_key_data *key_data);
1632 IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2(const void *key,
1633                                              struct gcm_key_data *key_data);
1634 IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4(const void *key,
1635                                              struct gcm_key_data *key_data);
1636 #endif /* !NO_GCM */
1637
1638 #ifdef __cplusplus
1639 }
1640 #endif
1641
1642 #endif /* IMB_IPSEC_MB_H */