ceph/src/crypto/isa-l/isa-l_crypto/include/sha256_mb.h

   1 /**********************************************************************
   2   Copyright(c) 2011-2016 Intel Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Intel Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29
  30 #ifndef _SHA256_MB_H_
  31 #define _SHA256_MB_H_
  32
  33 /**
  34  *  @file sha256_mb.h
  35  *  @brief Multi-buffer CTX API SHA256 function prototypes and structures
  36  *
  37  * Interface for multi-buffer SHA256 functions
  38  *
  39  * <b> Multi-buffer SHA256  Entire or First-Update..Update-Last </b>
  40  *
  41  * The interface to this multi-buffer hashing code is carried out through the
  42  * context-level (CTX) init, submit and flush functions and the SHA256_HASH_CTX_MGR and
  43  * SHA256_HASH_CTX objects. Numerous SHA256_HASH_CTX objects may be instantiated by the
  44  * application for use with a single SHA256_HASH_CTX_MGR.
  45  *
  46  * The CTX interface functions carry out the initialization and padding of the jobs
  47  * entered by the user and add them to the multi-buffer manager. The lower level "scheduler"
  48  * layer then processes the jobs in an out-of-order manner. The scheduler layer functions
  49  * are internal and are not intended to be invoked directly. Jobs can be submitted
  50  * to a CTX as a complete buffer to be hashed, using the HASH_ENTIRE flag, or as partial
  51  * jobs which can be started using the HASH_FIRST flag, and later resumed or finished
  52  * using the HASH_UPDATE and HASH_LAST flags respectively.
  53  *
  54  * <b>Note:</b> The submit function does not require data buffers to be block sized.
  55  *
  56  * The SHA256 CTX interface functions are available for 4 architectures: SSE, AVX, AVX2 and
  57  * AVX512. In addition, a multibinary interface is provided, which selects the appropriate
  58  * architecture-specific function at runtime.
  59  *
  60  * <b>Usage:</b> The application creates a SHA256_HASH_CTX_MGR object and initializes it
  61  * with a call to sha256_ctx_mgr_init*() function, where henceforth "*" stands for the
  62  * relevant suffix for each architecture; _sse, _avx, _avx2, _avx512(or no suffix for the
  63  * multibinary version). The SHA256_HASH_CTX_MGR object will be used to schedule processor
  64  * resources, with up to 4 SHA256_HASH_CTX objects (or 8 in the AVX2 case, 16 in the AVX512)
  65  * being processed at a time.
  66  *
  67  * Each SHA256_HASH_CTX must be initialized before first use by the hash_ctx_init macro
  68  * defined in multi_buffer.h. After initialization, the application may begin computing
  69  * a hash by giving the SHA256_HASH_CTX to a SHA256_HASH_CTX_MGR using the submit functions
  70  * sha256_ctx_mgr_submit*() with the HASH_FIRST flag set. When the SHA256_HASH_CTX is
  71  * returned to the application (via this or a later call to sha256_ctx_mgr_submit*() or
  72  * sha256_ctx_mgr_flush*()), the application can then re-submit it with another call to
  73  * sha256_ctx_mgr_submit*(), but without the HASH_FIRST flag set.
  74  *
  75  * Ideally, on the last buffer for that hash, sha256_ctx_mgr_submit_sse is called with
  76  * HASH_LAST, although it is also possible to submit the hash with HASH_LAST and a zero
  77  * length if necessary. When a SHA256_HASH_CTX is returned after having been submitted with
  78  * HASH_LAST, it will contain a valid hash. The SHA256_HASH_CTX can be reused immediately
  79  * by submitting with HASH_FIRST.
  80  *
  81  * For example, you would submit hashes with the following flags for the following numbers
  82  * of buffers:
  83  * <ul>
  84  *  <li> one buffer: HASH_FIRST | HASH_LAST  (or, equivalently, HASH_ENTIRE)
  85  *  <li> two buffers: HASH_FIRST, HASH_LAST
  86  *  <li> three buffers: HASH_FIRST, HASH_UPDATE, HASH_LAST
  87  * etc.
  88  * </ul>
  89  *
  90  * The order in which SHA256_CTX objects are returned is in general different from the order
  91  * in which they are submitted.
  92  *
  93  * A few possible error conditions exist:
  94  * <ul>
  95  *  <li> Submitting flags other than the allowed entire/first/update/last values
  96  *  <li> Submitting a context that is currently being managed by a SHA256_HASH_CTX_MGR.
  97  *  <li> Submitting a context after HASH_LAST is used but before HASH_FIRST is set.
  98  * </ul>
  99  *
 100  *  These error conditions are reported by returning the SHA256_HASH_CTX immediately after
 101  *  a submit with its error member set to a non-zero error code (defined in
 102  *  multi_buffer.h). No changes are made to the SHA256_HASH_CTX_MGR in the case of an
 103  *  error; no processing is done for other hashes.
 104  *
 105  */
 106
 107 #include <stdint.h>
 108 #include "multi_buffer.h"
 109 #include "types.h"
 110
 111 #ifndef _MSC_VER
 112 #include <stdbool.h>
 113 #endif
 114
 115 #ifdef __cplusplus
 116 extern "C" {
 117 #endif
 118
 119 // Hash Constants and Typedefs
 120 #define SHA256_DIGEST_NWORDS            8
 121 #define SHA256_MAX_LANES                16
 122 #define SHA256_X8_LANES                 8
 123 #define SHA256_MIN_LANES                4
 124 #define SHA256_BLOCK_SIZE               64
 125 #define SHA256_LOG2_BLOCK_SIZE          6
 126 #define SHA256_PADLENGTHFIELD_SIZE      8
 127 #define SHA256_INITIAL_DIGEST           \
 128         0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, \
 129         0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19
 130
 131 typedef uint32_t sha256_digest_array[SHA256_DIGEST_NWORDS][SHA256_MAX_LANES];
 132 typedef uint32_t SHA256_WORD_T;
 133
 134 /** @brief Scheduler layer - Holds info describing a single SHA256 job for the multi-buffer manager */
 135
 136 typedef struct {
 137         uint8_t*  buffer;       //!< pointer to data buffer for this job
 138         uint64_t  len;          //!< length of buffer for this job in blocks.
 139         DECLARE_ALIGNED(uint32_t result_digest[SHA256_DIGEST_NWORDS], 64);
 140         JOB_STS status;         //!< output job status
 141         void*   user_data;      //!< pointer for user's job-related data
 142 } SHA256_JOB;
 143
 144 /** @brief Scheduler layer -  Holds arguments for submitted SHA256 job */
 145
 146 typedef struct {
 147         sha256_digest_array digest;
 148         uint8_t* data_ptr[SHA256_MAX_LANES];
 149 } SHA256_MB_ARGS_X16;
 150
 151 /** @brief Scheduler layer - Lane data */
 152
 153 typedef struct {
 154         SHA256_JOB *job_in_lane;
 155 } SHA256_LANE_DATA;
 156
 157 /** @brief Scheduler layer - Holds state for multi-buffer SHA256 jobs */
 158
 159 typedef struct {
 160         SHA256_MB_ARGS_X16 args;
 161         uint32_t lens[SHA256_MAX_LANES];
 162         uint64_t unused_lanes; //!< each nibble is index (0...3 or 0...7) of unused lanes, nibble 4 or 8 is set to F as a flag
 163         SHA256_LANE_DATA ldata[SHA256_MAX_LANES];
 164         uint32_t num_lanes_inuse;
 165 } SHA256_MB_JOB_MGR;
 166
 167 /** @brief Context layer - Holds state for multi-buffer SHA256 jobs */
 168
 169 typedef struct {
 170         SHA256_MB_JOB_MGR mgr;
 171 } SHA256_HASH_CTX_MGR;
 172
 173 /** @brief Context layer - Holds info describing a single SHA256 job for the multi-buffer CTX manager */
 174
 175 typedef struct {
 176         SHA256_JOB      job; // Must be at struct offset 0.
 177         HASH_CTX_STS    status;         //!< Context status flag
 178         HASH_CTX_ERROR  error;          //!< Context error flag
 179         uint64_t        total_length;   //!< Running counter of length processed for this CTX's job
 180         const void*     incoming_buffer; //!< pointer to data input buffer for this CTX's job
 181         uint32_t        incoming_buffer_length; //!< length of buffer for this job in bytes.
 182         uint8_t         partial_block_buffer[SHA256_BLOCK_SIZE * 2]; //!< CTX partial blocks
 183         uint32_t        partial_block_buffer_length;
 184         void*           user_data;      //!< pointer for user to keep any job-related data
 185 } SHA256_HASH_CTX;
 186
 187 /******************** multibinary function prototypes **********************/
 188
 189 /**
 190  * @brief Initialize the SHA256 multi-buffer manager structure.
 191  * @requires SSE4.1 or AVX or AVX2
 192  *
 193  * @param mgr   Structure holding context level state info
 194  * @returns void
 195  */
 196 void      sha256_ctx_mgr_init   (SHA256_HASH_CTX_MGR* mgr);
 197
 198 /**
 199  * @brief  Submit a new SHA256 job to the multi-buffer manager.
 200  * @requires SSE4.1 or AVX or AVX2
 201  *
 202  * @param  mgr Structure holding context level state info
 203  * @param  ctx Structure holding ctx job info
 204  * @param  buffer Pointer to buffer to be processed
 205  * @param  len Length of buffer (in bytes) to be processed
 206  * @param  flags Input flag specifying job type (first, update, last or entire)
 207  * @returns NULL if no jobs complete or pointer to jobs structure.
 208  */
 209 SHA256_HASH_CTX* sha256_ctx_mgr_submit (SHA256_HASH_CTX_MGR* mgr, SHA256_HASH_CTX* ctx,
 210                                         const void* buffer, uint32_t len, HASH_CTX_FLAG flags);
 211
 212 /**
 213  * @brief Finish all submitted SHA256 jobs and return when complete.
 214  * @requires SSE4.1 or AVX or AVX2
 215  *
 216  * @param mgr   Structure holding context level state info
 217  * @returns NULL if no jobs to complete or pointer to jobs structure.
 218  */
 219 SHA256_HASH_CTX* sha256_ctx_mgr_flush  (SHA256_HASH_CTX_MGR* mgr);
 220
 221
 222 /*******************************************************************
 223  * CTX level API function prototypes
 224  ******************************************************************/
 225
 226 /**
 227  * @brief Initialize the context level SHA256 multi-buffer manager structure.
 228  * @requires SSE4.1
 229  *
 230  * @param mgr Structure holding context level state info
 231  * @returns void
 232  */
 233 void      sha256_ctx_mgr_init_sse   (SHA256_HASH_CTX_MGR* mgr);
 234
 235 /**
 236  * @brief  Submit a new SHA256 job to the context level multi-buffer manager.
 237  * @requires SSE4.1
 238  *
 239  * @param  mgr Structure holding context level state info
 240  * @param  ctx Structure holding ctx job info
 241  * @param  buffer Pointer to buffer to be processed
 242  * @param  len Length of buffer (in bytes) to be processed
 243  * @param  flags Input flag specifying job type (first, update, last or entire)
 244  * @returns NULL if no jobs complete or pointer to jobs structure.
 245  */
 246 SHA256_HASH_CTX* sha256_ctx_mgr_submit_sse (SHA256_HASH_CTX_MGR* mgr, SHA256_HASH_CTX* ctx,
 247                                         const void* buffer, uint32_t len, HASH_CTX_FLAG flags);
 248
 249 /**
 250  * @brief Finish all submitted SHA256 jobs and return when complete.
 251  * @requires SSE4.1
 252  *
 253  * @param mgr   Structure holding context level state info
 254  * @returns NULL if no jobs to complete or pointer to jobs structure.
 255  */
 256 SHA256_HASH_CTX* sha256_ctx_mgr_flush_sse  (SHA256_HASH_CTX_MGR* mgr);
 257
 258 /**
 259  * @brief Initialize the context level SHA256 multi-buffer manager structure.
 260  * @requires SSE4.1 and SHANI
 261  *
 262  * @param mgr Structure holding context level state info
 263  * @returns void
 264  */
 265 void      sha256_ctx_mgr_init_sse_ni (SHA256_HASH_CTX_MGR* mgr);
 266
 267 /**
 268  * @brief  Submit a new SHA256 job to the context level multi-buffer manager.
 269  * @requires SSE4.1 and SHANI
 270  *
 271  * @param  mgr Structure holding context level state info
 272  * @param  ctx Structure holding ctx job info
 273  * @param  buffer Pointer to buffer to be processed
 274  * @param  len Length of buffer (in bytes) to be processed
 275  * @param  flags Input flag specifying job type (first, update, last or entire)
 276  * @returns NULL if no jobs complete or pointer to jobs structure.
 277  */
 278 SHA256_HASH_CTX* sha256_ctx_mgr_submit_sse_ni (SHA256_HASH_CTX_MGR* mgr, SHA256_HASH_CTX* ctx,
 279                                 const void* buffer, uint32_t len, HASH_CTX_FLAG flags);
 280
 281 /**
 282  * @brief Finish all submitted SHA256 jobs and return when complete.
 283  * @requires SSE4.1 and SHANI
 284  *
 285  * @param mgr   Structure holding context level state info
 286  * @returns NULL if no jobs to complete or pointer to jobs structure.
 287  */
 288 SHA256_HASH_CTX* sha256_ctx_mgr_flush_sse_ni (SHA256_HASH_CTX_MGR* mgr);
 289
 290 /**
 291  * @brief Initialize the SHA256 multi-buffer manager structure.
 292  * @requires AVX
 293  *
 294  * @param mgr Structure holding context level state info
 295  * @returns void
 296  */
 297 void      sha256_ctx_mgr_init_avx   (SHA256_HASH_CTX_MGR* mgr);
 298
 299 /**
 300  * @brief  Submit a new SHA256 job to the multi-buffer manager.
 301  * @requires AVX
 302  *
 303  * @param  mgr Structure holding context level state info
 304  * @param  ctx Structure holding ctx job info
 305  * @param  buffer Pointer to buffer to be processed
 306  * @param  len Length of buffer (in bytes) to be processed
 307  * @param  flags Input flag specifying job type (first, update, last or entire)
 308  * @returns NULL if no jobs complete or pointer to jobs structure.
 309  */
 310 SHA256_HASH_CTX* sha256_ctx_mgr_submit_avx (SHA256_HASH_CTX_MGR* mgr, SHA256_HASH_CTX* ctx,
 311                                         const void* buffer, uint32_t len, HASH_CTX_FLAG flags);
 312
 313 /**
 314  * @brief Finish all submitted SHA256 jobs and return when complete.
 315  * @requires AVX
 316  *
 317  * @param mgr   Structure holding context level state info
 318  * @returns NULL if no jobs to complete or pointer to jobs structure.
 319  */
 320 SHA256_HASH_CTX* sha256_ctx_mgr_flush_avx  (SHA256_HASH_CTX_MGR* mgr);
 321
 322 /**
 323  * @brief Initialize the SHA256 multi-buffer manager structure.
 324  * @requires AVX2
 325  *
 326  * @param mgr   Structure holding context level state info
 327  * @returns void
 328  */
 329 void      sha256_ctx_mgr_init_avx2   (SHA256_HASH_CTX_MGR* mgr);
 330
 331 /**
 332  * @brief  Submit a new SHA256 job to the multi-buffer manager.
 333  * @requires AVX2
 334  *
 335  * @param  mgr Structure holding context level state info
 336  * @param  ctx Structure holding ctx job info
 337  * @param  buffer Pointer to buffer to be processed
 338  * @param  len Length of buffer (in bytes) to be processed
 339  * @param  flags Input flag specifying job type (first, update, last or entire)
 340  * @returns NULL if no jobs complete or pointer to jobs structure.
 341  */
 342 SHA256_HASH_CTX* sha256_ctx_mgr_submit_avx2 (SHA256_HASH_CTX_MGR* mgr, SHA256_HASH_CTX* ctx,
 343                                         const void* buffer, uint32_t len, HASH_CTX_FLAG flags);
 344
 345 /**
 346  * @brief Finish all submitted SHA256 jobs and return when complete.
 347  * @requires AVX2
 348  *
 349  * @param mgr   Structure holding context level state info
 350  * @returns NULL if no jobs to complete or pointer to jobs structure.
 351  */
 352 SHA256_HASH_CTX* sha256_ctx_mgr_flush_avx2  (SHA256_HASH_CTX_MGR* mgr);
 353
 354 /**
 355  * @brief Initialize the SHA256 multi-buffer manager structure.
 356  * @requires AVX512
 357  *
 358  * @param mgr   Structure holding context level state info
 359  * @returns void
 360  */
 361 void      sha256_ctx_mgr_init_avx512   (SHA256_HASH_CTX_MGR* mgr);
 362
 363 /**
 364  * @brief  Submit a new SHA256 job to the multi-buffer manager.
 365  * @requires AVX512
 366  *
 367  * @param  mgr Structure holding context level state info
 368  * @param  ctx Structure holding ctx job info
 369  * @param  buffer Pointer to buffer to be processed
 370  * @param  len Length of buffer (in bytes) to be processed
 371  * @param  flags Input flag specifying job type (first, update, last or entire)
 372  * @returns NULL if no jobs complete or pointer to jobs structure.
 373  */
 374 SHA256_HASH_CTX* sha256_ctx_mgr_submit_avx512 (SHA256_HASH_CTX_MGR* mgr, SHA256_HASH_CTX* ctx,
 375                                         const void* buffer, uint32_t len, HASH_CTX_FLAG flags);
 376
 377 /**
 378  * @brief Finish all submitted SHA256 jobs and return when complete.
 379  * @requires AVX512
 380  *
 381  * @param mgr   Structure holding context level state info
 382  * @returns NULL if no jobs to complete or pointer to jobs structure.
 383  */
 384 SHA256_HASH_CTX* sha256_ctx_mgr_flush_avx512  (SHA256_HASH_CTX_MGR* mgr);
 385
 386 /**
 387  * @brief Initialize the SHA256 multi-buffer manager structure.
 388  * @requires AVX512 and SHANI
 389  *
 390  * @param mgr   Structure holding context level state info
 391  * @returns void
 392  */
 393 void      sha256_ctx_mgr_init_avx512_ni (SHA256_HASH_CTX_MGR* mgr);
 394
 395 /**
 396  * @brief  Submit a new SHA256 job to the multi-buffer manager.
 397  * @requires AVX512 and SHANI
 398  *
 399  * @param  mgr Structure holding context level state info
 400  * @param  ctx Structure holding ctx job info
 401  * @param  buffer Pointer to buffer to be processed
 402  * @param  len Length of buffer (in bytes) to be processed
 403  * @param  flags Input flag specifying job type (first, update, last or entire)
 404  * @returns NULL if no jobs complete or pointer to jobs structure.
 405  */
 406 SHA256_HASH_CTX* sha256_ctx_mgr_submit_avx512_ni (SHA256_HASH_CTX_MGR* mgr, SHA256_HASH_CTX* ctx,
 407                                 const void* buffer, uint32_t len, HASH_CTX_FLAG flags);
 408
 409 /**
 410  * @brief Finish all submitted SHA256 jobs and return when complete.
 411  * @requires AVX512 and SHANI
 412  *
 413  * @param mgr   Structure holding context level state info
 414  * @returns NULL if no jobs to complete or pointer to jobs structure.
 415  */
 416 SHA256_HASH_CTX* sha256_ctx_mgr_flush_avx512_ni (SHA256_HASH_CTX_MGR* mgr);
 417
 418
 419 /*******************************************************************
 420  * Scheduler (internal) level out-of-order function prototypes
 421  ******************************************************************/
 422
 423 void        sha256_mb_mgr_init_sse   (SHA256_MB_JOB_MGR *state);
 424 SHA256_JOB* sha256_mb_mgr_submit_sse (SHA256_MB_JOB_MGR *state, SHA256_JOB* job);
 425 SHA256_JOB* sha256_mb_mgr_flush_sse  (SHA256_MB_JOB_MGR *state);
 426
 427 #define     sha256_mb_mgr_init_avx   sha256_mb_mgr_init_sse
 428 SHA256_JOB* sha256_mb_mgr_submit_avx (SHA256_MB_JOB_MGR *state, SHA256_JOB* job);
 429 SHA256_JOB* sha256_mb_mgr_flush_avx  (SHA256_MB_JOB_MGR *state);
 430
 431 void        sha256_mb_mgr_init_avx2   (SHA256_MB_JOB_MGR *state);
 432 SHA256_JOB* sha256_mb_mgr_submit_avx2 (SHA256_MB_JOB_MGR *state, SHA256_JOB* job);
 433 SHA256_JOB* sha256_mb_mgr_flush_avx2  (SHA256_MB_JOB_MGR *state);
 434
 435 void        sha256_mb_mgr_init_avx512   (SHA256_MB_JOB_MGR *state);
 436 SHA256_JOB* sha256_mb_mgr_submit_avx512 (SHA256_MB_JOB_MGR *state, SHA256_JOB* job);
 437 SHA256_JOB* sha256_mb_mgr_flush_avx512  (SHA256_MB_JOB_MGR *state);
 438
 439 void        sha256_mb_mgr_init_sse_ni    (SHA256_MB_JOB_MGR *state);
 440 SHA256_JOB* sha256_mb_mgr_submit_sse_ni  (SHA256_MB_JOB_MGR *state, SHA256_JOB* job);
 441 SHA256_JOB* sha256_mb_mgr_flush_sse_ni   (SHA256_MB_JOB_MGR *state);
 442
 443 void        sha256_mb_mgr_init_avx512_ni    (SHA256_MB_JOB_MGR *state);
 444 SHA256_JOB* sha256_mb_mgr_submit_avx512_ni  (SHA256_MB_JOB_MGR *state, SHA256_JOB* job);
 445 SHA256_JOB* sha256_mb_mgr_flush_avx512_ni   (SHA256_MB_JOB_MGR *state);
 446
 447 #ifdef __cplusplus
 448 }
 449 #endif
 450
 451 #endif // _SHA256_MB_H_