ceph/src/spdk/intel-ipsec-mb/avx/kasumi_avx.c

   1 /*******************************************************************************
   2   Copyright (c) 2009-2019, Intel Corporation
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions are met:
   6
   7       * Redistributions of source code must retain the above copyright notice,
   8         this list of conditions and the following disclaimer.
   9       * Redistributions in binary form must reproduce the above copyright
  10         notice, this list of conditions and the following disclaimer in the
  11         documentation and/or other materials provided with the distribution.
  12       * Neither the name of Intel Corporation nor the names of its contributors
  13         may be used to endorse or promote products derived from this software
  14         without specific prior written permission.
  15
  16   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19   DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23   CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 *******************************************************************************/
  27
  28 #include <limits.h>
  29
  30 #define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_avx
  31
  32 #include "include/save_xmms.h"
  33 #include "include/kasumi_internal.h"
  34 #include "include/save_xmms.h"
  35 #include "include/clear_regs_mem.h"
  36
  37 #define SAVE_XMMS               save_xmms_avx
  38 #define RESTORE_XMMS            restore_xmms_avx
  39
  40 void
  41 kasumi_f8_1_buffer_avx(const kasumi_key_sched_t *pCtx, const uint64_t IV,
  42                        const void *pBufferIn, void *pBufferOut,
  43                        const uint32_t cipherLengthInBytes)
  44 {
  45 #ifndef LINUX
  46         DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
  47
  48         SAVE_XMMS(xmm_save);
  49 #endif
  50 #ifdef SAFE_PARAM
  51         /* Check for NULL pointers */
  52         if (pCtx == NULL || pBufferIn == NULL || pBufferOut == NULL)
  53                 return;
  54
  55         /* Check input data is in range of supported length */
  56         if (cipherLengthInBytes == 0 ||
  57             cipherLengthInBytes > (KASUMI_MAX_LEN / CHAR_BIT))
  58                 return;
  59 #endif
  60         kasumi_f8_1_buffer(pCtx, IV, pBufferIn, pBufferOut,
  61                            cipherLengthInBytes);
  62 #ifdef SAFE_DATA
  63         /* Clear sensitive data in registers */
  64         CLEAR_SCRATCH_GPS();
  65         CLEAR_SCRATCH_SIMD_REGS();
  66 #endif
  67 #ifndef LINUX
  68         RESTORE_XMMS(xmm_save);
  69 #endif
  70 }
  71
  72 void
  73 kasumi_f8_1_buffer_bit_avx(const kasumi_key_sched_t *pCtx, const uint64_t IV,
  74                            const void *pBufferIn,
  75                            void *pBufferOut,
  76                            const uint32_t cipherLengthInBits,
  77                            const uint32_t offsetInBits)
  78 {
  79 #ifndef LINUX
  80         DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
  81
  82         SAVE_XMMS(xmm_save);
  83 #endif
  84 #ifdef SAFE_PARAM
  85         /* Check for NULL pointers */
  86         if (pCtx == NULL || pBufferIn == NULL || pBufferOut == NULL)
  87                 return;
  88
  89         /* Check input data is in range of supported length */
  90         if (cipherLengthInBits == 0 ||
  91             cipherLengthInBits > KASUMI_MAX_LEN)
  92                 return;
  93 #endif
  94         kasumi_f8_1_buffer_bit(pCtx, IV, pBufferIn, pBufferOut,
  95                                cipherLengthInBits, offsetInBits);
  96 #ifdef SAFE_DATA
  97         /* Clear sensitive data in registers */
  98         CLEAR_SCRATCH_GPS();
  99         CLEAR_SCRATCH_SIMD_REGS();
 100 #endif
 101 #ifndef LINUX
 102         RESTORE_XMMS(xmm_save);
 103 #endif
 104 }
 105
 106 void
 107 kasumi_f8_2_buffer_avx(const kasumi_key_sched_t *pCtx, const uint64_t IV1,
 108                        const uint64_t IV2, const void *pBufferIn1,
 109                        void *pBufferOut1, const uint32_t lengthInBytes1,
 110                        const void *pBufferIn2, void *pBufferOut2,
 111                        const uint32_t lengthInBytes2)
 112 {
 113 #ifndef LINUX
 114         DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
 115
 116         SAVE_XMMS(xmm_save);
 117 #endif
 118 #ifdef SAFE_PARAM
 119         /* Check for NULL pointers */
 120         if (pCtx == NULL)
 121                 return;
 122
 123         if (pBufferIn1 == NULL || pBufferOut1 == NULL)
 124                 return;
 125
 126         if (pBufferIn2 == NULL || pBufferOut2 == NULL)
 127                 return;
 128
 129         /* Check input data is in range of supported length */
 130         if (lengthInBytes1 == 0 || lengthInBytes1 > (KASUMI_MAX_LEN / CHAR_BIT))
 131                 return;
 132
 133         if (lengthInBytes2 == 0 || lengthInBytes2 > (KASUMI_MAX_LEN / CHAR_BIT))
 134                 return;
 135 #endif
 136         kasumi_f8_2_buffer(pCtx, IV1, IV2,
 137                            pBufferIn1, pBufferOut1, lengthInBytes1,
 138                            pBufferIn2, pBufferOut2, lengthInBytes2);
 139 #ifdef SAFE_DATA
 140         /* Clear sensitive data in registers */
 141         CLEAR_SCRATCH_GPS();
 142         CLEAR_SCRATCH_SIMD_REGS();
 143 #endif
 144 #ifndef LINUX
 145         RESTORE_XMMS(xmm_save);
 146 #endif
 147 }
 148
 149 void
 150 kasumi_f8_3_buffer_avx(const kasumi_key_sched_t *pCtx, const uint64_t IV1,
 151                        const uint64_t IV2, const uint64_t IV3,
 152                        const void *pBufferIn1, void *pBufferOut1,
 153                        const void *pBufferIn2, void *pBufferOut2,
 154                        const void *pBufferIn3, void *pBufferOut3,
 155                        const uint32_t lengthInBytes)
 156 {
 157 #ifndef LINUX
 158         DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
 159
 160         SAVE_XMMS(xmm_save);
 161 #endif
 162 #ifdef SAFE_PARAM
 163         /* Check for NULL pointers */
 164         if (pCtx == NULL)
 165                 return;
 166
 167         if (pBufferIn1 == NULL || pBufferOut1 == NULL)
 168                 return;
 169
 170         if (pBufferIn2 == NULL || pBufferOut2 == NULL)
 171                 return;
 172
 173         if (pBufferIn3 == NULL || pBufferOut3 == NULL)
 174                 return;
 175
 176         /* Check input data is in range of supported length */
 177         if (lengthInBytes == 0 || lengthInBytes > (KASUMI_MAX_LEN / CHAR_BIT))
 178                 return;
 179 #endif
 180         kasumi_f8_3_buffer(pCtx, IV1, IV2, IV3,
 181                            pBufferIn1, pBufferOut1,
 182                            pBufferIn2, pBufferOut2,
 183                            pBufferIn3, pBufferOut3, lengthInBytes);
 184 #ifdef SAFE_DATA
 185         /* Clear sensitive data in registers */
 186         CLEAR_SCRATCH_GPS();
 187         CLEAR_SCRATCH_SIMD_REGS();
 188 #endif
 189 #ifndef LINUX
 190         RESTORE_XMMS(xmm_save);
 191 #endif
 192 }
 193
 194 void
 195 kasumi_f8_4_buffer_avx(const kasumi_key_sched_t *pCtx,
 196                        const uint64_t IV1, const uint64_t IV2,
 197                        const uint64_t IV3, const uint64_t IV4,
 198                        const void *pBufferIn1, void *pBufferOut1,
 199                        const void *pBufferIn2, void *pBufferOut2,
 200                        const void *pBufferIn3, void *pBufferOut3,
 201                        const void *pBufferIn4, void *pBufferOut4,
 202                        const uint32_t lengthInBytes)
 203 {
 204 #ifndef LINUX
 205         DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
 206
 207         SAVE_XMMS(xmm_save);
 208 #endif
 209 #ifdef SAFE_PARAM
 210         /* Check for NULL pointers */
 211         if (pCtx == NULL)
 212                 return;
 213
 214         if (pBufferIn1 == NULL || pBufferOut1 == NULL)
 215                 return;
 216
 217         if (pBufferIn2 == NULL || pBufferOut2 == NULL)
 218                 return;
 219
 220         if (pBufferIn3 == NULL || pBufferOut3 == NULL)
 221                 return;
 222
 223         if (pBufferIn4 == NULL || pBufferOut4 == NULL)
 224                 return;
 225
 226         /* Check input data is in range of supported length */
 227         if (lengthInBytes == 0 || lengthInBytes > (KASUMI_MAX_LEN / CHAR_BIT))
 228                 return;
 229 #endif
 230         kasumi_f8_4_buffer(pCtx, IV1, IV2, IV3, IV4,
 231                            pBufferIn1, pBufferOut1,
 232                            pBufferIn2, pBufferOut2,
 233                            pBufferIn3, pBufferOut3,
 234                            pBufferIn4, pBufferOut4,
 235                            lengthInBytes);
 236 #ifdef SAFE_DATA
 237         /* Clear sensitive data in registers */
 238         CLEAR_SCRATCH_GPS();
 239         CLEAR_SCRATCH_SIMD_REGS();
 240 #endif
 241 #ifndef LINUX
 242         RESTORE_XMMS(xmm_save);
 243 #endif
 244 }
 245
 246 void
 247 kasumi_f8_n_buffer_avx(const kasumi_key_sched_t *pKeySchedule,
 248                        const uint64_t IV[],
 249                        const void * const pDataIn[], void *pDataOut[],
 250                        const uint32_t dataLen[], const uint32_t dataCount)
 251 {
 252 #ifndef LINUX
 253         DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
 254
 255         SAVE_XMMS(xmm_save);
 256 #endif
 257         uint32_t numLeft = dataCount;
 258         const uint64_t *IVPtr;
 259         const void * const *pDataInPtr;
 260         void **pDataOutPtr;
 261         const uint32_t *dataLenPtr;
 262         uint32_t i = 0;
 263         uint32_t numBuffs;
 264
 265 #ifdef SAFE_PARAM
 266         /* Check for NULL pointers */
 267         if (pKeySchedule == NULL || pDataIn == NULL || pDataOut == NULL ||
 268             dataLen == NULL || IV == NULL)
 269                 return;
 270
 271         for (i = 0; i < dataCount; i++) {
 272                 /* Check for NULL pointers */
 273                 if (pDataIn[i] == NULL || pDataOut[i] == NULL)
 274                         return;
 275
 276                 /* Check input data is in range of supported length */
 277                 if (dataLen[i] == 0 || dataLen[i] > (KASUMI_MAX_LEN / CHAR_BIT))
 278                         return;
 279         }
 280 #endif
 281
 282         i = 0;
 283
 284         /* KASUMI F8 n buffer function can handle up to 16 buffers */
 285         while (numLeft > 0) {
 286                 IVPtr = &IV[i];
 287                 pDataInPtr = &pDataIn[i];
 288                 pDataOutPtr = &pDataOut[i];
 289                 dataLenPtr = &dataLen[i];
 290                 numBuffs = (numLeft > 16) ? 16 : numLeft;
 291
 292                 kasumi_f8_n_buffer(pKeySchedule, IVPtr, pDataInPtr, pDataOutPtr,
 293                                    dataLenPtr, numBuffs);
 294                 i += numBuffs;
 295                 numLeft -= numBuffs;
 296         }
 297 #ifdef SAFE_DATA
 298         /* Clear sensitive data in registers */
 299         CLEAR_SCRATCH_GPS();
 300         CLEAR_SCRATCH_SIMD_REGS();
 301 #endif
 302 #ifndef LINUX
 303         RESTORE_XMMS(xmm_save);
 304 #endif
 305 }
 306
 307
 308 void
 309 kasumi_f9_1_buffer_avx(const kasumi_key_sched_t *pCtx, const void *pBufferIn,
 310                        const uint32_t lengthInBytes, void *pDigest)
 311 {
 312 #ifndef LINUX
 313         DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
 314
 315         SAVE_XMMS(xmm_save);
 316 #endif
 317 #ifdef SAFE_PARAM
 318         /* Check for NULL pointers */
 319         if (pCtx == NULL || pBufferIn == NULL || pDigest == NULL)
 320                 return;
 321
 322         /* Check input data is in range of supported length */
 323         if (lengthInBytes == 0 || lengthInBytes > (KASUMI_MAX_LEN / CHAR_BIT))
 324                 return;
 325 #endif
 326         kasumi_f9_1_buffer(pCtx, pBufferIn, lengthInBytes, pDigest);
 327 #ifdef SAFE_DATA
 328         /* Clear sensitive data in registers */
 329         CLEAR_SCRATCH_GPS();
 330         CLEAR_SCRATCH_SIMD_REGS();
 331 #endif
 332 #ifndef LINUX
 333         RESTORE_XMMS(xmm_save);
 334 #endif
 335 }
 336
 337 void
 338 kasumi_f9_1_buffer_user_avx(const kasumi_key_sched_t *pCtx, const uint64_t IV,
 339                             const void *pBufferIn, const uint32_t lengthInBits,
 340                             void *pDigest, const uint32_t direction)
 341 {
 342 #ifndef LINUX
 343         DECLARE_ALIGNED(uint128_t xmm_save[10], 16);
 344
 345         SAVE_XMMS(xmm_save);
 346 #endif
 347 #ifdef SAFE_PARAM
 348         /* Check for NULL pointers */
 349         if (pCtx == NULL || pBufferIn == NULL || pDigest == NULL)
 350                 return;
 351
 352         /* Check input data is in range of supported length */
 353         if (lengthInBits == 0 || lengthInBits > KASUMI_MAX_LEN)
 354                 return;
 355 #endif
 356         kasumi_f9_1_buffer_user(pCtx, IV, pBufferIn, lengthInBits,
 357                                 pDigest, direction);
 358 #ifdef SAFE_DATA
 359         /* Clear sensitive data in registers */
 360         CLEAR_SCRATCH_GPS();
 361         CLEAR_SCRATCH_SIMD_REGS();
 362 #endif
 363 #ifndef LINUX
 364         RESTORE_XMMS(xmm_save);
 365 #endif
 366 }
 367
 368 int
 369 kasumi_init_f8_key_sched_avx(const void *const pKey,
 370                              kasumi_key_sched_t *pCtx)
 371 {
 372         return kasumi_init_f8_key_sched(pKey, pCtx);
 373 }
 374
 375 int
 376 kasumi_init_f9_key_sched_avx(const void *const pKey,
 377                              kasumi_key_sched_t *pCtx)
 378 {
 379         return kasumi_init_f9_key_sched(pKey, pCtx);
 380 }
 381
 382 size_t
 383 kasumi_key_sched_size_avx(void)
 384 {
 385         return kasumi_key_sched_size();
 386 }