ceph/src/spdk/intel-ipsec-mb/avx512/sha512_x8_avx512.asm

   1 ;;
   2 ;; Copyright (c) 2017-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 ;; Stack must be aligned to 32 bytes before call
  29 ;;
  30 ;; Registers:           RAX RBX RCX RDX RBP RSI RDI R8  R9  R10 R11 R12 R13 R14 R15
  31 ;;                      -----------------------------------------------------------
  32 ;; Windows clobbers:    RAX         RDX         RDI R8  R9  R10 R11 R12 R13 R14 R15
  33 ;; Windows preserves:       RBX RCX     RBP RSI
  34 ;;                      -----------------------------------------------------------
  35 ;; Linux clobbers:      RAX         RDX     RSI     R8  R9  R10 R11 R12 R13 R14 R15
  36 ;; Linux preserves:         RBX RCX     RBP     RDI
  37 ;;                      -----------------------------------------------------------
  38 ;; Clobbers ZMM0-31
  39
  40 ;; code to compute quad SHA512 using AVX512
  41
  42 %include "include/os.asm"
  43 ;%define DO_DBGPRINT
  44 %include "include/dbgprint.asm"
  45 %include "mb_mgr_datastruct.asm"
  46 %include "include/transpose_avx512.asm"
  47
  48 %define APPEND(a,b) a %+ b
  49
  50 %ifdef LINUX
  51 ; Linux register definitions
  52 %define arg1    rdi
  53 %define arg2    rsi
  54 %define arg3    rcx
  55 %define arg4    rdx
  56 %else
  57 ; Windows definitions
  58 %define arg1    rcx
  59 %define arg2    rdx
  60 %define arg3    rsi
  61 %define arg4    rdi
  62 %endif
  63
  64 %define STATE           arg1
  65 %define INP_SIZE        arg2
  66
  67 %define IDX     arg4
  68 %define TBL     r8
  69
  70 ;; retaining XMM_SAVE,  because the top half of YMM registers no saving required, only bottom half, the XMM part
  71 %define NUM_LANES          8
  72 %define XMM_SAVE           (15-5)*16
  73 %define SZ                 8
  74 %define SZ8                8 * SZ
  75 %define DIGEST_SZ          8 * SZ8
  76 %define DIGEST_SAVE        NUM_LANES * DIGEST_SZ
  77 %define RSP_SAVE           1*8
  78
  79
  80 ; Define Stack Layout
  81 START_FIELDS
  82 ;;;     name            size            align
  83 FIELD   _DIGEST_SAVE,   NUM_LANES*8*64, 64
  84 FIELD   _XMM_SAVE,      XMM_SAVE,       16
  85 FIELD   _RSP,           8,              8
  86 %assign STACK_SPACE     _FIELD_OFFSET
  87
  88 %define inp0    r9
  89 %define inp1    r10
  90 %define inp2    r11
  91 %define inp3    r12
  92 %define inp4    r13
  93 %define inp5    r14
  94 %define inp6    r15
  95 %define inp7    rax
  96
  97 %define A       zmm0
  98 %define B       zmm1
  99 %define C       zmm2
 100 %define D       zmm3
 101 %define E       zmm4
 102 %define F       zmm5
 103 %define G       zmm6
 104 %define H       zmm7
 105 %define T1      zmm8
 106 %define TMP0    zmm9
 107 %define TMP1    zmm10
 108 %define TMP2    zmm11
 109 %define TMP3    zmm12
 110 %define TMP4    zmm13
 111 %define TMP5    zmm14
 112 %define TMP6    zmm15
 113
 114
 115 %define W0      zmm16
 116 %define W1      zmm17
 117 %define W2      zmm18
 118 %define W3      zmm19
 119 %define W4      zmm20
 120 %define W5      zmm21
 121 %define W6      zmm22
 122 %define W7      zmm23
 123 %define W8      zmm24
 124 %define W9      zmm25
 125 %define W10     zmm26
 126 %define W11     zmm27
 127 %define W12     zmm28
 128 %define W13     zmm29
 129 %define W14     zmm30
 130 %define W15     zmm31
 131
 132 ; from sha256_fips180-2.pdf
 133 ; define rotates for Sigma function for main loop steps
 134 %define BIG_SIGMA_0_0 28        ; Sigma0
 135 %define BIG_SIGMA_0_1 34
 136 %define BIG_SIGMA_0_2 39
 137 %define BIG_SIGMA_1_0 14        ; Sigma1
 138 %define BIG_SIGMA_1_1 18
 139 %define BIG_SIGMA_1_2 41
 140
 141 ; define rotates for Sigma function for scheduling steps
 142 %define SMALL_SIGMA_0_0 1       ; sigma0
 143 %define SMALL_SIGMA_0_1 8
 144 %define SMALL_SIGMA_0_2 7
 145 %define SMALL_SIGMA_1_0 19      ; sigma1
 146 %define SMALL_SIGMA_1_1 61
 147 %define SMALL_SIGMA_1_2 6
 148
 149 %define SHA_MAX_ROUNDS 80
 150 %define SHA_ROUNDS_LESS_16 (SHA_MAX_ROUNDS - 16)
 151
 152 %macro ROTATE_ARGS 0
 153 %xdefine TMP_ H
 154 %xdefine H G
 155 %xdefine G F
 156 %xdefine F E
 157 %xdefine E D
 158 %xdefine D C
 159 %xdefine C B
 160 %xdefine B A
 161 %xdefine A TMP_
 162 %endm
 163
 164 ;;  CH(A, B, C) = (A&B) ^ (~A&C)
 165 ;; MAJ(E, F, G) = (E&F) ^ (E&G) ^ (F&G)
 166 ;; SIGMA0 = ROR_28  ^ ROR_34 ^ ROR_39
 167 ;; SIGMA1 = ROR_14  ^ ROR_18 ^ ROR_41
 168 ;; sigma0 = ROR_1  ^ ROR_8 ^ SHR_7
 169 ;; sigma1 = ROR_19 ^ ROR_61 ^ SHR_6
 170
 171 ;; Main processing loop per round
 172 ;;  equivalent to %macro ROUND_00_15 2
 173 %macro PROCESS_LOOP 2
 174 %define %%WT    %1
 175 %define %%ROUND %2
 176         ;; T1 = H + BIG_SIGMA_1(E) + CH(E, F, G) + Kt + Wt
 177         ;; T2 = BIG_SIGMA_0(A) + MAJ(A, B, C)
 178         ;; H=G, G=F, F=E, E=D+T1, D=C, C=B, B=A, A=T1+T2
 179
 180         ;; H becomes T2, then add T1 for A
 181         ;; D becomes D + T1 for E
 182
 183         vpaddq          T1, H, TMP3             ; T1 = H + Kt
 184         vmovdqa32       TMP0, E
 185         ;; compute BIG_SIGMA_1(E)
 186         vprorq          TMP1, E, BIG_SIGMA_1_0          ; ROR_14(E)
 187         vprorq          TMP2, E, BIG_SIGMA_1_1          ; ROR_18(E)
 188         vprorq          TMP3, E, BIG_SIGMA_1_2          ; ROR_41(E)
 189         vpternlogq      TMP1, TMP2, TMP3, 0x96  ; TMP1 = BIG_SIGMA_1(E)
 190         vpternlogq      TMP0, F, G, 0xCA        ; TMP0 = CH(E,F,G)
 191         vpaddq          T1, T1, %%WT            ; T1 = T1 + Wt
 192         vpaddq          T1, T1, TMP0            ; T1 = T1 + CH(E,F,G)
 193         vpaddq          T1, T1, TMP1            ; T1 = T1 + BIG_SIGMA_1(E)
 194         vpaddq          D, D, T1                ; D = D + T1
 195         vprorq          H, A, BIG_SIGMA_0_0     ;ROR_28(A)
 196         vprorq          TMP2, A, BIG_SIGMA_0_1  ;ROR_34(A)
 197         vprorq          TMP3, A, BIG_SIGMA_0_2  ;ROR_39(A)
 198         vmovdqa32       TMP0, A
 199         vpternlogq      TMP0, B, C, 0xE8        ; TMP0 = MAJ(A,B,C)
 200         vpternlogq      H, TMP2, TMP3, 0x96     ; H(T2) = BIG_SIGMA_0(A)
 201         vpaddq          H, H, TMP0              ; H(T2) = BIG_SIGMA_0(A) + MAJ(A,B,C)
 202         vpaddq          H, H, T1                ; H(A) = H(T2) + T1
 203         vmovdqa32       TMP3, [TBL + ((%%ROUND+1)*64)]  ; Next Kt
 204
 205         ;; Rotate the args A-H (rotation of names associated with regs)
 206         ROTATE_ARGS
 207 %endmacro
 208
 209 %macro MSG_SCHED_ROUND_16_79 4
 210 %define %%WT    %1
 211 %define %%WTp1  %2
 212 %define %%WTp9  %3
 213 %define %%WTp14 %4
 214         vprorq          TMP4, %%WTp14, SMALL_SIGMA_1_0  ; ROR_19(Wt-2)
 215         vprorq          TMP5, %%WTp14, SMALL_SIGMA_1_1  ; ROR_61(Wt-2)
 216         vpsrlq          TMP6, %%WTp14, SMALL_SIGMA_1_2  ; SHR_6(Wt-2)
 217         vpternlogq      TMP4, TMP5, TMP6, 0x96          ; TMP4 = sigma_1(Wt-2)
 218
 219         vpaddq          %%WT, %%WT, TMP4        ; Wt = Wt-16 + sigma_1(Wt-2)
 220         vpaddq          %%WT, %%WT, %%WTp9      ; Wt = Wt-16 + sigma_1(Wt-2) + Wt-7
 221
 222         vprorq          TMP4, %%WTp1, SMALL_SIGMA_0_0   ; ROR_1(Wt-15)
 223         vprorq          TMP5, %%WTp1, SMALL_SIGMA_0_1   ; ROR_8(Wt-15)
 224         vpsrlq          TMP6, %%WTp1, SMALL_SIGMA_0_2   ; SHR_7(Wt-15)
 225         vpternlogq      TMP4, TMP5, TMP6, 0x96          ; TMP4 = sigma_0(Wt-15)
 226
 227         vpaddq          %%WT, %%WT, TMP4        ; Wt = Wt-16 + sigma_1(Wt-2) +
 228                                                 ; Wt-7 + sigma_0(Wt-15) +
 229 %endmacro
 230
 231 section .data
 232 default rel
 233
 234 align 64
 235 ; 80 constants for SHA512
 236 ; replicating for each lane, thus 8*80
 237 ; to aid in SIMD .. space tradeoff for time!
 238 ; local to asm file, used nowhere else
 239 TABLE:
 240         dq      0x428a2f98d728ae22, 0x428a2f98d728ae22, 0x428a2f98d728ae22, 0x428a2f98d728ae22
 241         dq      0x428a2f98d728ae22, 0x428a2f98d728ae22, 0x428a2f98d728ae22, 0x428a2f98d728ae22
 242         dq      0x7137449123ef65cd, 0x7137449123ef65cd, 0x7137449123ef65cd, 0x7137449123ef65cd
 243         dq      0x7137449123ef65cd, 0x7137449123ef65cd, 0x7137449123ef65cd, 0x7137449123ef65cd
 244         dq      0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f
 245         dq      0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f
 246         dq      0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc
 247         dq      0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc
 248         dq      0x3956c25bf348b538, 0x3956c25bf348b538, 0x3956c25bf348b538, 0x3956c25bf348b538
 249         dq      0x3956c25bf348b538, 0x3956c25bf348b538, 0x3956c25bf348b538, 0x3956c25bf348b538
 250         dq      0x59f111f1b605d019, 0x59f111f1b605d019, 0x59f111f1b605d019, 0x59f111f1b605d019
 251         dq      0x59f111f1b605d019, 0x59f111f1b605d019, 0x59f111f1b605d019, 0x59f111f1b605d019
 252         dq      0x923f82a4af194f9b, 0x923f82a4af194f9b, 0x923f82a4af194f9b, 0x923f82a4af194f9b
 253         dq      0x923f82a4af194f9b, 0x923f82a4af194f9b, 0x923f82a4af194f9b, 0x923f82a4af194f9b
 254         dq      0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118
 255         dq      0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118
 256         dq      0xd807aa98a3030242, 0xd807aa98a3030242, 0xd807aa98a3030242, 0xd807aa98a3030242
 257         dq      0xd807aa98a3030242, 0xd807aa98a3030242, 0xd807aa98a3030242, 0xd807aa98a3030242
 258         dq      0x12835b0145706fbe, 0x12835b0145706fbe, 0x12835b0145706fbe, 0x12835b0145706fbe
 259         dq      0x12835b0145706fbe, 0x12835b0145706fbe, 0x12835b0145706fbe, 0x12835b0145706fbe
 260         dq      0x243185be4ee4b28c, 0x243185be4ee4b28c, 0x243185be4ee4b28c, 0x243185be4ee4b28c
 261         dq      0x243185be4ee4b28c, 0x243185be4ee4b28c, 0x243185be4ee4b28c, 0x243185be4ee4b28c
 262         dq      0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2
 263         dq      0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2
 264         dq      0x72be5d74f27b896f, 0x72be5d74f27b896f, 0x72be5d74f27b896f, 0x72be5d74f27b896f
 265         dq      0x72be5d74f27b896f, 0x72be5d74f27b896f, 0x72be5d74f27b896f, 0x72be5d74f27b896f
 266         dq      0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1
 267         dq      0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1
 268         dq      0x9bdc06a725c71235, 0x9bdc06a725c71235, 0x9bdc06a725c71235, 0x9bdc06a725c71235
 269         dq      0x9bdc06a725c71235, 0x9bdc06a725c71235, 0x9bdc06a725c71235, 0x9bdc06a725c71235
 270         dq      0xc19bf174cf692694, 0xc19bf174cf692694, 0xc19bf174cf692694, 0xc19bf174cf692694
 271         dq      0xc19bf174cf692694, 0xc19bf174cf692694, 0xc19bf174cf692694, 0xc19bf174cf692694
 272         dq      0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2
 273         dq      0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2
 274         dq      0xefbe4786384f25e3, 0xefbe4786384f25e3, 0xefbe4786384f25e3, 0xefbe4786384f25e3
 275         dq      0xefbe4786384f25e3, 0xefbe4786384f25e3, 0xefbe4786384f25e3, 0xefbe4786384f25e3
 276         dq      0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5
 277         dq      0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5
 278         dq      0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65
 279         dq      0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65
 280         dq      0x2de92c6f592b0275, 0x2de92c6f592b0275, 0x2de92c6f592b0275, 0x2de92c6f592b0275
 281         dq      0x2de92c6f592b0275, 0x2de92c6f592b0275, 0x2de92c6f592b0275, 0x2de92c6f592b0275
 282         dq      0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483
 283         dq      0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483
 284         dq      0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4
 285         dq      0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4
 286         dq      0x76f988da831153b5, 0x76f988da831153b5, 0x76f988da831153b5, 0x76f988da831153b5
 287         dq      0x76f988da831153b5, 0x76f988da831153b5, 0x76f988da831153b5, 0x76f988da831153b5
 288         dq      0x983e5152ee66dfab, 0x983e5152ee66dfab, 0x983e5152ee66dfab, 0x983e5152ee66dfab
 289         dq      0x983e5152ee66dfab, 0x983e5152ee66dfab, 0x983e5152ee66dfab, 0x983e5152ee66dfab
 290         dq      0xa831c66d2db43210, 0xa831c66d2db43210, 0xa831c66d2db43210, 0xa831c66d2db43210
 291         dq      0xa831c66d2db43210, 0xa831c66d2db43210, 0xa831c66d2db43210, 0xa831c66d2db43210
 292         dq      0xb00327c898fb213f, 0xb00327c898fb213f, 0xb00327c898fb213f, 0xb00327c898fb213f
 293         dq      0xb00327c898fb213f, 0xb00327c898fb213f, 0xb00327c898fb213f, 0xb00327c898fb213f
 294         dq      0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4
 295         dq      0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4
 296         dq      0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2
 297         dq      0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2
 298         dq      0xd5a79147930aa725, 0xd5a79147930aa725, 0xd5a79147930aa725, 0xd5a79147930aa725
 299         dq      0xd5a79147930aa725, 0xd5a79147930aa725, 0xd5a79147930aa725, 0xd5a79147930aa725
 300         dq      0x06ca6351e003826f, 0x06ca6351e003826f, 0x06ca6351e003826f, 0x06ca6351e003826f
 301         dq      0x06ca6351e003826f, 0x06ca6351e003826f, 0x06ca6351e003826f, 0x06ca6351e003826f
 302         dq      0x142929670a0e6e70, 0x142929670a0e6e70, 0x142929670a0e6e70, 0x142929670a0e6e70
 303         dq      0x142929670a0e6e70, 0x142929670a0e6e70, 0x142929670a0e6e70, 0x142929670a0e6e70
 304         dq      0x27b70a8546d22ffc, 0x27b70a8546d22ffc, 0x27b70a8546d22ffc, 0x27b70a8546d22ffc
 305         dq      0x27b70a8546d22ffc, 0x27b70a8546d22ffc, 0x27b70a8546d22ffc, 0x27b70a8546d22ffc
 306         dq      0x2e1b21385c26c926, 0x2e1b21385c26c926, 0x2e1b21385c26c926, 0x2e1b21385c26c926
 307         dq      0x2e1b21385c26c926, 0x2e1b21385c26c926, 0x2e1b21385c26c926, 0x2e1b21385c26c926
 308         dq      0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed
 309         dq      0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed
 310         dq      0x53380d139d95b3df, 0x53380d139d95b3df, 0x53380d139d95b3df, 0x53380d139d95b3df
 311         dq      0x53380d139d95b3df, 0x53380d139d95b3df, 0x53380d139d95b3df, 0x53380d139d95b3df
 312         dq      0x650a73548baf63de, 0x650a73548baf63de, 0x650a73548baf63de, 0x650a73548baf63de
 313         dq      0x650a73548baf63de, 0x650a73548baf63de, 0x650a73548baf63de, 0x650a73548baf63de
 314         dq      0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8
 315         dq      0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8
 316         dq      0x81c2c92e47edaee6, 0x81c2c92e47edaee6, 0x81c2c92e47edaee6, 0x81c2c92e47edaee6
 317         dq      0x81c2c92e47edaee6, 0x81c2c92e47edaee6, 0x81c2c92e47edaee6, 0x81c2c92e47edaee6
 318         dq      0x92722c851482353b, 0x92722c851482353b, 0x92722c851482353b, 0x92722c851482353b
 319         dq      0x92722c851482353b, 0x92722c851482353b, 0x92722c851482353b, 0x92722c851482353b
 320         dq      0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364
 321         dq      0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364
 322         dq      0xa81a664bbc423001, 0xa81a664bbc423001, 0xa81a664bbc423001, 0xa81a664bbc423001
 323         dq      0xa81a664bbc423001, 0xa81a664bbc423001, 0xa81a664bbc423001, 0xa81a664bbc423001
 324         dq      0xc24b8b70d0f89791, 0xc24b8b70d0f89791, 0xc24b8b70d0f89791, 0xc24b8b70d0f89791
 325         dq      0xc24b8b70d0f89791, 0xc24b8b70d0f89791, 0xc24b8b70d0f89791, 0xc24b8b70d0f89791
 326         dq      0xc76c51a30654be30, 0xc76c51a30654be30, 0xc76c51a30654be30, 0xc76c51a30654be30
 327         dq      0xc76c51a30654be30, 0xc76c51a30654be30, 0xc76c51a30654be30, 0xc76c51a30654be30
 328         dq      0xd192e819d6ef5218, 0xd192e819d6ef5218, 0xd192e819d6ef5218, 0xd192e819d6ef5218
 329         dq      0xd192e819d6ef5218, 0xd192e819d6ef5218, 0xd192e819d6ef5218, 0xd192e819d6ef5218
 330         dq      0xd69906245565a910, 0xd69906245565a910, 0xd69906245565a910, 0xd69906245565a910
 331         dq      0xd69906245565a910, 0xd69906245565a910, 0xd69906245565a910, 0xd69906245565a910
 332         dq      0xf40e35855771202a, 0xf40e35855771202a, 0xf40e35855771202a, 0xf40e35855771202a
 333         dq      0xf40e35855771202a, 0xf40e35855771202a, 0xf40e35855771202a, 0xf40e35855771202a
 334         dq      0x106aa07032bbd1b8, 0x106aa07032bbd1b8, 0x106aa07032bbd1b8, 0x106aa07032bbd1b8
 335         dq      0x106aa07032bbd1b8, 0x106aa07032bbd1b8, 0x106aa07032bbd1b8, 0x106aa07032bbd1b8
 336         dq      0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8
 337         dq      0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8
 338         dq      0x1e376c085141ab53, 0x1e376c085141ab53, 0x1e376c085141ab53, 0x1e376c085141ab53
 339         dq      0x1e376c085141ab53, 0x1e376c085141ab53, 0x1e376c085141ab53, 0x1e376c085141ab53
 340         dq      0x2748774cdf8eeb99, 0x2748774cdf8eeb99, 0x2748774cdf8eeb99, 0x2748774cdf8eeb99
 341         dq      0x2748774cdf8eeb99, 0x2748774cdf8eeb99, 0x2748774cdf8eeb99, 0x2748774cdf8eeb99
 342         dq      0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8
 343         dq      0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8
 344         dq      0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63
 345         dq      0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63
 346         dq      0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb
 347         dq      0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb
 348         dq      0x5b9cca4f7763e373, 0x5b9cca4f7763e373, 0x5b9cca4f7763e373, 0x5b9cca4f7763e373
 349         dq      0x5b9cca4f7763e373, 0x5b9cca4f7763e373, 0x5b9cca4f7763e373, 0x5b9cca4f7763e373
 350         dq      0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3
 351         dq      0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3
 352         dq      0x748f82ee5defb2fc, 0x748f82ee5defb2fc, 0x748f82ee5defb2fc, 0x748f82ee5defb2fc
 353         dq      0x748f82ee5defb2fc, 0x748f82ee5defb2fc, 0x748f82ee5defb2fc, 0x748f82ee5defb2fc
 354         dq      0x78a5636f43172f60, 0x78a5636f43172f60, 0x78a5636f43172f60, 0x78a5636f43172f60
 355         dq      0x78a5636f43172f60, 0x78a5636f43172f60, 0x78a5636f43172f60, 0x78a5636f43172f60
 356         dq      0x84c87814a1f0ab72, 0x84c87814a1f0ab72, 0x84c87814a1f0ab72, 0x84c87814a1f0ab72
 357         dq      0x84c87814a1f0ab72, 0x84c87814a1f0ab72, 0x84c87814a1f0ab72, 0x84c87814a1f0ab72
 358         dq      0x8cc702081a6439ec, 0x8cc702081a6439ec, 0x8cc702081a6439ec, 0x8cc702081a6439ec
 359         dq      0x8cc702081a6439ec, 0x8cc702081a6439ec, 0x8cc702081a6439ec, 0x8cc702081a6439ec
 360         dq      0x90befffa23631e28, 0x90befffa23631e28, 0x90befffa23631e28, 0x90befffa23631e28
 361         dq      0x90befffa23631e28, 0x90befffa23631e28, 0x90befffa23631e28, 0x90befffa23631e28
 362         dq      0xa4506cebde82bde9, 0xa4506cebde82bde9, 0xa4506cebde82bde9, 0xa4506cebde82bde9
 363         dq      0xa4506cebde82bde9, 0xa4506cebde82bde9, 0xa4506cebde82bde9, 0xa4506cebde82bde9
 364         dq      0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915
 365         dq      0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915
 366         dq      0xc67178f2e372532b, 0xc67178f2e372532b, 0xc67178f2e372532b, 0xc67178f2e372532b
 367         dq      0xc67178f2e372532b, 0xc67178f2e372532b, 0xc67178f2e372532b, 0xc67178f2e372532b
 368         dq      0xca273eceea26619c, 0xca273eceea26619c, 0xca273eceea26619c, 0xca273eceea26619c
 369         dq      0xca273eceea26619c, 0xca273eceea26619c, 0xca273eceea26619c, 0xca273eceea26619c
 370         dq      0xd186b8c721c0c207, 0xd186b8c721c0c207, 0xd186b8c721c0c207, 0xd186b8c721c0c207
 371         dq      0xd186b8c721c0c207, 0xd186b8c721c0c207, 0xd186b8c721c0c207, 0xd186b8c721c0c207
 372         dq      0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e
 373         dq      0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e
 374         dq      0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178
 375         dq      0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178
 376         dq      0x06f067aa72176fba, 0x06f067aa72176fba, 0x06f067aa72176fba, 0x06f067aa72176fba
 377         dq      0x06f067aa72176fba, 0x06f067aa72176fba, 0x06f067aa72176fba, 0x06f067aa72176fba
 378         dq      0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6
 379         dq      0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6
 380         dq      0x113f9804bef90dae, 0x113f9804bef90dae, 0x113f9804bef90dae, 0x113f9804bef90dae
 381         dq      0x113f9804bef90dae, 0x113f9804bef90dae, 0x113f9804bef90dae, 0x113f9804bef90dae
 382         dq      0x1b710b35131c471b, 0x1b710b35131c471b, 0x1b710b35131c471b, 0x1b710b35131c471b
 383         dq      0x1b710b35131c471b, 0x1b710b35131c471b, 0x1b710b35131c471b, 0x1b710b35131c471b
 384         dq      0x28db77f523047d84, 0x28db77f523047d84, 0x28db77f523047d84, 0x28db77f523047d84
 385         dq      0x28db77f523047d84, 0x28db77f523047d84, 0x28db77f523047d84, 0x28db77f523047d84
 386         dq      0x32caab7b40c72493, 0x32caab7b40c72493, 0x32caab7b40c72493, 0x32caab7b40c72493
 387         dq      0x32caab7b40c72493, 0x32caab7b40c72493, 0x32caab7b40c72493, 0x32caab7b40c72493
 388         dq      0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc
 389         dq      0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc
 390         dq      0x431d67c49c100d4c, 0x431d67c49c100d4c, 0x431d67c49c100d4c, 0x431d67c49c100d4c
 391         dq      0x431d67c49c100d4c, 0x431d67c49c100d4c, 0x431d67c49c100d4c, 0x431d67c49c100d4c
 392         dq      0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6
 393         dq      0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6
 394         dq      0x597f299cfc657e2a, 0x597f299cfc657e2a, 0x597f299cfc657e2a, 0x597f299cfc657e2a
 395         dq      0x597f299cfc657e2a, 0x597f299cfc657e2a, 0x597f299cfc657e2a, 0x597f299cfc657e2a
 396         dq      0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec
 397         dq      0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec
 398         dq      0x6c44198c4a475817, 0x6c44198c4a475817, 0x6c44198c4a475817, 0x6c44198c4a475817
 399         dq      0x6c44198c4a475817, 0x6c44198c4a475817, 0x6c44198c4a475817, 0x6c44198c4a475817
 400
 401 align 64
 402 ; this does the big endian to little endian conversion over a quad word .. ZMM
 403 ;; shuffle on ZMM is shuffle on 4 XMM size chunks, 128 bits
 404 PSHUFFLE_BYTE_FLIP_MASK:
 405         ;ddq 0x08090a0b0c0d0e0f0001020304050607
 406         dq      0x0001020304050607, 0x08090a0b0c0d0e0f
 407         ;ddq 0x18191a1b1c1d1e1f1011121314151617
 408         dq      0x1011121314151617, 0x18191a1b1c1d1e1f
 409         ;ddq 0x28292a2b2c2d2e2f2021222324252627
 410         dq      0x2021222324252627, 0x28292a2b2c2d2e2f
 411         ;ddq 0x38393a3b3c3d3e3f3031323334353637
 412         dq      0x3031323334353637, 0x38393a3b3c3d3e3f
 413
 414 section .text
 415
 416 ;; void sha512_x8_avx512(void *input_data, UINT64 *digest[NUM_LANES], const int size)
 417 ;; arg 1 : rcx : pointer to input data
 418 ;; arg 2 : rdx : pointer to UINT64 digest[8][num_lanes]
 419 ;; arg 3 : size in message block lengths (= 128 bytes)
 420 MKGLOBAL(sha512_x8_avx512,function,internal)
 421 align 64
 422 sha512_x8_avx512:
 423         mov     rax, rsp
 424         sub     rsp, STACK_SPACE
 425         and     rsp, ~63        ; align stack to multiple of 64
 426         mov     [rsp + _RSP], rax
 427
 428         ;; Initialize digests ; organized uint64 digest[8][num_lanes]; no transpose required
 429         ;; Digest is an array of pointers to digests
 430         vmovdqu32       A,    [STATE + 0*SHA512_DIGEST_ROW_SIZE]
 431         vmovdqu32       B,    [STATE + 1*SHA512_DIGEST_ROW_SIZE]
 432         vmovdqu32       C,    [STATE + 2*SHA512_DIGEST_ROW_SIZE]
 433         vmovdqu32       D,    [STATE + 3*SHA512_DIGEST_ROW_SIZE]
 434         vmovdqu32       E,    [STATE + 4*SHA512_DIGEST_ROW_SIZE]
 435         vmovdqu32       F,    [STATE + 5*SHA512_DIGEST_ROW_SIZE]
 436         vmovdqu32       G,    [STATE + 6*SHA512_DIGEST_ROW_SIZE]
 437         vmovdqu32       H,    [STATE + 7*SHA512_DIGEST_ROW_SIZE]
 438
 439         lea     TBL,[rel TABLE]
 440         xor     IDX, IDX
 441         ;; Read in input data address, saving them in registers because
 442         ;; they will serve as variables, which we shall keep incrementing
 443         mov     inp0, [STATE + _data_ptr_sha512 + 0*PTR_SZ]
 444         mov     inp1, [STATE + _data_ptr_sha512 + 1*PTR_SZ]
 445         mov     inp2, [STATE + _data_ptr_sha512 + 2*PTR_SZ]
 446         mov     inp3, [STATE + _data_ptr_sha512 + 3*PTR_SZ]
 447         mov     inp4, [STATE + _data_ptr_sha512 + 4*PTR_SZ]
 448         mov     inp5, [STATE + _data_ptr_sha512 + 5*PTR_SZ]
 449         mov     inp6, [STATE + _data_ptr_sha512 + 6*PTR_SZ]
 450         mov     inp7, [STATE + _data_ptr_sha512 + 7*PTR_SZ]
 451         jmp     lloop
 452
 453 align 32
 454 lloop:
 455         ;; Load 64-byte blocks of data into ZMM registers before
 456         ;; performing a 8x8 64-bit transpose.
 457         ;; To speed up the transpose, data is loaded in chunks of 32 bytes,
 458         ;; interleaving data between lane X and lane X+4.
 459         ;; This way, final shuffles between top half and bottom half
 460         ;; of the matrix are avoided.
 461         TRANSPOSE8_U64_LOAD8 W0, W1, W2, W3, W4, W5, W6, W7, \
 462                              inp0, inp1, inp2, inp3, inp4, inp5, \
 463                              inp6, inp7, IDX
 464
 465         TRANSPOSE8_U64 W0, W1, W2, W3, W4, W5, W6, W7, TMP0, TMP1, TMP2, TMP3
 466         ;;  Load next 512 bytes
 467         TRANSPOSE8_U64_LOAD8 W8, W9, W10, W11, W12, W13, W14, W15, \
 468                              inp0, inp1, inp2, inp3, inp4, inp5, \
 469                              inp6, inp7, IDX+SZ8
 470
 471         TRANSPOSE8_U64 W8, W9, W10, W11, W12, W13, W14, W15, TMP0, TMP1, TMP2, TMP3
 472
 473         vmovdqa32       TMP2, [rel PSHUFFLE_BYTE_FLIP_MASK]
 474
 475         vmovdqa32       TMP3, [TBL]     ; First K
 476
 477         ; Save digests for later addition
 478         vmovdqa32       [rsp + _DIGEST_SAVE + 64*0], A
 479         vmovdqa32       [rsp + _DIGEST_SAVE + 64*1], B
 480         vmovdqa32       [rsp + _DIGEST_SAVE + 64*2], C
 481         vmovdqa32       [rsp + _DIGEST_SAVE + 64*3], D
 482         vmovdqa32       [rsp + _DIGEST_SAVE + 64*4], E
 483         vmovdqa32       [rsp + _DIGEST_SAVE + 64*5], F
 484         vmovdqa32       [rsp + _DIGEST_SAVE + 64*6], G
 485         vmovdqa32       [rsp + _DIGEST_SAVE + 64*7], H
 486
 487         add     IDX, 128        ; increment by message block length in bytes
 488
 489 %assign I 0
 490 %rep 16
 491 ;;;  little endian to big endian
 492         vpshufb APPEND(W,I), APPEND(W,I), TMP2
 493 %assign I (I+1)
 494 %endrep
 495
 496         ; MSG Schedule for W0-W15 is now complete in registers
 497         ; Process first (max-rounds -16)
 498         ; Calculate next Wt+16 after processing is complete and Wt is unneeded
 499         ; PROCESS_LOOP_00_79 APPEND(W,J), I, APPEND(W,K), APPEND(W,L), APPEND(W,M)
 500
 501 %assign I 0
 502 %assign J 0
 503 %assign K 1
 504 %assign L 9
 505 %assign M 14
 506 %rep SHA_ROUNDS_LESS_16
 507         PROCESS_LOOP  APPEND(W,J),  I
 508         MSG_SCHED_ROUND_16_79  APPEND(W,J), APPEND(W,K), APPEND(W,L), APPEND(W,M)
 509 %assign I (I+1)
 510 %assign J ((J+1)% 16)
 511 %assign K ((K+1)% 16)
 512 %assign L ((L+1)% 16)
 513 %assign M ((M+1)% 16)
 514 %endrep
 515         ; Check is this is the last block
 516         sub     INP_SIZE, 1
 517         je      lastLoop
 518
 519         ; Process last 16 rounds
 520         ; Read in next block msg data for use in first 16 words of msg sched
 521 %assign I SHA_ROUNDS_LESS_16
 522 %assign J 0
 523 %rep 16
 524         PROCESS_LOOP  APPEND(W,J), I
 525 %assign I (I+1)
 526 %assign J (J+1)
 527 %endrep
 528        ; Add old digest
 529         vpaddq          A, A, [rsp + _DIGEST_SAVE + 64*0]
 530         vpaddq          B, B, [rsp + _DIGEST_SAVE + 64*1]
 531         vpaddq          C, C, [rsp + _DIGEST_SAVE + 64*2]
 532         vpaddq          D, D, [rsp + _DIGEST_SAVE + 64*3]
 533         vpaddq          E, E, [rsp + _DIGEST_SAVE + 64*4]
 534         vpaddq          F, F, [rsp + _DIGEST_SAVE + 64*5]
 535         vpaddq          G, G, [rsp + _DIGEST_SAVE + 64*6]
 536         vpaddq          H, H, [rsp + _DIGEST_SAVE + 64*7]
 537
 538         jmp     lloop
 539
 540 align 32
 541 lastLoop:
 542         ; Process last 16 rounds
 543 %assign I SHA_ROUNDS_LESS_16
 544 %assign J 0
 545 %rep 16
 546         PROCESS_LOOP  APPEND(W,J), I
 547 %assign I (I+1)
 548 %assign J (J+1)
 549 %endrep
 550
 551         ; Add old digest
 552         vpaddq          A, A, [rsp + _DIGEST_SAVE + 64*0]
 553         vpaddq          B, B, [rsp + _DIGEST_SAVE + 64*1]
 554         vpaddq          C, C, [rsp + _DIGEST_SAVE + 64*2]
 555         vpaddq          D, D, [rsp + _DIGEST_SAVE + 64*3]
 556         vpaddq          E, E, [rsp + _DIGEST_SAVE + 64*4]
 557         vpaddq          F, F, [rsp + _DIGEST_SAVE + 64*5]
 558         vpaddq          G, G, [rsp + _DIGEST_SAVE + 64*6]
 559         vpaddq          H, H, [rsp + _DIGEST_SAVE + 64*7]
 560
 561         ; Write out digest
 562         ;;  results in A, B, C, D, E, F, G, H
 563         vmovdqu32       [STATE + 0*SHA512_DIGEST_ROW_SIZE], A
 564         vmovdqu32       [STATE + 1*SHA512_DIGEST_ROW_SIZE], B
 565         vmovdqu32       [STATE + 2*SHA512_DIGEST_ROW_SIZE], C
 566         vmovdqu32       [STATE + 3*SHA512_DIGEST_ROW_SIZE], D
 567         vmovdqu32       [STATE + 4*SHA512_DIGEST_ROW_SIZE], E
 568         vmovdqu32       [STATE + 5*SHA512_DIGEST_ROW_SIZE], F
 569         vmovdqu32       [STATE + 6*SHA512_DIGEST_ROW_SIZE], G
 570         vmovdqu32       [STATE + 7*SHA512_DIGEST_ROW_SIZE], H
 571
 572         ; update input pointers
 573 %assign I 0
 574 %rep 8
 575         add     [STATE + _data_ptr_sha512 + I*PTR_SZ], IDX
 576 %assign I (I+1)
 577 %endrep
 578
 579
 580 %ifdef SAFE_DATA
 581         ;; Clear stack frame ((NUM_LANES*8)*64 bytes)
 582         vpxorq  zmm0, zmm0
 583 %assign i 0
 584 %rep (NUM_LANES*8)
 585         vmovdqa64 [rsp + i*64], zmm0
 586 %assign i (i+1)
 587 %endrep
 588 %endif
 589         mov     rsp, [rsp + _RSP]
 590 ;hash_done:
 591         ret
 592
 593 %ifdef LINUX
 594 section .note.GNU-stack noalloc noexec nowrite progbits
 595 %endif