ceph/src/crypto/isa-l/isa-l_crypto/md5_mb/md5_mb_x16x2_avx512.asm

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;  Copyright(c) 2011-2016 Intel Corporation All rights reserved.
   3 ;
   4 ;  Redistribution and use in source and binary forms, with or without
   5 ;  modification, are permitted provided that the following conditions
   6 ;  are met:
   7 ;    * Redistributions of source code must retain the above copyright
   8 ;      notice, this list of conditions and the following disclaimer.
   9 ;    * Redistributions in binary form must reproduce the above copyright
  10 ;      notice, this list of conditions and the following disclaimer in
  11 ;      the documentation and/or other materials provided with the
  12 ;      distribution.
  13 ;    * Neither the name of Intel Corporation nor the names of its
  14 ;      contributors may be used to endorse or promote products derived
  15 ;      from this software without specific prior written permission.
  16 ;
  17 ;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 ;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 ;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 ;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 ;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 ;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 ;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 ;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 ;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 ;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 ;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29
  30 %include "md5_mb_mgr_datastruct.asm"
  31 %include "reg_sizes.asm"
  32
  33 %ifdef HAVE_AS_KNOWS_AVX512
  34 default rel
  35
  36 ;; code to compute double octal MD5 using AVX512
  37
  38 ;; Stack must be aligned to 64 bytes before call
  39
  40 ;; Windows clobbers:  rax rbx     rdx rsi rdi     r8 r9 r10 r11 r12 r13 r14 r15
  41 ;; Windows preserves:         rcx             rbp
  42 ;;
  43 ;; Linux clobbers:    rax rbx rcx rdx rsi         r8 r9 r10 r11 r12 r13 r14 r15
  44 ;; Linux preserves:                       rdi rbp
  45 ;;
  46 ;; clobbers zmm0-8, 14-31
  47
  48 ;; clobbers all GPRs other than arg1 and rbp
  49
  50 %ifidn __OUTPUT_FORMAT__, win64
  51    %define arg1 rcx     ; arg0
  52    %define arg2 rdx     ; arg1
  53    %define reg3 r8      ; arg2
  54    %define reg4 r9      ; arg3
  55    %define var1 rdi
  56    %define var2 rsi
  57    %define local_func_decl(func_name) global func_name
  58  %else
  59    %define arg1 rdi     ; arg0
  60    %define arg2 rsi     ; arg1
  61    %define var1 rdx     ; arg2
  62    %define var2 rcx     ; arg3
  63    %define local_func_decl(func_name) global func_name:function internal
  64 %endif
  65
  66 %define state    arg1
  67 %define num_blks arg2
  68
  69 %define IN      (state + _data_ptr)
  70 %define DIGEST  state
  71 %define SIZE    num_blks
  72 ;; These are pointers to data block1 and block2 in the stack
  73 ; which will ping pong back and forth
  74 %define DPTR1   rbx
  75 %define DPTR2   var2
  76 %define IDX     var1
  77 %define TBL     rax
  78
  79 %define inp0 r8
  80 %define inp1 r9
  81 %define inp2 r10
  82 %define inp3 r11
  83 %define inp4 r12
  84 %define inp5 r13
  85 %define inp6 r14
  86 %define inp7 r15
  87
  88 ;; Transposed Digest Storage
  89 %define A       zmm0
  90 %define B       zmm1
  91 %define C       zmm2
  92 %define D       zmm3
  93 %define A1      zmm4
  94 %define B1      zmm5
  95 %define C1      zmm6
  96 %define D1      zmm7
  97
  98 %define md5c    zmm16
  99
 100 %define MASK0   zmm17
 101 %define MASK1   zmm18
 102
 103 %define TMP0    zmm20
 104 %define TMP1    zmm21
 105
 106
 107 ;; Data are stored into the Wx after transposition
 108 %define W0      zmm8
 109 %define W1      zmm9
 110 %define W2      zmm10
 111 %define W3      zmm11
 112 %define W4      zmm12
 113 %define W5      zmm13
 114 %define W6      zmm14
 115 %define W7      zmm15
 116
 117 %define W8      zmm24
 118 %define W9      zmm25
 119 %define W10     zmm26
 120 %define W11     zmm27
 121 %define W12     zmm28
 122 %define W13     zmm29
 123 %define W14     zmm30
 124 %define W15     zmm31
 125
 126 %define MD5_DIGEST_ROW_SIZE (16*4)
 127 %define APPEND(a,b) a %+ b
 128 %define APPEND3(a,b,c) a %+ b %+ c
 129
 130 ;; Temporary registers used during data transposition
 131
 132 %define RESZ    resb 64*
 133 ;; Assume stack aligned to 64 bytes before call
 134 ;; Therefore FRAMESIZE mod 64 must be 64-8 = 56
 135 struc STACK
 136 _DATA:          RESZ    2*2*16  ; 2 blocks * 2 sets of lanes * 16 regs
 137 _DIGEST:        RESZ    8       ; stores Z_AA-Z_DD, Z_AA2-Z_DD2
 138 _TMPDIGEST:     RESZ    2       ; stores Z_AA, Z_BB temporarily
 139 _RSP_SAVE:      RESQ    1       ; original RSP
 140 endstruc
 141
 142 %define Z_AA    rsp + _DIGEST + 64*0
 143 %define Z_BB    rsp + _DIGEST + 64*1
 144 %define Z_CC    rsp + _DIGEST + 64*2
 145 %define Z_DD    rsp + _DIGEST + 64*3
 146 %define Z_AA1   rsp + _DIGEST + 64*4
 147 %define Z_BB1   rsp + _DIGEST + 64*5
 148 %define Z_CC1   rsp + _DIGEST + 64*6
 149 %define Z_DD1   rsp + _DIGEST + 64*7
 150
 151 %define MD5_DIGEST_ROW_SIZE (32*4)
 152
 153
 154 ;;
 155 ;; MD5 left rotations (number of bits)
 156 ;;
 157 %define rot11 7
 158 %define rot12 12
 159 %define rot13 17
 160 %define rot14 22
 161 %define rot21 5
 162 %define rot22 9
 163 %define rot23 14
 164 %define rot24 20
 165 %define rot31 4
 166 %define rot32 11
 167 %define rot33 16
 168 %define rot34 23
 169 %define rot41 6
 170 %define rot42 10
 171 %define rot43 15
 172 %define rot44 21
 173
 174 %macro TRANSPOSE16 18
 175 %define %%r0 %1
 176 %define %%r1 %2
 177 %define %%r2 %3
 178 %define %%r3 %4
 179 %define %%r4 %5
 180 %define %%r5 %6
 181 %define %%r6 %7
 182 %define %%r7 %8
 183 %define %%r8 %9
 184 %define %%r9 %10
 185 %define %%r10 %11
 186 %define %%r11 %12
 187 %define %%r12 %13
 188 %define %%r13 %14
 189 %define %%r14 %15
 190 %define %%r15 %16
 191 %define %%t0 %17
 192 %define %%t1 %18
 193
 194 ; r0  = {a15 a14 a13 a12   a11 a10 a9 a8   a7 a6 a5 a4   a3 a2 a1 a0}
 195 ; r1  = {b15 b14 b13 b12   b11 b10 b9 b8   b7 b6 b5 b4   b3 b2 b1 b0}
 196 ; r2  = {c15 c14 c13 c12   c11 c10 c9 c8   c7 c6 c5 c4   c3 c2 c1 c0}
 197 ; r3  = {d15 d14 d13 d12   d11 d10 d9 d8   d7 d6 d5 d4   d3 d2 d1 d0}
 198 ; r4  = {e15 e14 e13 e12   e11 e10 e9 e8   e7 e6 e5 e4   e3 e2 e1 e0}
 199 ; r5  = {f15 f14 f13 f12   f11 f10 f9 f8   f7 f6 f5 f4   f3 f2 f1 f0}
 200 ; r6  = {g15 g14 g13 g12   g11 g10 g9 g8   g7 g6 g5 g4   g3 g2 g1 g0}
 201 ; r7  = {h15 h14 h13 h12   h11 h10 h9 h8   h7 h6 h5 h4   h3 h2 h1 h0}
 202 ; r8  = {i15 i14 i13 i12   i11 i10 i9 i8   i7 i6 i5 i4   i3 i2 i1 i0}
 203 ; r9  = {j15 j14 j13 j12   j11 j10 j9 j8   j7 j6 j5 j4   j3 j2 j1 j0}
 204 ; r10 = {k15 k14 k13 k12   k11 k10 k9 k8   k7 k6 k5 k4   k3 k2 k1 k0}
 205 ; r11 = {l15 l14 l13 l12   l11 l10 l9 l8   l7 l6 l5 l4   l3 l2 l1 l0}
 206 ; r12 = {m15 m14 m13 m12   m11 m10 m9 m8   m7 m6 m5 m4   m3 m2 m1 m0}
 207 ; r13 = {n15 n14 n13 n12   n11 n10 n9 n8   n7 n6 n5 n4   n3 n2 n1 n0}
 208 ; r14 = {o15 o14 o13 o12   o11 o10 o9 o8   o7 o6 o5 o4   o3 o2 o1 o0}
 209 ; r15 = {p15 p14 p13 p12   p11 p10 p9 p8   p7 p6 p5 p4   p3 p2 p1 p0}
 210
 211 ; r0   = {p0  o0  n0  m0    l0  k0  j0  i0    h0  g0  f0  e0    d0  c0  b0  a0}
 212 ; r1   = {p1  o1  n1  m1    l1  k1  j1  i1    h1  g1  f1  e1    d1  c1  b1  a1}
 213 ; r2   = {p2  o2  n2  m2    l2  k2  j2  i2    h2  g2  f2  e2    d2  c2  b2  a2}
 214 ; r3   = {p3  o3  n3  m3    l3  k3  j3  i3    h3  g3  f3  e3    d3  c3  b3  a3}
 215 ; r4   = {p4  o4  n4  m4    l4  k4  j4  i4    h4  g4  f4  e4    d4  c4  b4  a4}
 216 ; r5   = {p5  o5  n5  m5    l5  k5  j5  i5    h5  g5  f5  e5    d5  c5  b5  a5}
 217 ; r6   = {p6  o6  n6  m6    l6  k6  j6  i6    h6  g6  f6  e6    d6  c6  b6  a6}
 218 ; r7   = {p7  o7  n7  m7    l7  k7  j7  i7    h7  g7  f7  e7    d7  c7  b7  a7}
 219 ; r8   = {p8  o8  n8  m8    l8  k8  j8  i8    h8  g8  f8  e8    d8  c8  b8  a8}
 220 ; r9   = {p9  o9  n9  m9    l9  k9  j9  i9    h9  g9  f9  e9    d9  c9  b9  a9}
 221 ; r10  = {p10 o10 n10 m10   l10 k10 j10 i10   h10 g10 f10 e10   d10 c10 b10 a10}
 222 ; r11  = {p11 o11 n11 m11   l11 k11 j11 i11   h11 g11 f11 e11   d11 c11 b11 a11}
 223 ; r12  = {p12 o12 n12 m12   l12 k12 j12 i12   h12 g12 f12 e12   d12 c12 b12 a12}
 224 ; r13  = {p13 o13 n13 m13   l13 k13 j13 i13   h13 g13 f13 e13   d13 c13 b13 a13}
 225 ; r14  = {p14 o14 n14 m14   l14 k14 j14 i14   h14 g14 f14 e14   d14 c14 b14 a14}
 226 ; r15  = {p15 o15 n15 m15   l15 k15 j15 i15   h15 g15 f15 e15   d15 c15 b15 a15}
 227
 228
 229         ; process top half (r0..r3) {a...d}
 230         vshufps %%t0, %%r0, %%r1, 0x44  ; t0 = {b13 b12 a13 a12   b9  b8  a9  a8   b5 b4 a5 a4   b1 b0 a1 a0}
 231         vshufps %%r0, %%r0, %%r1, 0xEE  ; r0 = {b15 b14 a15 a14   b11 b10 a11 a10  b7 b6 a7 a6   b3 b2 a3 a2}
 232         vshufps %%t1, %%r2, %%r3, 0x44  ; t1 = {d13 d12 c13 c12   d9  d8  c9  c8   d5 d4 c5 c4   d1 d0 c1 c0}
 233         vshufps %%r2, %%r2, %%r3, 0xEE  ; r2 = {d15 d14 c15 c14   d11 d10 c11 c10  d7 d6 c7 c6   d3 d2 c3 c2}
 234
 235         vshufps %%r3, %%t0, %%t1, 0xDD  ; r3 = {d13 c13 b13 a13   d9  c9  b9  a9   d5 c5 b5 a5   d1 c1 b1 a1}
 236         vshufps %%r1, %%r0, %%r2, 0x88  ; r1 = {d14 c14 b14 a14   d10 c10 b10 a10  d6 c6 b6 a6   d2 c2 b2 a2}
 237         vshufps %%r0, %%r0, %%r2, 0xDD  ; r0 = {d15 c15 b15 a15   d11 c11 b11 a11  d7 c7 b7 a7   d3 c3 b3 a3}
 238         vshufps %%t0, %%t0, %%t1, 0x88  ; t0 = {d12 c12 b12 a12   d8  c8  b8  a8   d4 c4 b4 a4   d0 c0 b0 a0}
 239
 240         ; use r2 in place of t0
 241         vshufps %%r2, %%r4, %%r5, 0x44  ; r2 = {f13 f12 e13 e12   f9  f8  e9  e8   f5 f4 e5 e4   f1 f0 e1 e0}
 242         vshufps %%r4, %%r4, %%r5, 0xEE  ; r4 = {f15 f14 e15 e14   f11 f10 e11 e10  f7 f6 e7 e6   f3 f2 e3 e2}
 243         vshufps %%t1, %%r6, %%r7, 0x44  ; t1 = {h13 h12 g13 g12   h9  h8  g9  g8   h5 h4 g5 g4   h1 h0 g1 g0}
 244         vshufps %%r6, %%r6, %%r7, 0xEE  ; r6 = {h15 h14 g15 g14   h11 h10 g11 g10  h7 h6 g7 g6   h3 h2 g3 g2}
 245
 246         vshufps %%r7, %%r2, %%t1, 0xDD  ; r7 = {h13 g13 f13 e13   h9  g9  f9  e9   h5 g5 f5 e5   h1 g1 f1 e1}
 247         vshufps %%r5, %%r4, %%r6, 0x88  ; r5 = {h14 g14 f14 e14   h10 g10 f10 e10  h6 g6 f6 e6   h2 g2 f2 e2}
 248         vshufps %%r4, %%r4, %%r6, 0xDD  ; r4 = {h15 g15 f15 e15   h11 g11 f11 e11  h7 g7 f7 e7   h3 g3 f3 e3}
 249         vshufps %%r2, %%r2, %%t1, 0x88  ; r2 = {h12 g12 f12 e12   h8  g8  f8  e8   h4 g4 f4 e4   h0 g0 f0 e0}
 250
 251         ; use r6 in place of t0
 252         vshufps %%r6, %%r8, %%r9,    0x44       ; r6  = {j13 j12 i13 i12   j9  j8  i9  i8   j5 j4 i5 i4   j1 j0 i1 i0}
 253         vshufps %%r8, %%r8, %%r9,    0xEE       ; r8  = {j15 j14 i15 i14   j11 j10 i11 i10  j7 j6 i7 i6   j3 j2 i3 i2}
 254         vshufps %%t1, %%r10, %%r11,  0x44       ; t1  = {l13 l12 k13 k12   l9  l8  k9  k8   l5 l4 k5 k4   l1 l0 k1 k0}
 255         vshufps %%r10, %%r10, %%r11, 0xEE       ; r10 = {l15 l14 k15 k14   l11 l10 k11 k10  l7 l6 k7 k6   l3 l2 k3 k2}
 256
 257         vshufps %%r11, %%r6, %%t1, 0xDD         ; r11 = {l13 k13 j13 113   l9  k9  j9  i9   l5 k5 j5 i5   l1 k1 j1 i1}
 258         vshufps %%r9, %%r8, %%r10, 0x88         ; r9  = {l14 k14 j14 114   l10 k10 j10 i10  l6 k6 j6 i6   l2 k2 j2 i2}
 259         vshufps %%r8, %%r8, %%r10, 0xDD         ; r8  = {l15 k15 j15 115   l11 k11 j11 i11  l7 k7 j7 i7   l3 k3 j3 i3}
 260         vshufps %%r6, %%r6, %%t1,  0x88         ; r6  = {l12 k12 j12 112   l8  k8  j8  i8   l4 k4 j4 i4   l0 k0 j0 i0}
 261
 262         ; use r10 in place of t0
 263         vshufps %%r10, %%r12, %%r13, 0x44       ; r10 = {n13 n12 m13 m12   n9  n8  m9  m8   n5 n4 m5 m4   n1 n0 a1 m0}
 264         vshufps %%r12, %%r12, %%r13, 0xEE       ; r12 = {n15 n14 m15 m14   n11 n10 m11 m10  n7 n6 m7 m6   n3 n2 a3 m2}
 265         vshufps %%t1, %%r14, %%r15,  0x44       ; t1  = {p13 p12 013 012   p9  p8  09  08   p5 p4 05 04   p1 p0 01 00}
 266         vshufps %%r14, %%r14, %%r15, 0xEE       ; r14 = {p15 p14 015 014   p11 p10 011 010  p7 p6 07 06   p3 p2 03 02}
 267
 268         vshufps %%r15, %%r10, %%t1,  0xDD       ; r15 = {p13 013 n13 m13   p9  09  n9  m9   p5 05 n5 m5   p1 01 n1 m1}
 269         vshufps %%r13, %%r12, %%r14, 0x88       ; r13 = {p14 014 n14 m14   p10 010 n10 m10  p6 06 n6 m6   p2 02 n2 m2}
 270         vshufps %%r12, %%r12, %%r14, 0xDD       ; r12 = {p15 015 n15 m15   p11 011 n11 m11  p7 07 n7 m7   p3 03 n3 m3}
 271         vshufps %%r10, %%r10, %%t1,  0x88       ; r10 = {p12 012 n12 m12   p8  08  n8  m8   p4 04 n4 m4   p0 00 n0 m0}
 272
 273 ;; At this point, the registers that contain interesting data are:
 274 ;; t0, r3, r1, r0, r2, r7, r5, r4, r6, r11, r9, r8, r10, r15, r13, r12
 275 ;; Can use t1 and r14 as scratch registers
 276
 277         vmovdqa32 %%r14, MASK0
 278         vpermi2q  %%r14, %%t0, %%r2             ; r14 = {h8  g8  f8  e8   d8  c8  b8  a8   h0 g0 f0 e0   d0 c0 b0 a0}
 279         vmovdqa32 %%t1,  MASK1
 280         vpermi2q  %%t1,  %%t0, %%r2             ; t1  = {h12 g12 f12 e12  d12 c12 b12 a12  h4 g4 f4 e4   d4 c4 b4 a4}
 281
 282         vmovdqa32 %%r2, MASK0
 283         vpermi2q  %%r2, %%r3, %%r7              ; r2  = {h9  g9  f9  e9   d9  c9  b9  a9   h1 g1 f1 e1   d1 c1 b1 a1}
 284         vmovdqa32 %%t0, MASK1
 285         vpermi2q  %%t0, %%r3, %%r7              ; t0  = {h13 g13 f13 e13  d13 c13 b13 a13  h5 g5 f5 e5   d5 c5 b5 a5}
 286
 287         vmovdqa32 %%r3, MASK0
 288         vpermi2q  %%r3, %%r1, %%r5              ; r3  = {h10 g10 f10 e10  d10 c10 b10 a10  h2 g2 f2 e2   d2 c2 b2 a2}
 289         vmovdqa32 %%r7, MASK1
 290         vpermi2q  %%r7, %%r1, %%r5              ; r7  = {h14 g14 f14 e14  d14 c14 b14 a14  h6 g6 f6 e6   d6 c6 b6 a6}
 291
 292         vmovdqa32 %%r1, MASK0
 293         vpermi2q  %%r1, %%r0, %%r4              ; r1  = {h11 g11 f11 e11  d11 c11 b11 a11  h3 g3 f3 e3   d3 c3 b3 a3}
 294         vmovdqa32 %%r5, MASK1
 295         vpermi2q  %%r5, %%r0, %%r4              ; r5  = {h15 g15 f15 e15  d15 c15 b15 a15  h7 g7 f7 e7   d7 c7 b7 a7}
 296
 297         vmovdqa32 %%r0, MASK0
 298         vpermi2q  %%r0, %%r6, %%r10             ; r0 = {p8  o8  n8  m8   l8  k8  j8  i8   p0 o0 n0 m0    l0 k0 j0 i0}
 299         vmovdqa32 %%r4,  MASK1
 300         vpermi2q  %%r4, %%r6, %%r10             ; r4  = {p12 o12 n12 m12  l12 k12 j12 i12  p4 o4 n4 m4   l4 k4 j4 i4}
 301
 302         vmovdqa32 %%r6, MASK0
 303         vpermi2q  %%r6, %%r11, %%r15            ; r6  = {p9  o9  n9  m9   l9  k9  j9  i9   p1 o1 n1 m1   l1 k1 j1 i1}
 304         vmovdqa32 %%r10, MASK1
 305         vpermi2q  %%r10, %%r11, %%r15           ; r10 = {p13 o13 n13 m13  l13 k13 j13 i13  p5 o5 n5 m5   l5 k5 j5 i5}
 306
 307         vmovdqa32 %%r11, MASK0
 308         vpermi2q  %%r11, %%r9, %%r13            ; r11 = {p10 o10 n10 m10  l10 k10 j10 i10  p2 o2 n2 m2   l2 k2 j2 i2}
 309         vmovdqa32 %%r15, MASK1
 310         vpermi2q  %%r15, %%r9, %%r13            ; r15 = {p14 o14 n14 m14  l14 k14 j14 i14  p6 o6 n6 m6   l6 k6 j6 i6}
 311
 312         vmovdqa32 %%r9, MASK0
 313         vpermi2q  %%r9, %%r8, %%r12             ; r9  = {p11 o11 n11 m11  l11 k11 j11 i11  p3 o3 n3 m3   l3 k3 j3 i3}
 314         vmovdqa32 %%r13, MASK1
 315         vpermi2q  %%r13, %%r8, %%r12            ; r13 = {p15 o15 n15 m15  l15 k15 j15 i15  p7 o7 n7 m7   l7 k7 j7 i7}
 316
 317 ;; At this point r8 and r12 can be used as scratch registers
 318
 319         vshuff64x2 %%r8, %%r14, %%r0, 0xEE      ; r8  = {p8  o8  n8  m8   l8  k8  j8  i8   h8 g8 f8 e8   d8 c8 b8 a8}
 320         vshuff64x2 %%r0, %%r14, %%r0, 0x44      ; r0  = {p0  o0  n0  m0   l0  k0  j0  i0   h0 g0 f0 e0   d0 c0 b0 a0}
 321
 322         vshuff64x2 %%r12, %%t1, %%r4, 0xEE      ; r12 = {p12 o12 n12 m12  l12 k12 j12 i12  h12 g12 f12 e12  d12 c12 b12 a12}
 323         vshuff64x2 %%r4, %%t1, %%r4, 0x44       ; r4  = {p4  o4  n4  m4   l4  k4  j4  i4   h4 g4 f4 e4   d4 c4 b4 a4}
 324
 325         vshuff64x2 %%r14, %%r7, %%r15, 0xEE     ; r14 = {p14 o14 n14 m14  l14 k14 j14 i14  h14 g14 f14 e14  d14 c14 b14 a14}
 326         vshuff64x2 %%t1, %%r7, %%r15, 0x44      ; t1  = {p6  o6  n6  m6   l6  k6  j6  i6   h6 g6 f6 e6   d6 c6 b6 a6}
 327
 328         vshuff64x2 %%r15, %%r5, %%r13, 0xEE     ; r15 = {p15 o15 n15 m15  l15 k15 j15 i15  h15 g15 f15 e15  d15 c15 b15 a15}
 329         vshuff64x2 %%r7, %%r5, %%r13, 0x44      ; r7  = {p7  o7  n7  m7   l7  k7  j7  i7   h7 g7 f7 e7   d7 c7 b7 a7}
 330
 331         vshuff64x2 %%r13, %%t0, %%r10, 0xEE     ; r13 = {p13 o13 n13 m13  l13 k13 j13 i13  h13 g13 f13 e13  d13 c13 b13 a13}
 332         vshuff64x2 %%r5, %%t0, %%r10, 0x44      ; r5  = {p5  o5  n5  m5   l5  k5  j5  i5   h5 g5 f5 e5   d5 c5 b5 a5}
 333
 334         vshuff64x2 %%r10, %%r3, %%r11, 0xEE     ; r10 = {p10 o10 n10 m10  l10 k10 j10 i10  h10 g10 f10 e10  d10 c10 b10 a10}
 335         vshuff64x2 %%t0, %%r3, %%r11, 0x44      ; t0  = {p2  o2  n2  m2   l2  k2  j2  i2   h2 g2 f2 e2   d2 c2 b2 a2}
 336
 337         vshuff64x2 %%r11, %%r1, %%r9, 0xEE      ; r11 = {p11 o11 n11 m11  l11 k11 j11 i11  h11 g11 f11 e11  d11 c11 b11 a11}
 338         vshuff64x2 %%r3, %%r1, %%r9, 0x44       ; r3  = {p3  o3  n3  m3   l3  k3  j3  i3   h3 g3 f3 e3   d3 c3 b3 a3}
 339
 340         vshuff64x2 %%r9, %%r2, %%r6, 0xEE       ; r9  = {p9  o9  n9  m9   l9  k9  j9  i9   h9 g9 f9 e9   d9 c9 b9 a9}
 341         vshuff64x2 %%r1, %%r2, %%r6, 0x44       ; r1  = {p1  o1  n1  m1   l1  k1  j1  i1   h1 g1 f1 e1   d1 c1 b1 a1}
 342
 343         vmovdqa32 %%r2, %%t0                    ; r2  = {p2  o2  n2  m2   l2  k2  j2  i2   h2 g2 f2 e2   d2 c2 b2 a2}
 344         vmovdqa32 %%r6, %%t1                    ; r6  = {p6  o6  n6  m6   l6  k6  j6  i6   h6 g6 f6 e6   d6 c6 b6 a6}
 345
 346 %endmacro
 347
 348 %macro ROTATE_ARGS 0
 349 %xdefine TMP_ D
 350 %xdefine D C
 351 %xdefine C B
 352 %xdefine B A
 353 %xdefine A TMP_
 354 %endm
 355
 356 %macro ROTATE_ARGS1 0
 357 %xdefine TMP_ D1
 358 %xdefine D1 C1
 359 %xdefine C1 B1
 360 %xdefine B1 A1
 361 %xdefine A1 TMP_
 362 %endm
 363
 364 ;;
 365 ;; single MD5 step
 366 ;;
 367 ;; A = B +ROL32((A +Ft(B,C,D) +data +const), nrot)
 368 ;;eg: PROCESS_LOOP MD5constx, Mdatax, F_IMMEDx, NROTx
 369 %macro PROCESS_LOOP 6
 370 %define %%MD5const      %1
 371 %define %%data          %2
 372 %define %%F_IMMED       %3
 373 %define %%NROT          %4
 374 %define %%TMP_PR0       %5
 375 %define %%TMP_PR1       %6
 376         ; a=b+((a+Ft(b,c,d)+Mj+ti)<<s)
 377
 378         ; Ft
 379         ;  0-15 Ft:F(X,Y,Z)=(X&Y)|((~X)&Z)      0xca
 380         ; 16-31 Ft:G(X,Y,Z)=(X&Z)|(Y&(~Z))      0xe4
 381         ; 32-47 Ft:H(X,Y,Z)=X^Y^Z               0x96
 382         ; 48-63 Ft:I(X,Y,Z)=Y^(X|(~Z))          0x39
 383
 384         vpaddd          A, A, %%MD5const
 385                 vpaddd          A1, A1, %%MD5const
 386         vpaddd          A, A, [%%data]
 387                 vpaddd          A1, A1, [%%data + 16*64]
 388         vmovdqa32       %%TMP_PR0, B            ; Copy B
 389                 vmovdqa32       %%TMP_PR1, B1           ; Copy B
 390         vpternlogd      %%TMP_PR0, C, D, %%F_IMMED
 391                 vpternlogd      %%TMP_PR1, C1, D1, %%F_IMMED
 392         vpaddd          A, A, %%TMP_PR0
 393                 vpaddd          A1, A1, %%TMP_PR1
 394         vprold          A, A, %%NROT
 395                 vprold          A1, A1, %%NROT
 396         vpaddd          A, A, B
 397                 vpaddd          A1, A1, B1
 398
 399         ROTATE_ARGS
 400         ROTATE_ARGS1
 401 %endmacro
 402
 403 align 64
 404 default rel
 405 section .text
 406
 407 ; void md5_mb_x16x2_avx512(MD5_ARGS *args, UINT64 num_blks)
 408 ; arg 1 : pointer to MD5_ARGS structure
 409 ; arg 2 : number of blocks (>=1)
 410
 411 local_func_decl(md5_mb_x16x2_avx512)
 412 md5_mb_x16x2_avx512:
 413         mov     rax, rsp
 414         sub     rsp, STACK_size
 415         and     rsp, -64
 416         mov     [rsp + _RSP_SAVE], rax
 417
 418         mov     DPTR1, rsp
 419         lea     DPTR2, [rsp + 64*32]
 420
 421         ;; Load MD5 constant pointer to register
 422         lea     TBL, [MD5_TABLE]
 423         vmovdqa32 MASK0, [PSHUFFLE_TRANSPOSE16_MASK1]
 424         vmovdqa32 MASK1, [PSHUFFLE_TRANSPOSE16_MASK2]
 425
 426         ;; Preload input data from 16 segments.
 427         xor IDX, IDX
 428
 429         ;; transpose input onto stack
 430         ;; first 16 lanes read
 431         mov     inp0, [IN + 0*8]
 432         mov     inp1, [IN + 1*8]
 433         mov     inp2, [IN + 2*8]
 434         mov     inp3, [IN + 3*8]
 435         mov     inp4, [IN + 4*8]
 436         mov     inp5, [IN + 5*8]
 437         mov     inp6, [IN + 6*8]
 438         mov     inp7, [IN + 7*8]
 439         vmovdqu32       W0,[inp0+IDX]
 440         vmovdqu32       W1,[inp1+IDX]
 441         vmovdqu32       W2,[inp2+IDX]
 442         vmovdqu32       W3,[inp3+IDX]
 443         vmovdqu32       W4,[inp4+IDX]
 444         vmovdqu32       W5,[inp5+IDX]
 445         vmovdqu32       W6,[inp6+IDX]
 446         vmovdqu32       W7,[inp7+IDX]
 447         mov     inp0, [IN + 8*8]
 448         mov     inp1, [IN + 9*8]
 449         mov     inp2, [IN +10*8]
 450         mov     inp3, [IN +11*8]
 451         mov     inp4, [IN +12*8]
 452         mov     inp5, [IN +13*8]
 453         mov     inp6, [IN +14*8]
 454         mov     inp7, [IN +15*8]
 455         vmovdqu32       W8, [inp0+IDX]
 456         vmovdqu32       W9, [inp1+IDX]
 457         vmovdqu32       W10,[inp2+IDX]
 458         vmovdqu32       W11,[inp3+IDX]
 459         vmovdqu32       W12,[inp4+IDX]
 460         vmovdqu32       W13,[inp5+IDX]
 461         vmovdqu32       W14,[inp6+IDX]
 462         vmovdqu32       W15,[inp7+IDX]
 463         ;; first 16 lanes trans&write
 464         TRANSPOSE16 W0, W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, TMP0, TMP1
 465         vmovdqa32       [DPTR1+_DATA+(0)*64],W0
 466         vmovdqa32       [DPTR1+_DATA+(1)*64],W1
 467         vmovdqa32       [DPTR1+_DATA+(2)*64],W2
 468         vmovdqa32       [DPTR1+_DATA+(3)*64],W3
 469         vmovdqa32       [DPTR1+_DATA+(4)*64],W4
 470         vmovdqa32       [DPTR1+_DATA+(5)*64],W5
 471         vmovdqa32       [DPTR1+_DATA+(6)*64],W6
 472         vmovdqa32       [DPTR1+_DATA+(7)*64],W7
 473         vmovdqa32       [DPTR1+_DATA+(8)*64],W8
 474         vmovdqa32       [DPTR1+_DATA+(9)*64],W9
 475         vmovdqa32       [DPTR1+_DATA+(10)*64],W10
 476         vmovdqa32       [DPTR1+_DATA+(11)*64],W11
 477         vmovdqa32       [DPTR1+_DATA+(12)*64],W12
 478         vmovdqa32       [DPTR1+_DATA+(13)*64],W13
 479         vmovdqa32       [DPTR1+_DATA+(14)*64],W14
 480         vmovdqa32       [DPTR1+_DATA+(15)*64],W15
 481
 482         ;; second 16 lanes read
 483         mov     inp0, [IN + 16*8]
 484         mov     inp1, [IN + 17*8]
 485         mov     inp2, [IN + 18*8]
 486         mov     inp3, [IN + 19*8]
 487         mov     inp4, [IN + 20*8]
 488         mov     inp5, [IN + 21*8]
 489         mov     inp6, [IN + 22*8]
 490         mov     inp7, [IN + 23*8]
 491         vmovdqu32       W0,[inp0+IDX]
 492         vmovdqu32       W1,[inp1+IDX]
 493         vmovdqu32       W2,[inp2+IDX]
 494         vmovdqu32       W3,[inp3+IDX]
 495         vmovdqu32       W4,[inp4+IDX]
 496         vmovdqu32       W5,[inp5+IDX]
 497         vmovdqu32       W6,[inp6+IDX]
 498         vmovdqu32       W7,[inp7+IDX]
 499         mov     inp0, [IN + 24*8]
 500         mov     inp1, [IN + 25*8]
 501         mov     inp2, [IN + 26*8]
 502         mov     inp3, [IN + 27*8]
 503         mov     inp4, [IN + 28*8]
 504         mov     inp5, [IN + 29*8]
 505         mov     inp6, [IN + 30*8]
 506         mov     inp7, [IN + 31*8]
 507         vmovdqu32       W8, [inp0+IDX]
 508         vmovdqu32       W9, [inp1+IDX]
 509         vmovdqu32       W10,[inp2+IDX]
 510         vmovdqu32       W11,[inp3+IDX]
 511         vmovdqu32       W12,[inp4+IDX]
 512         vmovdqu32       W13,[inp5+IDX]
 513         vmovdqu32       W14,[inp6+IDX]
 514         vmovdqu32       W15,[inp7+IDX]
 515         ;; second 16 lanes trans&write
 516         TRANSPOSE16 W0, W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, TMP0, TMP1
 517         vmovdqa32       [DPTR1+_DATA+(16+0)*64],W0
 518         vmovdqa32       [DPTR1+_DATA+(16+1)*64],W1
 519         vmovdqa32       [DPTR1+_DATA+(16+2)*64],W2
 520         vmovdqa32       [DPTR1+_DATA+(16+3)*64],W3
 521         vmovdqa32       [DPTR1+_DATA+(16+4)*64],W4
 522         vmovdqa32       [DPTR1+_DATA+(16+5)*64],W5
 523         vmovdqa32       [DPTR1+_DATA+(16+6)*64],W6
 524         vmovdqa32       [DPTR1+_DATA+(16+7)*64],W7
 525         vmovdqa32       [DPTR1+_DATA+(16+8)*64],W8
 526         vmovdqa32       [DPTR1+_DATA+(16+9)*64],W9
 527         vmovdqa32       [DPTR1+_DATA+(16+10)*64],W10
 528         vmovdqa32       [DPTR1+_DATA+(16+11)*64],W11
 529         vmovdqa32       [DPTR1+_DATA+(16+12)*64],W12
 530         vmovdqa32       [DPTR1+_DATA+(16+13)*64],W13
 531         vmovdqa32       [DPTR1+_DATA+(16+14)*64],W14
 532         vmovdqa32       [DPTR1+_DATA+(16+15)*64],W15
 533
 534         ;; Initialize digests
 535         ;; vmovdqu32 replace vmovdqa32
 536         vmovdqu32       A, [DIGEST + 0 * MD5_DIGEST_ROW_SIZE]
 537         vmovdqu32       B, [DIGEST + 1 * MD5_DIGEST_ROW_SIZE]
 538         vmovdqu32       C, [DIGEST + 2 * MD5_DIGEST_ROW_SIZE]
 539         vmovdqu32       D, [DIGEST + 3 * MD5_DIGEST_ROW_SIZE]
 540         ; Load the digest for each stream (9-16)
 541         vmovdqu32       A1,[DIGEST + 0 * MD5_DIGEST_ROW_SIZE + 64]
 542         vmovdqu32       B1,[DIGEST + 1 * MD5_DIGEST_ROW_SIZE + 64]
 543         vmovdqu32       C1,[DIGEST + 2 * MD5_DIGEST_ROW_SIZE + 64]
 544         vmovdqu32       D1,[DIGEST + 3 * MD5_DIGEST_ROW_SIZE + 64]
 545
 546 .lloop:
 547         ;; Increment IDX to point to next data block (64 bytes per block)
 548         add     IDX, 64
 549
 550         ; Save digests for later addition
 551         vmovdqa32       [Z_AA], A
 552         vmovdqa32       [Z_BB], B
 553         vmovdqa32       [Z_CC], C
 554         vmovdqa32       [Z_DD], D
 555         vmovdqa32       [Z_AA1], A1
 556         vmovdqa32       [Z_BB1], B1
 557         vmovdqa32       [Z_CC1], C1
 558         vmovdqa32       [Z_DD1], D1
 559
 560         sub     SIZE, 1
 561         je      .LastLoop
 562
 563 %assign I       0
 564 %assign I_fimm  0xCA
 565 %rep 16         ; 0<=I<=15
 566  %assign I_rotX I/16+1
 567  %assign I_rotY (I % 4 + 1)
 568  %assign I_data I
 569         vpbroadcastd md5c, [TBL + I * 4]
 570         PROCESS_LOOP md5c, DPTR1+ I_data*64, I_fimm, APPEND3(rot, I_rotX, I_rotY), TMP0, TMP1
 571  %assign I (I+1)
 572 %endrep
 573         ;; first 16 lanes read
 574         mov     inp0, [IN + 0*8]
 575         mov     inp1, [IN + 1*8]
 576         mov     inp2, [IN + 2*8]
 577         mov     inp3, [IN + 3*8]
 578         mov     inp4, [IN + 4*8]
 579         mov     inp5, [IN + 5*8]
 580         mov     inp6, [IN + 6*8]
 581         mov     inp7, [IN + 7*8]
 582         vmovdqu32       W0,[inp0+IDX]
 583         vmovdqu32       W1,[inp1+IDX]
 584         vmovdqu32       W2,[inp2+IDX]
 585         vmovdqu32       W3,[inp3+IDX]
 586         vmovdqu32       W4,[inp4+IDX]
 587         vmovdqu32       W5,[inp5+IDX]
 588         vmovdqu32       W6,[inp6+IDX]
 589         vmovdqu32       W7,[inp7+IDX]
 590         mov     inp0, [IN + 8*8]
 591         mov     inp1, [IN + 9*8]
 592         mov     inp2, [IN +10*8]
 593         mov     inp3, [IN +11*8]
 594         mov     inp4, [IN +12*8]
 595         mov     inp5, [IN +13*8]
 596         mov     inp6, [IN +14*8]
 597         mov     inp7, [IN +15*8]
 598         vmovdqu32       W8, [inp0+IDX]
 599         vmovdqu32       W9, [inp1+IDX]
 600         vmovdqu32       W10,[inp2+IDX]
 601         vmovdqu32       W11,[inp3+IDX]
 602         vmovdqu32       W12,[inp4+IDX]
 603         vmovdqu32       W13,[inp5+IDX]
 604         vmovdqu32       W14,[inp6+IDX]
 605         vmovdqu32       W15,[inp7+IDX]
 606
 607 %assign I       16
 608 %assign I_fimm  0xE4
 609 %rep 16         ; 16<=I<=31
 610  %assign I_data ((5*I+1) % 16)
 611  %assign I_rotX I/16+1
 612  %assign I_rotY (I % 4 + 1)
 613         vpbroadcastd md5c, [TBL + I * 4]
 614         PROCESS_LOOP md5c, DPTR1+ I_data*64, I_fimm, APPEND3(rot, I_rotX, I_rotY), TMP0, TMP1
 615  %assign I (I+1)
 616 %endrep
 617
 618         ;; first 16 lanes trans&write
 619         TRANSPOSE16 W0, W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, TMP0, TMP1
 620         vmovdqa32       [DPTR2+_DATA+(0)*64],W0
 621         vmovdqa32       [DPTR2+_DATA+(1)*64],W1
 622         vmovdqa32       [DPTR2+_DATA+(2)*64],W2
 623         vmovdqa32       [DPTR2+_DATA+(3)*64],W3
 624         vmovdqa32       [DPTR2+_DATA+(4)*64],W4
 625         vmovdqa32       [DPTR2+_DATA+(5)*64],W5
 626         vmovdqa32       [DPTR2+_DATA+(6)*64],W6
 627         vmovdqa32       [DPTR2+_DATA+(7)*64],W7
 628         vmovdqa32       [DPTR2+_DATA+(8)*64],W8
 629         vmovdqa32       [DPTR2+_DATA+(9)*64],W9
 630         vmovdqa32       [DPTR2+_DATA+(10)*64],W10
 631         vmovdqa32       [DPTR2+_DATA+(11)*64],W11
 632         vmovdqa32       [DPTR2+_DATA+(12)*64],W12
 633         vmovdqa32       [DPTR2+_DATA+(13)*64],W13
 634         vmovdqa32       [DPTR2+_DATA+(14)*64],W14
 635         vmovdqa32       [DPTR2+_DATA+(15)*64],W15
 636
 637 %assign I       32
 638 %assign I_fimm  0x96
 639 %rep 16         ; 32<=I<=47
 640  %assign I_data ((3*I+5) % 16)
 641  %assign I_rotX I/16+1
 642  %assign I_rotY (I % 4 + 1)
 643         vpbroadcastd md5c, [TBL + I * 4]
 644         PROCESS_LOOP md5c, DPTR1+ I_data*64, I_fimm, APPEND3(rot, I_rotX, I_rotY), TMP0, TMP1
 645  %assign I (I+1)
 646 %endrep
 647
 648         ;; second 16 lanes read
 649         mov     inp0, [IN + 16*8]
 650         mov     inp1, [IN + 17*8]
 651         mov     inp2, [IN + 18*8]
 652         mov     inp3, [IN + 19*8]
 653         mov     inp4, [IN + 20*8]
 654         mov     inp5, [IN + 21*8]
 655         mov     inp6, [IN + 22*8]
 656         mov     inp7, [IN + 23*8]
 657         vmovdqu32       W0,[inp0+IDX]
 658         vmovdqu32       W1,[inp1+IDX]
 659         vmovdqu32       W2,[inp2+IDX]
 660         vmovdqu32       W3,[inp3+IDX]
 661         vmovdqu32       W4,[inp4+IDX]
 662         vmovdqu32       W5,[inp5+IDX]
 663         vmovdqu32       W6,[inp6+IDX]
 664         vmovdqu32       W7,[inp7+IDX]
 665         mov     inp0, [IN + 24*8]
 666         mov     inp1, [IN + 25*8]
 667         mov     inp2, [IN + 26*8]
 668         mov     inp3, [IN + 27*8]
 669         mov     inp4, [IN + 28*8]
 670         mov     inp5, [IN + 29*8]
 671         mov     inp6, [IN + 30*8]
 672         mov     inp7, [IN + 31*8]
 673         vmovdqu32       W8, [inp0+IDX]
 674         vmovdqu32       W9, [inp1+IDX]
 675         vmovdqu32       W10,[inp2+IDX]
 676         vmovdqu32       W11,[inp3+IDX]
 677         vmovdqu32       W12,[inp4+IDX]
 678         vmovdqu32       W13,[inp5+IDX]
 679         vmovdqu32       W14,[inp6+IDX]
 680         vmovdqu32       W15,[inp7+IDX]
 681
 682 %assign I       48
 683 %assign I_fimm  0x39
 684 %rep 16 ; 48<=I<=63
 685  %assign I_rotX (I/16+1)
 686  %assign I_rotY (I % 4 + 1)
 687  %assign I_data ((7*I) % 16)
 688         vpbroadcastd md5c, [TBL + I * 4]
 689         PROCESS_LOOP md5c,  DPTR1+ I_data*64, I_fimm, APPEND3(rot, I_rotX, I_rotY), TMP0, TMP1
 690  %assign I (I+1)
 691 %endrep
 692
 693         ;; second 16 lanes trans&write
 694         TRANSPOSE16 W0, W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, TMP0, TMP1
 695         vmovdqa32       [DPTR2+_DATA+(16+0)*64],W0
 696         vmovdqa32       [DPTR2+_DATA+(16+1)*64],W1
 697         vmovdqa32       [DPTR2+_DATA+(16+2)*64],W2
 698         vmovdqa32       [DPTR2+_DATA+(16+3)*64],W3
 699         vmovdqa32       [DPTR2+_DATA+(16+4)*64],W4
 700         vmovdqa32       [DPTR2+_DATA+(16+5)*64],W5
 701         vmovdqa32       [DPTR2+_DATA+(16+6)*64],W6
 702         vmovdqa32       [DPTR2+_DATA+(16+7)*64],W7
 703         vmovdqa32       [DPTR2+_DATA+(16+8)*64],W8
 704         vmovdqa32       [DPTR2+_DATA+(16+9)*64],W9
 705         vmovdqa32       [DPTR2+_DATA+(16+10)*64],W10
 706         vmovdqa32       [DPTR2+_DATA+(16+11)*64],W11
 707         vmovdqa32       [DPTR2+_DATA+(16+12)*64],W12
 708         vmovdqa32       [DPTR2+_DATA+(16+13)*64],W13
 709         vmovdqa32       [DPTR2+_DATA+(16+14)*64],W14
 710         vmovdqa32       [DPTR2+_DATA+(16+15)*64],W15
 711
 712         ; Add old digest
 713         vpaddd          A,A,[Z_AA]
 714         vpaddd          B,B,[Z_BB]
 715         vpaddd          C,C,[Z_CC]
 716         vpaddd          D,D,[Z_DD]
 717         vpaddd          A1,A1,[Z_AA1]
 718         vpaddd          B1,B1,[Z_BB1]
 719         vpaddd          C1,C1,[Z_CC1]
 720         vpaddd          D1,D1,[Z_DD1]
 721
 722         ; Swap DPTR1 and DPTR2
 723         xchg    DPTR1, DPTR2
 724          ;; Proceed to processing of next block
 725         jmp     .lloop
 726
 727 .LastLoop:
 728 %assign I       0
 729 %assign I_fimm  0xCA
 730 %rep 16         ; 0<=I<=15
 731  %assign I_rotX I/16+1
 732  %assign I_rotY (I % 4 + 1)
 733  %assign I_data I
 734         vpbroadcastd md5c, [TBL + I * 4]
 735         PROCESS_LOOP md5c, DPTR1+ I_data*64, I_fimm, APPEND3(rot, I_rotX, I_rotY), TMP0, TMP1
 736  %assign I (I+1)
 737 %endrep
 738
 739 %assign I       16
 740 %assign I_fimm  0xE4
 741 %rep 16         ; 16<=I<=31
 742  %assign I_data ((5*I+1) % 16)
 743  %assign I_rotX I/16+1
 744  %assign I_rotY (I % 4 + 1)
 745         vpbroadcastd md5c, [TBL + I * 4]
 746         PROCESS_LOOP md5c, DPTR1+ I_data*64, I_fimm, APPEND3(rot, I_rotX, I_rotY), TMP0, TMP1
 747  %assign I (I+1)
 748 %endrep
 749
 750 %assign I       32
 751 %assign I_fimm  0x96
 752 %rep 16         ; 32<=I<=47
 753  %assign I_data ((3*I+5) % 16)
 754  %assign I_rotX I/16+1
 755  %assign I_rotY (I % 4 + 1)
 756         vpbroadcastd md5c, [TBL + I * 4]
 757         PROCESS_LOOP md5c, DPTR1+ I_data*64, I_fimm, APPEND3(rot, I_rotX, I_rotY), TMP0, TMP1
 758  %assign I (I+1)
 759 %endrep
 760
 761 %assign I       48
 762 %assign I_fimm  0x39
 763 %rep 16 ; 48<=I<=63
 764  %assign I_rotX (I/16+1)
 765  %assign I_rotY (I % 4 + 1)
 766  %assign I_data ((7*I) % 16)
 767         vpbroadcastd md5c, [TBL + I * 4]
 768         PROCESS_LOOP md5c,  DPTR1+ I_data*64, I_fimm, APPEND3(rot, I_rotX, I_rotY), TMP0, TMP1
 769  %assign I (I+1)
 770 %endrep
 771
 772         ; Add old digest
 773         vpaddd          A,A,[Z_AA]
 774         vpaddd          B,B,[Z_BB]
 775         vpaddd          C,C,[Z_CC]
 776         vpaddd          D,D,[Z_DD]
 777         vpaddd          A1,A1,[Z_AA1]
 778         vpaddd          B1,B1,[Z_BB1]
 779         vpaddd          C1,C1,[Z_CC1]
 780         vpaddd          D1,D1,[Z_DD1]
 781
 782         ;; update into data pointers
 783 %assign I 0
 784 %rep 16
 785         mov    inp0, [IN + (2*I)*8]
 786         mov    inp1, [IN + (2*I +1)*8]
 787         add    inp0, IDX
 788         add    inp1, IDX
 789         mov    [IN + (2*I)*8], inp0
 790         mov    [IN + (2*I+1)*8], inp1
 791 %assign I (I+1)
 792 %endrep
 793
 794         vmovdqu32       [DIGEST + 0*MD5_DIGEST_ROW_SIZE  ], A
 795         vmovdqu32       [DIGEST + 1*MD5_DIGEST_ROW_SIZE  ], B
 796         vmovdqu32       [DIGEST + 2*MD5_DIGEST_ROW_SIZE  ], C
 797         vmovdqu32       [DIGEST + 3*MD5_DIGEST_ROW_SIZE  ], D
 798         ; Store the digest for each stream (9-16)
 799         vmovdqu32       [DIGEST + 0 * MD5_DIGEST_ROW_SIZE + 64], A1
 800         vmovdqu32       [DIGEST + 1 * MD5_DIGEST_ROW_SIZE + 64], B1
 801         vmovdqu32       [DIGEST + 2 * MD5_DIGEST_ROW_SIZE + 64], C1
 802         vmovdqu32       [DIGEST + 3 * MD5_DIGEST_ROW_SIZE + 64], D1
 803
 804         mov     rsp, [rsp + _RSP_SAVE]
 805         ret
 806
 807 section .data
 808 align 64
 809 MD5_TABLE:
 810         dd      0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee
 811         dd      0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501
 812         dd      0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be
 813         dd      0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821
 814         dd      0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa
 815         dd      0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8
 816         dd      0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed
 817         dd      0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a
 818         dd      0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c
 819         dd      0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70
 820         dd      0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05
 821         dd      0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665
 822         dd      0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039
 823         dd      0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1
 824         dd      0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1
 825         dd      0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391
 826
 827 PSHUFFLE_TRANSPOSE16_MASK1:     dq 0x0000000000000000
 828                                 dq 0x0000000000000001
 829                                 dq 0x0000000000000008
 830                                 dq 0x0000000000000009
 831                                 dq 0x0000000000000004
 832                                 dq 0x0000000000000005
 833                                 dq 0x000000000000000C
 834                                 dq 0x000000000000000D
 835
 836 PSHUFFLE_TRANSPOSE16_MASK2:     dq 0x0000000000000002
 837                                 dq 0x0000000000000003
 838                                 dq 0x000000000000000A
 839                                 dq 0x000000000000000B
 840                                 dq 0x0000000000000006
 841                                 dq 0x0000000000000007
 842                                 dq 0x000000000000000E
 843                                 dq 0x000000000000000F
 844
 845 %else
 846 %ifidn __OUTPUT_FORMAT__, win64
 847 global no_md5_mb_x16x2_avx512
 848 no_md5_mb_x16x2_avx512:
 849 %endif
 850 %endif ; HAVE_AS_KNOWS_AVX512