ceph/src/spdk/intel-ipsec-mb/sse/sha512_one_block_sse.asm

   1 ;;
   2 ;; Copyright (c) 2012-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 ; This code schedules 1 blocks at a time, with 4 lanes per block
  29 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  30 %include "os.asm"
  31
  32 %define MOVDQ movdqu ;; assume buffers not aligned
  33
  34 %ifndef FUNC
  35 %define FUNC sha512_one_block_sse
  36 %endif
  37
  38 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Define Macros
  39
  40 ; COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask
  41 ; Load xmm with mem and byte swap each dword
  42 %macro COPY_XMM_AND_BSWAP 3
  43         MOVDQ %1, %2
  44         pshufb %1, %3
  45 %endmacro
  46
  47 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  48
  49 %define X0 xmm4
  50 %define X1 xmm5
  51 %define X2 xmm6
  52 %define X3 xmm7
  53 %define X4 xmm8
  54 %define X5 xmm9
  55 %define X6 xmm10
  56 %define X7 xmm11
  57
  58 %define XTMP0 xmm0
  59 %define XTMP1 xmm1
  60 %define XTMP2 xmm2
  61 %define XTMP3 xmm3
  62 %define XFER  xmm13
  63
  64 %define BYTE_FLIP_MASK  xmm12
  65
  66 %ifdef LINUX
  67 %define CTX     rsi     ; 2nd arg
  68 %define INP     rdi     ; 1st arg
  69
  70 %define SRND    rdi     ; clobbers INP
  71 %define c       rcx
  72 %define d       r8
  73 %define e       rdx
  74 %else
  75 %define CTX     rdx     ; 2nd arg
  76 %define INP     rcx     ; 1st arg
  77
  78 %define SRND    rcx     ; clobbers INP
  79 %define c       rdi
  80 %define d       rsi
  81 %define e       r8
  82
  83 %endif
  84 %define TBL     rbp
  85 %define a rax
  86 %define b rbx
  87
  88 %define f r9
  89 %define g r10
  90 %define h r11
  91
  92 %define y0 r13
  93 %define y1 r14
  94 %define y2 r15
  95
  96
  97 struc STACK
  98 %ifndef LINUX
  99 _XMM_SAVE:      reso    8
 100 %endif
 101 _XFER:          reso    1
 102 endstruc
 103
 104 %ifndef H0
 105 %define H0 0x6a09e667f3bcc908
 106 %define H1 0xbb67ae8584caa73b
 107 %define H2 0x3c6ef372fe94f82b
 108 %define H3 0xa54ff53a5f1d36f1
 109 %define H4 0x510e527fade682d1
 110 %define H5 0x9b05688c2b3e6c1f
 111 %define H6 0x1f83d9abfb41bd6b
 112 %define H7 0x5be0cd19137e2179
 113 %endif
 114
 115 ; rotate_Xs
 116 ; Rotate values of symbols X0...X7
 117 %macro rotate_Xs 0
 118 %xdefine X_ X0
 119 %xdefine X0 X1
 120 %xdefine X1 X2
 121 %xdefine X2 X3
 122 %xdefine X3 X4
 123 %xdefine X4 X5
 124 %xdefine X5 X6
 125 %xdefine X6 X7
 126 %xdefine X7 X_
 127 %endm
 128
 129 ; ROTATE_ARGS
 130 ; Rotate values of symbols a...h
 131 %macro ROTATE_ARGS 0
 132 %xdefine TMP_ h
 133 %xdefine h g
 134 %xdefine g f
 135 %xdefine f e
 136 %xdefine e d
 137 %xdefine d c
 138 %xdefine c b
 139 %xdefine b a
 140 %xdefine a TMP_
 141 %endm
 142
 143 %macro TWO_ROUNDS_AND_SCHED 0
 144
 145                 ;; compute s0 four at a time and s1 two at a time
 146                 ;; compute W[-16] + W[-7] 4 at a time
 147                 movdqa  XTMP0, X5
 148         mov     y0, e           ; y0 = e
 149         mov     y1, a           ; y1 = a
 150         ror     y0, (41-18)     ; y0 = e >> (41-18)
 151                 palignr XTMP0, X4, 8    ; XTMP0 = W[-7]
 152         xor     y0, e           ; y0 = e ^ (e >> (41-18))
 153         mov     y2, f           ; y2 = f
 154         ror     y1, (39-34)     ; y1 = a >> (39-34)
 155         xor     y1, a           ; y1 = a ^ (a >> (39-34)
 156                 movdqa  XTMP1, X1
 157         ror     y0, (18-14)     ; y0 = (e >> (18-14)) ^ (e >> (41-14))
 158         xor     y2, g           ; y2 = f^g
 159                 paddq   XTMP0, X0       ; XTMP0 = W[-7] + W[-16]
 160         ror     y1, (34-28)     ; y1 = (a >> (34-28)) ^ (a >> (39-28))
 161         xor     y0, e           ; y0 = e ^ (e >> (18-14)) ^ (e >> (41-14))
 162         and     y2, e           ; y2 = (f^g)&e
 163                 ;; compute s0
 164                 palignr XTMP1, X0, 8    ; XTMP1 = W[-15]
 165         xor     y1, a           ; y1 = a ^ (a >> (34-28)) ^ (a >> (39-28))
 166         xor     y2, g           ; y2 = CH = ((f^g)&e)^g
 167                 movdqa  XTMP2, XTMP1    ; XTMP2 = W[-15]
 168         ror     y0, 14          ; y0 = S1 = (e>>14) & (e>>18) ^ (e>>41)
 169         add     y2, y0          ; y2 = S1 + CH
 170         add     y2, [rsp + _XFER + 0*8] ; y2 = k + w + S1 + CH
 171         ror     y1, 28          ; y1 = S0 = (a>>28) ^ (a>>34) ^ (a>>39)
 172                 movdqa  XTMP3, XTMP1    ; XTMP3 = W[-15]
 173         mov     y0, a           ; y0 = a
 174         add     h, y2           ; h = h + S1 + CH + k + w
 175                 psllq   XTMP1, (64-1)
 176         mov     y2, a           ; y2 = a
 177         or      y0, c           ; y0 = a|c
 178                 psrlq   XTMP2, 1
 179         add     d, h            ; d = d + t1
 180         and     y2, c           ; y2 = a&c
 181                 por     XTMP1, XTMP2    ; XTMP1 = W[-15] ror 1
 182         and     y0, b           ; y0 = (a|c)&b
 183         add     h, y1           ; h = t1 + S0
 184                 movdqa  XTMP2, XTMP3    ; XTMP2 = W[-15]
 185                 psrlq   XTMP2, 8
 186         or      y0, y2          ; y0 = MAJ = (a|c)&b)|(a&c)
 187         add     h, y0           ; h = t1 + S0 + MAJ
 188                 movdqa  X0, XTMP3       ; X0 = W[-15]
 189                 psllq   XTMP3, (64-8)
 190
 191
 192 ROTATE_ARGS
 193                 pxor    XTMP1, XTMP3
 194                 psrlq   X0, 7           ; X0 = W[-15] >> 7
 195         mov     y0, e           ; y0 = e
 196         mov     y1, a           ; y1 = a
 197                 pxor    XTMP1, XTMP2    ; XTMP1 = W[-15] ror 1 ^ W[-15] ror 8
 198         ror     y0, (41-18)     ; y0 = e >> (41-18)
 199         xor     y0, e           ; y0 = e ^ (e >> (41-18))
 200         mov     y2, f           ; y2 = f
 201                 pxor    XTMP1, X0       ; XTMP1 = s0
 202         ror     y1, (39-34)     ; y1 = a >> (39-34)
 203         xor     y1, a           ; y1 = a ^ (a >> (39-34)
 204                 ;; compute s1
 205                 movdqa  XTMP2, X7       ; XTMP2 = W[-2]
 206         ror     y0, (18-14)     ; y0 = (e >> (18-14)) ^ (e >> (41-14))
 207         xor     y2, g           ; y2 = f^g
 208                 paddq   XTMP0, XTMP1    ; XTMP0 = W[-16] + W[-7] + s0
 209         ror     y1, (34-28)     ; y1 = (a >> (34-28)) ^ (a >> (39-28))
 210         xor     y0, e           ; y0 = e ^ (e >> (18-14)) ^ (e >> (41-14))
 211                 movdqa  XTMP3, XTMP2    ; XTMP3 = W[-2]
 212                 movdqa  X0, XTMP2       ; X0 = W[-2]
 213         and     y2, e           ; y2 = (f^g)&e
 214         ror     y0, 14          ; y0 = S1 = (e>>14) & (e>>18) ^ (e>>41)
 215         xor     y1, a           ; y1 = a ^ (a >> (34-28)) ^ (a >> (39-28))
 216                 psllq   XTMP3, (64-19)
 217         xor     y2, g           ; y2 = CH = ((f^g)&e)^g
 218         add     y2, y0          ; y2 = S1 + CH
 219         add     y2, [rsp + _XFER + 1*8] ; y2 = k + w + S1 + CH
 220                 psrlq   X0, 19
 221         ror     y1, 28          ; y1 = S0 = (a>>28) ^ (a>>34) ^ (a>>39)
 222         mov     y0, a           ; y0 = a
 223         add     h, y2           ; h = h + S1 + CH + k + w
 224                 por     XTMP3, X0       ; XTMP3 = W[-2] ror 19
 225         mov     y2, a           ; y2 = a
 226         or      y0, c           ; y0 = a|c
 227                 movdqa  X0, XTMP2       ; X0 = W[-2]
 228                 movdqa  XTMP1, XTMP2    ; XTMP1 = W[-2]
 229         add     d, h            ; d = d + t1
 230         and     y2, c           ; y2 = a&c
 231                 psllq   X0, (64-61)
 232         and     y0, b           ; y0 = (a|c)&b
 233         add     h, y1           ; h = t1 + S0
 234                 psrlq   XTMP1, 61
 235         or      y0, y2          ; y0 = MAJ = (a|c)&b)|(a&c)
 236         add     h, y0           ; h = t1 + S0 + MAJ
 237                 por     X0, XTMP1       ; X0 = W[-2] ror 61
 238                 psrlq   XTMP2, 6        ; XTMP2 = W[-2] >> 6
 239                 pxor    XTMP2, XTMP3
 240                 pxor    X0, XTMP2       ; X0 = s1
 241                 paddq   X0, XTMP0       ; X0 = {W[1], W[0]}
 242
 243 ROTATE_ARGS
 244 rotate_Xs
 245 %endm
 246
 247 ;; input is [rsp + _XFER + %1 * 8]
 248 %macro DO_ROUND 1
 249         mov     y0, e           ; y0 = e
 250         ror     y0, (41-18)     ; y0 = e >> (41-18)
 251         mov     y1, a           ; y1 = a
 252         xor     y0, e           ; y0 = e ^ (e >> (41-18))
 253         ror     y1, (39-34)     ; y1 = a >> (39-34)
 254         mov     y2, f           ; y2 = f
 255         xor     y1, a           ; y1 = a ^ (a >> (39-34)
 256         ror     y0, (18-14)     ; y0 = (e >> (18-14)) ^ (e >> (41-14))
 257         xor     y2, g           ; y2 = f^g
 258         xor     y0, e           ; y0 = e ^ (e >> (18-14)) ^ (e >> (25-6))
 259         ror     y1, (34-28)     ; y1 = (a >> (34-28)) ^ (a >> (39-28))
 260         and     y2, e           ; y2 = (f^g)&e
 261         xor     y1, a           ; y1 = a ^ (a >> (34-28)) ^ (a >> (39-28))
 262         ror     y0, 14          ; y0 = S1 = (e>>14) & (e>>18) ^ (e>>41)
 263         xor     y2, g           ; y2 = CH = ((f^g)&e)^g
 264         add     y2, y0          ; y2 = S1 + CH
 265         ror     y1, 28          ; y1 = S0 = (a>>28) ^ (a>>34) ^ (a>>39)
 266         add     y2, [rsp + _XFER + %1*8] ; y2 = k + w + S1 + CH
 267         mov     y0, a           ; y0 = a
 268         add     h, y2           ; h = h + S1 + CH + k + w
 269         mov     y2, a           ; y2 = a
 270         or      y0, c           ; y0 = a|c
 271         add     d, h            ; d = d + t1
 272         and     y2, c           ; y2 = a&c
 273         and     y0, b           ; y0 = (a|c)&b
 274         add     h, y1           ; h = t1 + S0
 275         or      y0, y2          ; y0 = MAJ = (a|c)&b)|(a&c)
 276         add     h, y0           ; h = t1 + S0 + MAJ
 277         ROTATE_ARGS
 278 %endm
 279
 280 section .data
 281 default rel
 282 align 64
 283 K512:
 284         dq      0x428a2f98d728ae22,0x7137449123ef65cd
 285         dq      0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
 286         dq      0x3956c25bf348b538,0x59f111f1b605d019
 287         dq      0x923f82a4af194f9b,0xab1c5ed5da6d8118
 288         dq      0xd807aa98a3030242,0x12835b0145706fbe
 289         dq      0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
 290         dq      0x72be5d74f27b896f,0x80deb1fe3b1696b1
 291         dq      0x9bdc06a725c71235,0xc19bf174cf692694
 292         dq      0xe49b69c19ef14ad2,0xefbe4786384f25e3
 293         dq      0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
 294         dq      0x2de92c6f592b0275,0x4a7484aa6ea6e483
 295         dq      0x5cb0a9dcbd41fbd4,0x76f988da831153b5
 296         dq      0x983e5152ee66dfab,0xa831c66d2db43210
 297         dq      0xb00327c898fb213f,0xbf597fc7beef0ee4
 298         dq      0xc6e00bf33da88fc2,0xd5a79147930aa725
 299         dq      0x06ca6351e003826f,0x142929670a0e6e70
 300         dq      0x27b70a8546d22ffc,0x2e1b21385c26c926
 301         dq      0x4d2c6dfc5ac42aed,0x53380d139d95b3df
 302         dq      0x650a73548baf63de,0x766a0abb3c77b2a8
 303         dq      0x81c2c92e47edaee6,0x92722c851482353b
 304         dq      0xa2bfe8a14cf10364,0xa81a664bbc423001
 305         dq      0xc24b8b70d0f89791,0xc76c51a30654be30
 306         dq      0xd192e819d6ef5218,0xd69906245565a910
 307         dq      0xf40e35855771202a,0x106aa07032bbd1b8
 308         dq      0x19a4c116b8d2d0c8,0x1e376c085141ab53
 309         dq      0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
 310         dq      0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
 311         dq      0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
 312         dq      0x748f82ee5defb2fc,0x78a5636f43172f60
 313         dq      0x84c87814a1f0ab72,0x8cc702081a6439ec
 314         dq      0x90befffa23631e28,0xa4506cebde82bde9
 315         dq      0xbef9a3f7b2c67915,0xc67178f2e372532b
 316         dq      0xca273eceea26619c,0xd186b8c721c0c207
 317         dq      0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
 318         dq      0x06f067aa72176fba,0x0a637dc5a2c898a6
 319         dq      0x113f9804bef90dae,0x1b710b35131c471b
 320         dq      0x28db77f523047d84,0x32caab7b40c72493
 321         dq      0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
 322         dq      0x4cc5d4becb3e42b6,0x597f299cfc657e2a
 323         dq      0x5fcb6fab3ad6faec,0x6c44198c4a475817
 324
 325 h0:     dq      H0
 326 h1:     dq      H1
 327 h2:     dq      H2
 328 h3:     dq      H3
 329 h4:     dq      H4
 330 h5:     dq      H5
 331 h6:     dq      H6
 332 h7:     dq      H7
 333
 334 align 16
 335 PSHUFFLE_BYTE_FLIP_MASK: ;ddq 0x08090a0b0c0d0e0f0001020304050607
 336         dq 0x0001020304050607, 0x08090a0b0c0d0e0f
 337
 338 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 339 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 340 ;; void FUNC(void *input_data, UINT64 digest[8])
 341 ;; arg 1 : pointer to input data
 342 ;; arg 2 : pointer to digest
 343 section .text
 344 MKGLOBAL(FUNC,function,)
 345 align 32
 346 FUNC:
 347         push    rbx
 348 %ifndef LINUX
 349         push    rsi
 350         push    rdi
 351 %endif
 352         push    rbp
 353         push    r13
 354         push    r14
 355         push    r15
 356
 357         sub     rsp,STACK_size
 358 %ifndef LINUX
 359         movdqa  [rsp + _XMM_SAVE + 0*16],xmm6
 360         movdqa  [rsp + _XMM_SAVE + 1*16],xmm7
 361         movdqa  [rsp + _XMM_SAVE + 2*16],xmm8
 362         movdqa  [rsp + _XMM_SAVE + 3*16],xmm9
 363         movdqa  [rsp + _XMM_SAVE + 4*16],xmm10
 364         movdqa  [rsp + _XMM_SAVE + 5*16],xmm11
 365         movdqa  [rsp + _XMM_SAVE + 6*16],xmm12
 366         movdqa  [rsp + _XMM_SAVE + 7*16],xmm13
 367 %endif
 368
 369         ;; load initial digest
 370         mov     a,[rel h0]
 371         mov     b,[rel h1]
 372         mov     c,[rel h2]
 373         mov     d,[rel h3]
 374         mov     e,[rel h4]
 375         mov     f,[rel h5]
 376         mov     g,[rel h6]
 377         mov     h,[rel h7]
 378
 379         movdqa  BYTE_FLIP_MASK, [rel PSHUFFLE_BYTE_FLIP_MASK]
 380
 381         lea     TBL,[rel K512]
 382
 383         ;; byte swap first 16 qwords
 384         COPY_XMM_AND_BSWAP      X0, [INP + 0*16], BYTE_FLIP_MASK
 385         COPY_XMM_AND_BSWAP      X1, [INP + 1*16], BYTE_FLIP_MASK
 386         COPY_XMM_AND_BSWAP      X2, [INP + 2*16], BYTE_FLIP_MASK
 387         COPY_XMM_AND_BSWAP      X3, [INP + 3*16], BYTE_FLIP_MASK
 388         COPY_XMM_AND_BSWAP      X4, [INP + 4*16], BYTE_FLIP_MASK
 389         COPY_XMM_AND_BSWAP      X5, [INP + 5*16], BYTE_FLIP_MASK
 390         COPY_XMM_AND_BSWAP      X6, [INP + 6*16], BYTE_FLIP_MASK
 391         COPY_XMM_AND_BSWAP      X7, [INP + 7*16], BYTE_FLIP_MASK
 392
 393         ;; schedule 64 input qwords, by doing 4 iterations of 16 rounds
 394         mov     SRND, 4
 395 align 16
 396 loop1:
 397
 398 %assign i 0
 399 %rep 7
 400         movdqa  XFER, X0
 401         paddq   XFER, [TBL + i*16]
 402         movdqa  [rsp + _XFER], XFER
 403         TWO_ROUNDS_AND_SCHED
 404 %assign i (i+1)
 405 %endrep
 406
 407         movdqa  XFER, X0
 408         paddq   XFER, [TBL + 7*16]
 409         movdqa  [rsp + _XFER], XFER
 410         add     TBL, 8*16
 411         TWO_ROUNDS_AND_SCHED
 412
 413         sub     SRND, 1
 414         jne     loop1
 415
 416         mov     SRND, 2
 417         jmp loop2a
 418 loop2:
 419         movdqa  X0, X4
 420         movdqa  X1, X5
 421         movdqa  X2, X6
 422         movdqa  X3, X7
 423
 424 loop2a:
 425         paddq   X0, [TBL + 0*16]
 426         movdqa  [rsp + _XFER], X0
 427         DO_ROUND 0
 428         DO_ROUND 1
 429
 430         paddq   X1, [TBL + 1*16]
 431         movdqa  [rsp + _XFER], X1
 432         DO_ROUND 0
 433         DO_ROUND 1
 434
 435         paddq   X2, [TBL + 2*16]
 436         movdqa  [rsp + _XFER], X2
 437         DO_ROUND 0
 438         DO_ROUND 1
 439
 440         paddq   X3, [TBL + 3*16]
 441         movdqa  [rsp + _XFER], X3
 442         add     TBL, 4*16
 443         DO_ROUND 0
 444         DO_ROUND 1
 445
 446         sub     SRND, 1
 447         jne     loop2
 448
 449         add     a,[rel h0]
 450         add     b,[rel h1]
 451         add     c,[rel h2]
 452         add     d,[rel h3]
 453         add     e,[rel h4]
 454         add     f,[rel h5]
 455         add     g,[rel h6]
 456         mov     [8*0 + CTX],a
 457         mov     [8*1 + CTX],b
 458         mov     [8*2 + CTX],c
 459         mov     [8*3 + CTX],d
 460         mov     [8*4 + CTX],e
 461         mov     [8*5 + CTX],f
 462         mov     [8*6 + CTX],g
 463         add     h,[rel h7]
 464         mov     [8*7 + CTX],h
 465
 466 done_hash:
 467 %ifndef LINUX
 468         movdqa  xmm6,[rsp + _XMM_SAVE + 0*16]
 469         movdqa  xmm7,[rsp + _XMM_SAVE + 1*16]
 470         movdqa  xmm8,[rsp + _XMM_SAVE + 2*16]
 471         movdqa  xmm9,[rsp + _XMM_SAVE + 3*16]
 472         movdqa  xmm10,[rsp + _XMM_SAVE + 4*16]
 473         movdqa  xmm11,[rsp + _XMM_SAVE + 5*16]
 474         movdqa  xmm12,[rsp + _XMM_SAVE + 6*16]
 475         movdqa  xmm13,[rsp + _XMM_SAVE + 7*16]
 476 %endif
 477
 478         add     rsp, STACK_size
 479
 480         pop     r15
 481         pop     r14
 482         pop     r13
 483         pop     rbp
 484 %ifndef LINUX
 485         pop     rdi
 486         pop     rsi
 487 %endif
 488         pop     rbx
 489
 490         ret
 491
 492
 493 %ifdef LINUX
 494 section .note.GNU-stack noalloc noexec nowrite progbits
 495 %endif