1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2 ; Copyright(c) 2011-2016 Intel Corporation All rights reserved.
4 ; Redistribution and use in source and binary forms, with or without
5 ; modification, are permitted provided that the following conditions
7 ; * Redistributions of source code must retain the above copyright
8 ; notice, this list of conditions and the following disclaimer.
9 ; * Redistributions in binary form must reproduce the above copyright
10 ; notice, this list of conditions and the following disclaimer in
11 ; the documentation and/or other materials provided with the
13 ; * Neither the name of Intel Corporation nor the names of its
14 ; contributors may be used to endorse or promote products derived
15 ; from this software without specific prior written permission.
17 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 ; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 ; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 ; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 ; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 ; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 ; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 ; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 ; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
29 ; XTS encrypt function with 128-bit AES
30 ; expanded keys are not aligned
31 ; plaintext and ciphertext are not aligned
32 ; second key is stored in the stack as aligned to 16 Bytes
33 ; first key is required only once, no need for storage of this key
35 %include "reg_sizes.asm"
38 %define TW rsp ; store 8 tweak values
39 %define keys rsp + 16*8 ; store 11 expanded keys
41 %ifidn __OUTPUT_FORMAT__, win64
42 %define _xmm rsp + 16*19 ; store xmm6:xmm15
45 %ifidn __OUTPUT_FORMAT__, elf64
46 %define _gpr rsp + 16*19 ; store rbx
47 %define VARIABLE_OFFSET 16*8 + 16*11 + 8*1 ; VARIABLE_OFFSET has to be an odd multiple of 8
49 %define _gpr rsp + 16*29 ; store rdi, rsi, rbx
50 %define VARIABLE_OFFSET 16*8 + 16*11 + 16*10 + 8*3 ; VARIABLE_OFFSET has to be an odd multiple of 8
53 %define GHASH_POLY 0x87
55 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
56 ;void XTS_AES_128_enc_expanded_key_sse(
57 ; UINT8 *k2, // key used for tweaking, 16*11 bytes
58 ; UINT8 *k1, // key used for "ECB" encryption, 16*11 bytes
59 ; UINT8 *TW_initial, // initial tweak value, 16 bytes
60 ; UINT64 N, // sector size, in bytes
61 ; const UINT8 *pt, // plaintext sector input data
62 ; UINT8 *ct); // ciphertext sector output data
63 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
65 ; arguments for input parameters
66 %ifidn __OUTPUT_FORMAT__, elf64
71 %xdefine ptr_plaintext r8
72 %xdefine ptr_ciphertext r9
78 %xdefine ptr_plaintext r10; [rsp + VARIABLE_OFFSET + 8*5]
79 %xdefine ptr_ciphertext r11; [rsp + VARIABLE_OFFSET + 8*6]
82 ; arguments for temp parameters
83 %ifidn __OUTPUT_FORMAT__, elf64
85 %define target_ptr_val rsi
86 %define ghash_poly_8b r10
87 %define ghash_poly_8b_temp r11
90 %define target_ptr_val rdx
91 %define ghash_poly_8b rdi
92 %define ghash_poly_8b_temp rsi
95 %define twtempl rax ; global temp registers used for tweak computation
99 ; macro to encrypt the tweak value
103 %define %%xstate_tweak %2
105 %define %%xraw_key %4
107 %define %%ptr_key2 %6
108 %define %%ptr_key1 %7
109 %define %%ptr_expanded_keys %8
111 movdqu %%xkey2, [%%ptr_key2]
112 pxor %%xstate_tweak, %%xkey2 ; ARK for tweak encryption
114 movdqu %%xkey1, [%%ptr_key1]
115 movdqa [%%ptr_expanded_keys+16*0], %%xkey1 ; store round keys in stack
117 movdqu %%xkey2, [%%ptr_key2 + 16*1]
118 aesenc %%xstate_tweak, %%xkey2 ; round 1 for tweak encryption
120 movdqu %%xkey1, [%%ptr_key1 + 16*1]
121 movdqa [%%ptr_expanded_keys+16*1], %%xkey1 ; store round keys in stack
124 movdqu %%xkey2, [%%ptr_key2 + 16*2]
125 aesenc %%xstate_tweak, %%xkey2 ; round 2 for tweak encryption
127 movdqu %%xkey1, [%%ptr_key1 + 16*2]
128 movdqa [%%ptr_expanded_keys+16*2], %%xkey1 ; store round keys in stack
130 movdqu %%xkey2, [%%ptr_key2 + 16*3]
131 aesenc %%xstate_tweak, %%xkey2 ; round 3 for tweak encryption
133 movdqu %%xkey1, [%%ptr_key1 + 16*3]
134 movdqa [%%ptr_expanded_keys+16*3], %%xkey1 ; store round keys in stack
136 movdqu %%xkey2, [%%ptr_key2 + 16*4]
137 aesenc %%xstate_tweak, %%xkey2 ; round 4 for tweak encryption
139 movdqu %%xkey1, [%%ptr_key1 + 16*4]
140 movdqa [%%ptr_expanded_keys+16*4], %%xkey1 ; store round keys in stack
142 movdqu %%xkey2, [%%ptr_key2 + 16*5]
143 aesenc %%xstate_tweak, %%xkey2 ; round 5 for tweak encryption
145 movdqu %%xkey1, [%%ptr_key1 + 16*5]
146 movdqa [%%ptr_expanded_keys+16*5], %%xkey1 ; store round keys in stack
148 movdqu %%xkey2, [%%ptr_key2 + 16*6]
149 aesenc %%xstate_tweak, %%xkey2 ; round 6 for tweak encryption
151 movdqu %%xkey1, [%%ptr_key1 + 16*6]
152 movdqa [%%ptr_expanded_keys+16*6], %%xkey1 ; store round keys in stack
154 movdqu %%xkey2, [%%ptr_key2 + 16*7]
155 aesenc %%xstate_tweak, %%xkey2 ; round 7 for tweak encryption
157 movdqu %%xkey1, [%%ptr_key1 + 16*7]
158 movdqa [%%ptr_expanded_keys+16*7], %%xkey1 ; store round keys in stack
161 movdqu %%xkey2, [%%ptr_key2 + 16*8]
162 aesenc %%xstate_tweak, %%xkey2 ; round 8 for tweak encryption
164 movdqu %%xkey1, [%%ptr_key1 + 16*8]
165 movdqa [%%ptr_expanded_keys+16*8], %%xkey1 ; store round keys in stack
168 movdqu %%xkey2, [%%ptr_key2 + 16*9]
169 aesenc %%xstate_tweak, %%xkey2 ; round 9 for tweak encryption
171 movdqu %%xkey1, [%%ptr_key1 + 16*9]
172 movdqa [%%ptr_expanded_keys+16*9], %%xkey1 ; store round keys in stack
175 movdqu %%xkey2, [%%ptr_key2 + 16*10]
176 aesenclast %%xstate_tweak, %%xkey2 ; round 10 for tweak encryption
178 movdqu %%xkey1, [%%ptr_key1 + 16*10]
179 movdqa [%%ptr_expanded_keys+16*10], %%xkey1 ; store round keys in stack
181 movdqa [TW], %%xstate_tweak ; Store the encrypted Tweak value
185 ; generate initial tweak values
186 ; load initial plaintext values
189 %define %%ST1 %1 ; state 1
190 %define %%ST2 %2 ; state 2
191 %define %%ST3 %3 ; state 3
192 %define %%ST4 %4 ; state 4
193 %define %%ST5 %5 ; state 5
194 %define %%ST6 %6 ; state 6
195 %define %%ST7 %7 ; state 7
196 %define %%ST8 %8 ; state 8
198 %define %%TW1 %9 ; tweak 1
199 %define %%TW2 %10 ; tweak 2
200 %define %%TW3 %11 ; tweak 3
201 %define %%TW4 %12 ; tweak 4
202 %define %%TW5 %13 ; tweak 5
203 %define %%TW6 %14 ; tweak 6
204 %define %%TW7 %15 ; tweak 7
206 %define %%num_initial_blocks %16
209 ; generate next Tweak values
210 movdqa %%TW1, [TW+16*0]
211 mov twtempl, [TW+8*0]
212 mov twtemph, [TW+8*1]
213 movdqu %%ST1, [ptr_plaintext+16*0]
214 %if (%%num_initial_blocks>=2)
215 xor ghash_poly_8b_temp, ghash_poly_8b_temp
218 cmovc ghash_poly_8b_temp, ghash_poly_8b
219 xor twtempl, ghash_poly_8b_temp
220 mov [TW+8*2], twtempl
221 mov [TW+8*3], twtemph;
222 movdqa %%TW2, [TW+16*1]
223 movdqu %%ST2, [ptr_plaintext+16*1]
225 %if (%%num_initial_blocks>=3)
226 xor ghash_poly_8b_temp, ghash_poly_8b_temp
229 cmovc ghash_poly_8b_temp, ghash_poly_8b
230 xor twtempl, ghash_poly_8b_temp
231 mov [TW+8*4], twtempl
232 mov [TW+8*5], twtemph;
233 movdqa %%TW3, [TW+16*2]
234 movdqu %%ST3, [ptr_plaintext+16*2]
236 %if (%%num_initial_blocks>=4)
237 xor ghash_poly_8b_temp, ghash_poly_8b_temp
240 cmovc ghash_poly_8b_temp, ghash_poly_8b
241 xor twtempl, ghash_poly_8b_temp
242 mov [TW+8*6], twtempl
243 mov [TW+8*7], twtemph;
244 movdqa %%TW4, [TW+16*3]
245 movdqu %%ST4, [ptr_plaintext+16*3]
247 %if (%%num_initial_blocks>=5)
248 xor ghash_poly_8b_temp, ghash_poly_8b_temp
251 cmovc ghash_poly_8b_temp, ghash_poly_8b
252 xor twtempl, ghash_poly_8b_temp
253 mov [TW+8*8], twtempl
254 mov [TW+8*9], twtemph;
255 movdqa %%TW5, [TW+16*4]
256 movdqu %%ST5, [ptr_plaintext+16*4]
258 %if (%%num_initial_blocks>=6)
259 xor ghash_poly_8b_temp, ghash_poly_8b_temp
262 cmovc ghash_poly_8b_temp, ghash_poly_8b
263 xor twtempl, ghash_poly_8b_temp
264 mov [TW+8*10], twtempl
265 mov [TW+8*11], twtemph;
266 movdqa %%TW6, [TW+16*5]
267 movdqu %%ST6, [ptr_plaintext+16*5]
269 %if (%%num_initial_blocks>=7)
270 xor ghash_poly_8b_temp, ghash_poly_8b_temp
273 cmovc ghash_poly_8b_temp, ghash_poly_8b
274 xor twtempl, ghash_poly_8b_temp
275 mov [TW+8*12], twtempl
276 mov [TW+8*13], twtemph;
277 movdqa %%TW7, [TW+16*6]
278 movdqu %%ST7, [ptr_plaintext+16*6]
286 ; encrypt initial blocks of AES
287 ; 1, 2, 3, 4, 5, 6 or 7 blocks are encrypted
288 ; next 8 Tweak values are generated
289 %macro encrypt_initial 18
290 %define %%ST1 %1 ; state 1
291 %define %%ST2 %2 ; state 2
292 %define %%ST3 %3 ; state 3
293 %define %%ST4 %4 ; state 4
294 %define %%ST5 %5 ; state 5
295 %define %%ST6 %6 ; state 6
296 %define %%ST7 %7 ; state 7
297 %define %%ST8 %8 ; state 8
299 %define %%TW1 %9 ; tweak 1
300 %define %%TW2 %10 ; tweak 2
301 %define %%TW3 %11 ; tweak 3
302 %define %%TW4 %12 ; tweak 4
303 %define %%TW5 %13 ; tweak 5
304 %define %%TW6 %14 ; tweak 6
305 %define %%TW7 %15 ; tweak 7
306 %define %%T0 %16 ; Temp register
307 %define %%num_blocks %17
308 ; %%num_blocks blocks encrypted
309 ; %%num_blocks can be 1, 2, 3, 4, 5, 6, 7
311 %define %%lt128 %18 ; less than 128 bytes
315 %if (%%num_blocks>=2)
318 %if (%%num_blocks>=3)
321 %if (%%num_blocks>=4)
324 %if (%%num_blocks>=5)
327 %if (%%num_blocks>=6)
330 %if (%%num_blocks>=7)
338 %if (%%num_blocks>=2)
341 %if (%%num_blocks>=3)
344 %if (%%num_blocks>=4)
347 %if (%%num_blocks>=5)
350 %if (%%num_blocks>=6)
353 %if (%%num_blocks>=7)
359 xor ghash_poly_8b_temp, ghash_poly_8b_temp
365 movdqa %%T0, [keys + 16*1]
367 %if (%%num_blocks>=2)
370 %if (%%num_blocks>=3)
373 %if (%%num_blocks>=4)
376 %if (%%num_blocks>=5)
379 %if (%%num_blocks>=6)
382 %if (%%num_blocks>=7)
386 cmovc ghash_poly_8b_temp, ghash_poly_8b
387 xor twtempl, ghash_poly_8b_temp
388 mov [TW + 8*0], twtempl ; next Tweak1 generated
389 mov [TW + 8*1], twtemph
390 xor ghash_poly_8b_temp, ghash_poly_8b_temp
394 movdqa %%T0, [keys + 16*2]
396 %if (%%num_blocks>=2)
399 %if (%%num_blocks>=3)
402 %if (%%num_blocks>=4)
405 %if (%%num_blocks>=5)
408 %if (%%num_blocks>=6)
411 %if (%%num_blocks>=7)
418 cmovc ghash_poly_8b_temp, ghash_poly_8b
419 xor twtempl, ghash_poly_8b_temp
420 mov [TW + 8*2], twtempl ; next Tweak2 generated
424 movdqa %%T0, [keys + 16*3]
426 %if (%%num_blocks>=2)
429 %if (%%num_blocks>=3)
432 %if (%%num_blocks>=4)
435 %if (%%num_blocks>=5)
438 %if (%%num_blocks>=6)
441 %if (%%num_blocks>=7)
445 mov [TW + 8*3], twtemph
446 xor ghash_poly_8b_temp, ghash_poly_8b_temp
449 cmovc ghash_poly_8b_temp, ghash_poly_8b
453 movdqa %%T0, [keys + 16*4]
455 %if (%%num_blocks>=2)
458 %if (%%num_blocks>=3)
461 %if (%%num_blocks>=4)
464 %if (%%num_blocks>=5)
467 %if (%%num_blocks>=6)
470 %if (%%num_blocks>=7)
475 xor twtempl, ghash_poly_8b_temp
476 mov [TW + 8*4], twtempl ; next Tweak3 generated
477 mov [TW + 8*5], twtemph
478 xor ghash_poly_8b_temp, ghash_poly_8b_temp
483 movdqa %%T0, [keys + 16*5]
485 %if (%%num_blocks>=2)
488 %if (%%num_blocks>=3)
491 %if (%%num_blocks>=4)
494 %if (%%num_blocks>=5)
497 %if (%%num_blocks>=6)
500 %if (%%num_blocks>=7)
506 cmovc ghash_poly_8b_temp, ghash_poly_8b
507 xor twtempl, ghash_poly_8b_temp
508 mov [TW + 8*6], twtempl ; next Tweak4 generated
509 mov [TW + 8*7], twtemph
513 movdqa %%T0, [keys + 16*6]
515 %if (%%num_blocks>=2)
518 %if (%%num_blocks>=3)
521 %if (%%num_blocks>=4)
524 %if (%%num_blocks>=5)
527 %if (%%num_blocks>=6)
530 %if (%%num_blocks>=7)
535 xor ghash_poly_8b_temp, ghash_poly_8b_temp
538 cmovc ghash_poly_8b_temp, ghash_poly_8b
539 xor twtempl, ghash_poly_8b_temp
540 mov [TW + 8*8], twtempl ; next Tweak5 generated
541 mov [TW + 8*9], twtemph
545 movdqa %%T0, [keys + 16*7]
547 %if (%%num_blocks>=2)
550 %if (%%num_blocks>=3)
553 %if (%%num_blocks>=4)
556 %if (%%num_blocks>=5)
559 %if (%%num_blocks>=6)
562 %if (%%num_blocks>=7)
567 xor ghash_poly_8b_temp, ghash_poly_8b_temp
570 cmovc ghash_poly_8b_temp, ghash_poly_8b
571 xor twtempl, ghash_poly_8b_temp
572 mov [TW + 8*10], twtempl ; next Tweak6 generated
573 mov [TW + 8*11], twtemph
576 movdqa %%T0, [keys + 16*8]
578 %if (%%num_blocks>=2)
581 %if (%%num_blocks>=3)
584 %if (%%num_blocks>=4)
587 %if (%%num_blocks>=5)
590 %if (%%num_blocks>=6)
593 %if (%%num_blocks>=7)
598 xor ghash_poly_8b_temp, ghash_poly_8b_temp
601 cmovc ghash_poly_8b_temp, ghash_poly_8b
602 xor twtempl, ghash_poly_8b_temp
603 mov [TW + 8*12], twtempl ; next Tweak7 generated
604 mov [TW + 8*13], twtemph
607 movdqa %%T0, [keys + 16*9]
609 %if (%%num_blocks>=2)
612 %if (%%num_blocks>=3)
615 %if (%%num_blocks>=4)
618 %if (%%num_blocks>=5)
621 %if (%%num_blocks>=6)
624 %if (%%num_blocks>=7)
629 xor ghash_poly_8b_temp, ghash_poly_8b_temp
632 cmovc ghash_poly_8b_temp, ghash_poly_8b
633 xor twtempl, ghash_poly_8b_temp
634 mov [TW + 8*14], twtempl ; next Tweak8 generated
635 mov [TW + 8*15], twtemph
642 movdqa %%T0, [keys + 16*10]
643 aesenclast %%ST1, %%T0
644 %if (%%num_blocks>=2)
645 aesenclast %%ST2, %%T0
647 %if (%%num_blocks>=3)
648 aesenclast %%ST3, %%T0
650 %if (%%num_blocks>=4)
651 aesenclast %%ST4, %%T0
653 %if (%%num_blocks>=5)
654 aesenclast %%ST5, %%T0
656 %if (%%num_blocks>=6)
657 aesenclast %%ST6, %%T0
659 %if (%%num_blocks>=7)
660 aesenclast %%ST7, %%T0
665 %if (%%num_blocks>=2)
668 %if (%%num_blocks>=3)
671 %if (%%num_blocks>=4)
674 %if (%%num_blocks>=5)
677 %if (%%num_blocks>=6)
680 %if (%%num_blocks>=7)
686 ; load next Tweak values
687 movdqa %%TW1, [TW + 16*0]
688 movdqa %%TW2, [TW + 16*1]
689 movdqa %%TW3, [TW + 16*2]
690 movdqa %%TW4, [TW + 16*3]
691 movdqa %%TW5, [TW + 16*4]
692 movdqa %%TW6, [TW + 16*5]
693 movdqa %%TW7, [TW + 16*6]
700 ; Encrypt 8 blocks in parallel
701 ; generate next 8 tweak values
702 %macro encrypt_by_eight 18
703 %define %%ST1 %1 ; state 1
704 %define %%ST2 %2 ; state 2
705 %define %%ST3 %3 ; state 3
706 %define %%ST4 %4 ; state 4
707 %define %%ST5 %5 ; state 5
708 %define %%ST6 %6 ; state 6
709 %define %%ST7 %7 ; state 7
710 %define %%ST8 %8 ; state 8
711 %define %%TW1 %9 ; tweak 1
712 %define %%TW2 %10 ; tweak 2
713 %define %%TW3 %11 ; tweak 3
714 %define %%TW4 %12 ; tweak 4
715 %define %%TW5 %13 ; tweak 5
716 %define %%TW6 %14 ; tweak 6
717 %define %%TW7 %15 ; tweak 7
718 %define %%TW8 %16 ; tweak 8
719 %define %%T0 %17 ; Temp register
720 %define %%last_eight %18
743 %if (0 == %%last_eight)
744 xor ghash_poly_8b_temp, ghash_poly_8b_temp
747 cmovc ghash_poly_8b_temp, ghash_poly_8b
750 movdqa %%T0, [keys + 16*1]
759 %if (0 == %%last_eight)
760 xor twtempl, ghash_poly_8b_temp
761 mov [TW + 8*0], twtempl
762 mov [TW + 8*1], twtemph
763 xor ghash_poly_8b_temp, ghash_poly_8b_temp
766 movdqa %%T0, [keys + 16*2]
775 %if (0 == %%last_eight)
778 cmovc ghash_poly_8b_temp, ghash_poly_8b
779 xor twtempl, ghash_poly_8b_temp
783 movdqa %%T0, [keys + 16*3]
792 %if (0 == %%last_eight)
793 mov [TW + 8*2], twtempl
794 mov [TW + 8*3], twtemph
795 xor ghash_poly_8b_temp, ghash_poly_8b_temp
799 movdqa %%T0, [keys + 16*4]
808 %if (0 == %%last_eight)
810 cmovc ghash_poly_8b_temp, ghash_poly_8b
811 xor twtempl, ghash_poly_8b_temp
812 mov [TW + 8*4], twtempl
815 movdqa %%T0, [keys + 16*5]
824 %if (0 == %%last_eight)
825 mov [TW + 8*5], twtemph
826 xor ghash_poly_8b_temp, ghash_poly_8b_temp
831 movdqa %%T0, [keys + 16*6]
840 %if (0 == %%last_eight)
841 cmovc ghash_poly_8b_temp, ghash_poly_8b
842 xor twtempl, ghash_poly_8b_temp
843 mov [TW + 8*6], twtempl
844 mov [TW + 8*7], twtemph
847 movdqa %%T0, [keys + 16*7]
856 %if (0 == %%last_eight)
857 xor ghash_poly_8b_temp, ghash_poly_8b_temp
860 cmovc ghash_poly_8b_temp, ghash_poly_8b
863 movdqa %%T0, [keys + 16*8]
872 %if (0 == %%last_eight)
873 xor twtempl, ghash_poly_8b_temp
874 mov [TW + 8*8], twtempl
875 mov [TW + 8*9], twtemph
876 xor ghash_poly_8b_temp, ghash_poly_8b_temp
879 movdqa %%T0, [keys + 16*9]
888 %if (0 == %%last_eight)
891 cmovc ghash_poly_8b_temp, ghash_poly_8b
892 xor twtempl, ghash_poly_8b_temp
895 %if (0 == %%last_eight)
896 mov [TW + 8*10], twtempl
897 mov [TW + 8*11], twtemph
898 xor ghash_poly_8b_temp, ghash_poly_8b_temp
902 %if (0 == %%last_eight)
904 cmovc ghash_poly_8b_temp, ghash_poly_8b
905 xor twtempl, ghash_poly_8b_temp
906 mov [TW + 8*12], twtempl
909 %if (0 == %%last_eight)
910 mov [TW + 8*13], twtemph
911 xor ghash_poly_8b_temp, ghash_poly_8b_temp
916 %if (0 == %%last_eight)
917 cmovc ghash_poly_8b_temp, ghash_poly_8b
918 xor twtempl, ghash_poly_8b_temp
919 ; mov [TW + 8*14], twtempl
920 ; mov [TW + 8*15], twtemph
923 movdqa %%T0, [keys + 16*10]
924 aesenclast %%ST1, %%T0
925 aesenclast %%ST2, %%T0
926 aesenclast %%ST3, %%T0
927 aesenclast %%ST4, %%T0
928 aesenclast %%ST5, %%T0
929 aesenclast %%ST6, %%T0
930 aesenclast %%ST7, %%T0
931 aesenclast %%ST8, %%T0
943 mov [TW + 8*14], twtempl
944 mov [TW + 8*15], twtemph
945 ; load next Tweak values
946 movdqa %%TW1, [TW + 16*0]
947 movdqa %%TW2, [TW + 16*1]
948 movdqa %%TW3, [TW + 16*2]
949 movdqa %%TW4, [TW + 16*3]
950 movdqa %%TW5, [TW + 16*4]
951 movdqa %%TW6, [TW + 16*5]
952 movdqa %%TW7, [TW + 16*6]
959 mk_global XTS_AES_128_enc_expanded_key_sse, function
960 XTS_AES_128_enc_expanded_key_sse:
963 sub rsp, VARIABLE_OFFSET
965 mov [_gpr + 8*0], rbx
966 %ifidn __OUTPUT_FORMAT__, win64
967 mov [_gpr + 8*1], rdi
968 mov [_gpr + 8*2], rsi
970 movdqa [_xmm + 16*0], xmm6
971 movdqa [_xmm + 16*1], xmm7
972 movdqa [_xmm + 16*2], xmm8
973 movdqa [_xmm + 16*3], xmm9
974 movdqa [_xmm + 16*4], xmm10
975 movdqa [_xmm + 16*5], xmm11
976 movdqa [_xmm + 16*6], xmm12
977 movdqa [_xmm + 16*7], xmm13
978 movdqa [_xmm + 16*8], xmm14
979 movdqa [_xmm + 16*9], xmm15
982 mov ghash_poly_8b, GHASH_POLY ; load 0x87 to ghash_poly_8b
985 movdqu xmm1, [T_val] ; read initial Tweak value
986 pxor xmm4, xmm4 ; for key expansion
987 encrypt_T xmm0, xmm1, xmm2, xmm3, xmm4, ptr_key2, ptr_key1, keys
990 %ifidn __OUTPUT_FORMAT__, win64
991 mov ptr_plaintext, [rsp + VARIABLE_OFFSET + 8*5] ; plaintext pointer
992 mov ptr_ciphertext, [rsp + VARIABLE_OFFSET + 8*6] ; ciphertext pointer
997 mov target_ptr_val, N_val
998 and target_ptr_val, -16 ; target_ptr_val = target_ptr_val - (target_ptr_val mod 16)
999 sub target_ptr_val, 128 ; adjust target_ptr_val because last 4 blocks will not be stitched with Tweak calculations
1000 jl _less_than_128_bytes
1002 add target_ptr_val, ptr_ciphertext
1007 jz _initial_num_blocks_is_0
1010 je _initial_num_blocks_is_4
1015 je _initial_num_blocks_is_6
1018 je _initial_num_blocks_is_5
1023 je _initial_num_blocks_is_3
1026 je _initial_num_blocks_is_2
1029 je _initial_num_blocks_is_1
1031 _initial_num_blocks_is_7:
1032 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 7
1033 add ptr_plaintext, 16*7
1034 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 7, 0
1036 movdqu [ptr_ciphertext+16*0], xmm1
1037 movdqu [ptr_ciphertext+16*1], xmm2
1038 movdqu [ptr_ciphertext+16*2], xmm3
1039 movdqu [ptr_ciphertext+16*3], xmm4
1040 movdqu [ptr_ciphertext+16*4], xmm5
1041 movdqu [ptr_ciphertext+16*5], xmm6
1042 movdqu [ptr_ciphertext+16*6], xmm7
1043 add ptr_ciphertext, 16*7
1045 cmp ptr_ciphertext, target_ptr_val
1049 _initial_num_blocks_is_6:
1050 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 6
1051 add ptr_plaintext, 16*6
1052 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 6, 0
1054 movdqu [ptr_ciphertext+16*0], xmm1
1055 movdqu [ptr_ciphertext+16*1], xmm2
1056 movdqu [ptr_ciphertext+16*2], xmm3
1057 movdqu [ptr_ciphertext+16*3], xmm4
1058 movdqu [ptr_ciphertext+16*4], xmm5
1059 movdqu [ptr_ciphertext+16*5], xmm6
1060 add ptr_ciphertext, 16*6
1062 cmp ptr_ciphertext, target_ptr_val
1066 _initial_num_blocks_is_5:
1067 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 5
1068 add ptr_plaintext, 16*5
1069 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 5, 0
1071 movdqu [ptr_ciphertext+16*0], xmm1
1072 movdqu [ptr_ciphertext+16*1], xmm2
1073 movdqu [ptr_ciphertext+16*2], xmm3
1074 movdqu [ptr_ciphertext+16*3], xmm4
1075 movdqu [ptr_ciphertext+16*4], xmm5
1076 add ptr_ciphertext, 16*5
1078 cmp ptr_ciphertext, target_ptr_val
1082 _initial_num_blocks_is_4:
1083 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 4
1084 add ptr_plaintext, 16*4
1085 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 4, 0
1087 movdqu [ptr_ciphertext+16*0], xmm1
1088 movdqu [ptr_ciphertext+16*1], xmm2
1089 movdqu [ptr_ciphertext+16*2], xmm3
1090 movdqu [ptr_ciphertext+16*3], xmm4
1091 add ptr_ciphertext, 16*4
1093 cmp ptr_ciphertext, target_ptr_val
1099 _initial_num_blocks_is_3:
1100 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 3
1101 add ptr_plaintext, 16*3
1102 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 3, 0
1104 movdqu [ptr_ciphertext+16*0], xmm1
1105 movdqu [ptr_ciphertext+16*1], xmm2
1106 movdqu [ptr_ciphertext+16*2], xmm3
1107 add ptr_ciphertext, 16*3
1109 cmp ptr_ciphertext, target_ptr_val
1113 _initial_num_blocks_is_2:
1114 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 2
1115 add ptr_plaintext, 16*2
1116 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 2, 0
1118 movdqu [ptr_ciphertext], xmm1
1119 movdqu [ptr_ciphertext+16], xmm2
1120 add ptr_ciphertext, 16*2
1122 cmp ptr_ciphertext, target_ptr_val
1127 _initial_num_blocks_is_1:
1128 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 1
1129 add ptr_plaintext, 16*1
1130 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 1, 0
1132 movdqu [ptr_ciphertext], xmm1
1133 add ptr_ciphertext, 16
1135 cmp ptr_ciphertext, target_ptr_val
1140 _initial_num_blocks_is_0:
1141 mov twtempl, [TW+8*0]
1142 mov twtemph, [TW+8*1]
1143 movdqa xmm9, [TW+16*0]
1145 xor ghash_poly_8b_temp, ghash_poly_8b_temp
1147 adc twtemph, twtemph
1148 cmovc ghash_poly_8b_temp, ghash_poly_8b
1149 xor twtempl, ghash_poly_8b_temp
1150 mov [TW+8*2], twtempl
1151 mov [TW+8*3], twtemph
1152 movdqa xmm10, [TW+16*1]
1154 xor ghash_poly_8b_temp, ghash_poly_8b_temp
1156 adc twtemph, twtemph
1157 cmovc ghash_poly_8b_temp, ghash_poly_8b
1158 xor twtempl, ghash_poly_8b_temp
1159 mov [TW+8*4], twtempl
1160 mov [TW+8*5], twtemph
1161 movdqa xmm11, [TW+16*2]
1164 xor ghash_poly_8b_temp, ghash_poly_8b_temp
1166 adc twtemph, twtemph
1167 cmovc ghash_poly_8b_temp, ghash_poly_8b
1168 xor twtempl, ghash_poly_8b_temp
1169 mov [TW+8*6], twtempl
1170 mov [TW+8*7], twtemph
1171 movdqa xmm12, [TW+16*3]
1174 xor ghash_poly_8b_temp, ghash_poly_8b_temp
1176 adc twtemph, twtemph
1177 cmovc ghash_poly_8b_temp, ghash_poly_8b
1178 xor twtempl, ghash_poly_8b_temp
1179 mov [TW+8*8], twtempl
1180 mov [TW+8*9], twtemph
1181 movdqa xmm13, [TW+16*4]
1183 xor ghash_poly_8b_temp, ghash_poly_8b_temp
1185 adc twtemph, twtemph
1186 cmovc ghash_poly_8b_temp, ghash_poly_8b
1187 xor twtempl, ghash_poly_8b_temp
1188 mov [TW+8*10], twtempl
1189 mov [TW+8*11], twtemph
1190 movdqa xmm14, [TW+16*5]
1192 xor ghash_poly_8b_temp, ghash_poly_8b_temp
1194 adc twtemph, twtemph
1195 cmovc ghash_poly_8b_temp, ghash_poly_8b
1196 xor twtempl, ghash_poly_8b_temp
1197 mov [TW+8*12], twtempl
1198 mov [TW+8*13], twtemph
1199 movdqa xmm15, [TW+16*6]
1201 xor ghash_poly_8b_temp, ghash_poly_8b_temp
1203 adc twtemph, twtemph
1204 cmovc ghash_poly_8b_temp, ghash_poly_8b
1205 xor twtempl, ghash_poly_8b_temp
1206 mov [TW+8*14], twtempl
1207 mov [TW+8*15], twtemph
1208 ;movdqa xmm16, [TW+16*7]
1210 cmp ptr_ciphertext, target_ptr_val
1214 movdqu xmm1, [ptr_plaintext+16*0]
1215 movdqu xmm2, [ptr_plaintext+16*1]
1216 movdqu xmm3, [ptr_plaintext+16*2]
1217 movdqu xmm4, [ptr_plaintext+16*3]
1218 movdqu xmm5, [ptr_plaintext+16*4]
1219 movdqu xmm6, [ptr_plaintext+16*5]
1220 movdqu xmm7, [ptr_plaintext+16*6]
1221 movdqu xmm8, [ptr_plaintext+16*7]
1223 add ptr_plaintext, 128
1225 encrypt_by_eight xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, [TW+16*7], xmm0, 0
1228 movdqu [ptr_ciphertext+16*0], xmm1
1229 movdqu [ptr_ciphertext+16*1], xmm2
1230 movdqu [ptr_ciphertext+16*2], xmm3
1231 movdqu [ptr_ciphertext+16*3], xmm4
1232 movdqu [ptr_ciphertext+16*4], xmm5
1233 movdqu [ptr_ciphertext+16*5], xmm6
1234 movdqu [ptr_ciphertext+16*6], xmm7
1235 movdqu [ptr_ciphertext+16*7], xmm8
1236 add ptr_ciphertext, 128
1238 cmp ptr_ciphertext, target_ptr_val
1243 movdqu xmm1, [ptr_plaintext+16*0]
1244 movdqu xmm2, [ptr_plaintext+16*1]
1245 movdqu xmm3, [ptr_plaintext+16*2]
1246 movdqu xmm4, [ptr_plaintext+16*3]
1247 movdqu xmm5, [ptr_plaintext+16*4]
1248 movdqu xmm6, [ptr_plaintext+16*5]
1249 movdqu xmm7, [ptr_plaintext+16*6]
1250 movdqu xmm8, [ptr_plaintext+16*7]
1251 encrypt_by_eight xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, [TW+16*7], xmm0, 1
1254 movdqu [ptr_ciphertext+16*0], xmm1
1255 movdqu [ptr_ciphertext+16*1], xmm2
1256 movdqu [ptr_ciphertext+16*2], xmm3
1257 movdqu [ptr_ciphertext+16*3], xmm4
1258 movdqu [ptr_ciphertext+16*4], xmm5
1259 movdqu [ptr_ciphertext+16*5], xmm6
1260 movdqu [ptr_ciphertext+16*6], xmm7
1263 and N_val, 15 ; N_val = N_val mod 16
1266 ; start cipher stealing
1268 ; generate next Tweak value
1269 xor ghash_poly_8b_temp, ghash_poly_8b_temp
1271 adc twtemph, twtemph
1272 cmovc ghash_poly_8b_temp, ghash_poly_8b
1273 xor twtempl, ghash_poly_8b_temp
1275 mov [TW + 8], twtemph
1279 ; shift xmm8 to the left by 16-N_val bytes
1280 lea twtempl, [pshufb_shf_table]
1281 movdqu xmm0, [twtempl+N_val]
1285 movdqu xmm3, [ptr_plaintext + 112 + N_val] ; state register is temporarily xmm3 to eliminate a move
1286 movdqu [ptr_ciphertext + 112 + N_val], xmm8
1288 ; shift xmm3 to the right by 16-N_val bytes
1289 lea twtempl, [pshufb_shf_table +16]
1291 movdqu xmm0, [twtempl]
1295 pblendvb xmm3, xmm2 ;xmm0 is implicit
1299 pxor xmm8, xmm3 ; state register is xmm8, instead of a move from xmm3 to xmm8, destination register of pxor instruction is swapped
1302 ;encrypt last block with cipher stealing
1303 pxor xmm8, [keys] ; ARK
1304 aesenc xmm8, [keys + 16*1] ; round 1
1305 aesenc xmm8, [keys + 16*2] ; round 2
1306 aesenc xmm8, [keys + 16*3] ; round 3
1307 aesenc xmm8, [keys + 16*4] ; round 4
1308 aesenc xmm8, [keys + 16*5] ; round 5
1309 aesenc xmm8, [keys + 16*6] ; round 6
1310 aesenc xmm8, [keys + 16*7] ; round 7
1311 aesenc xmm8, [keys + 16*8] ; round 8
1312 aesenc xmm8, [keys + 16*9] ; round 9
1313 aesenclast xmm8, [keys + 16*10] ; round 10
1319 ; store last ciphertext value
1320 movdqu [ptr_ciphertext+16*7], xmm8
1324 mov rbx, [_gpr + 8*0]
1325 %ifidn __OUTPUT_FORMAT__, win64
1326 mov rdi, [_gpr + 8*1]
1327 mov rsi, [_gpr + 8*2]
1330 movdqa xmm6, [_xmm + 16*0]
1331 movdqa xmm7, [_xmm + 16*1]
1332 movdqa xmm8, [_xmm + 16*2]
1333 movdqa xmm9, [_xmm + 16*3]
1334 movdqa xmm10, [_xmm + 16*4]
1335 movdqa xmm11, [_xmm + 16*5]
1336 movdqa xmm12, [_xmm + 16*6]
1337 movdqa xmm13, [_xmm + 16*7]
1338 movdqa xmm14, [_xmm + 16*8]
1339 movdqa xmm15, [_xmm + 16*9]
1342 add rsp, VARIABLE_OFFSET
1350 _less_than_128_bytes:
1370 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 7
1371 sub ptr_plaintext, 16*1
1372 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 7, 1
1374 movdqu [ptr_ciphertext+16*0], xmm1
1375 movdqu [ptr_ciphertext+16*1], xmm2
1376 movdqu [ptr_ciphertext+16*2], xmm3
1377 movdqu [ptr_ciphertext+16*3], xmm4
1378 movdqu [ptr_ciphertext+16*4], xmm5
1379 movdqu [ptr_ciphertext+16*5], xmm6
1381 sub ptr_ciphertext, 16*1
1384 and N_val, 15 ; N_val = N_val mod 16
1388 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 6
1389 sub ptr_plaintext, 16*2
1390 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 6, 1
1392 movdqu [ptr_ciphertext+16*0], xmm1
1393 movdqu [ptr_ciphertext+16*1], xmm2
1394 movdqu [ptr_ciphertext+16*2], xmm3
1395 movdqu [ptr_ciphertext+16*3], xmm4
1396 movdqu [ptr_ciphertext+16*4], xmm5
1398 sub ptr_ciphertext, 16*2
1401 and N_val, 15 ; N_val = N_val mod 16
1405 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 5
1406 sub ptr_plaintext, 16*3
1407 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 5, 1
1409 movdqu [ptr_ciphertext+16*0], xmm1
1410 movdqu [ptr_ciphertext+16*1], xmm2
1411 movdqu [ptr_ciphertext+16*2], xmm3
1412 movdqu [ptr_ciphertext+16*3], xmm4
1414 sub ptr_ciphertext, 16*3
1417 and N_val, 15 ; N_val = N_val mod 16
1421 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 4
1422 sub ptr_plaintext, 16*4
1423 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 4, 1
1425 movdqu [ptr_ciphertext+16*0], xmm1
1426 movdqu [ptr_ciphertext+16*1], xmm2
1427 movdqu [ptr_ciphertext+16*2], xmm3
1429 sub ptr_ciphertext, 16*4
1432 and N_val, 15 ; N_val = N_val mod 16
1436 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 3
1437 sub ptr_plaintext, 16*5
1438 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 3, 1
1440 movdqu [ptr_ciphertext+16*0], xmm1
1441 movdqu [ptr_ciphertext+16*1], xmm2
1443 sub ptr_ciphertext, 16*5
1446 and N_val, 15 ; N_val = N_val mod 16
1451 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 2
1452 sub ptr_plaintext, 16*6
1453 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 2, 1
1455 movdqu [ptr_ciphertext], xmm1
1457 sub ptr_ciphertext, 16*6
1460 and N_val, 15 ; N_val = N_val mod 16
1466 initialize xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, 1
1468 sub ptr_plaintext, 16*7
1469 encrypt_initial xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm0, 1, 1
1472 sub ptr_ciphertext, 16*7
1475 and N_val, 15 ; N_val = N_val mod 16
1483 ; use these values for shift constants for the pshufb instruction
1484 ; different alignments result in values as shown:
1485 ; dq 0x8887868584838281, 0x008f8e8d8c8b8a89 ; shl 15 (16-1) / shr1
1486 ; dq 0x8988878685848382, 0x01008f8e8d8c8b8a ; shl 14 (16-3) / shr2
1487 ; dq 0x8a89888786858483, 0x0201008f8e8d8c8b ; shl 13 (16-4) / shr3
1488 ; dq 0x8b8a898887868584, 0x030201008f8e8d8c ; shl 12 (16-4) / shr4
1489 ; dq 0x8c8b8a8988878685, 0x04030201008f8e8d ; shl 11 (16-5) / shr5
1490 ; dq 0x8d8c8b8a89888786, 0x0504030201008f8e ; shl 10 (16-6) / shr6
1491 ; dq 0x8e8d8c8b8a898887, 0x060504030201008f ; shl 9 (16-7) / shr7
1492 ; dq 0x8f8e8d8c8b8a8988, 0x0706050403020100 ; shl 8 (16-8) / shr8
1493 ; dq 0x008f8e8d8c8b8a89, 0x0807060504030201 ; shl 7 (16-9) / shr9
1494 ; dq 0x01008f8e8d8c8b8a, 0x0908070605040302 ; shl 6 (16-10) / shr10
1495 ; dq 0x0201008f8e8d8c8b, 0x0a09080706050403 ; shl 5 (16-11) / shr11
1496 ; dq 0x030201008f8e8d8c, 0x0b0a090807060504 ; shl 4 (16-12) / shr12
1497 ; dq 0x04030201008f8e8d, 0x0c0b0a0908070605 ; shl 3 (16-13) / shr13
1498 ; dq 0x0504030201008f8e, 0x0d0c0b0a09080706 ; shl 2 (16-14) / shr14
1499 ; dq 0x060504030201008f, 0x0e0d0c0b0a090807 ; shl 1 (16-15) / shr15
1500 dq 0x8786858483828100, 0x8f8e8d8c8b8a8988
1501 dq 0x0706050403020100, 0x000e0d0c0b0a0908
1504 dq 0x8080808080808080, 0x8080808080808080