module/zfs/vdev_raidz_math_avx2.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
  23  */
  24 #include <sys/isa_defs.h>
  25
  26 #if defined(__x86_64) && defined(HAVE_AVX2)
  27
  28 #include <sys/types.h>
  29 #include <sys/simd.h>
  30
  31 #ifdef __linux__
  32 #define __asm __asm__ __volatile__
  33 #endif
  34
  35 #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
  36 #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)
  37
  38 #define VR0_(REG, ...) "ymm"#REG
  39 #define VR1_(_1, REG, ...) "ymm"#REG
  40 #define VR2_(_1, _2, REG, ...) "ymm"#REG
  41 #define VR3_(_1, _2, _3, REG, ...) "ymm"#REG
  42 #define VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG
  43 #define VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG
  44 #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG
  45 #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG
  46
  47 #define VR0(r...) VR0_(r)
  48 #define VR1(r...) VR1_(r)
  49 #define VR2(r...) VR2_(r, 1)
  50 #define VR3(r...) VR3_(r, 1, 2)
  51 #define VR4(r...) VR4_(r, 1, 2)
  52 #define VR5(r...) VR5_(r, 1, 2, 3)
  53 #define VR6(r...) VR6_(r, 1, 2, 3, 4)
  54 #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5)
  55
  56 #define R_01(REG1, REG2, ...) REG1, REG2
  57 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3
  58 #define R_23(REG...) _R_23(REG, 1, 2, 3)
  59
  60 #define ZFS_ASM_BUG()   ASSERT(0)
  61
  62 extern const uint8_t gf_clmul_mod_lt[4*256][16];
  63
  64 #define ELEM_SIZE 32
  65
  66 typedef struct v {
  67         uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
  68 } v_t;
  69
  70
  71 #define XOR_ACC(src, r...)                                              \
  72 {                                                                       \
  73         switch (REG_CNT(r)) {                                           \
  74         case 4:                                                         \
  75                 __asm(                                                  \
  76                     "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"   \
  77                     "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"   \
  78                     "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n"   \
  79                     "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n"   \
  80                     : : [SRC] "r" (src));                               \
  81                 break;                                                  \
  82         case 2:                                                         \
  83                 __asm(                                                  \
  84                     "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"   \
  85                     "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"   \
  86                     : : [SRC] "r" (src));                               \
  87                 break;                                                  \
  88         default:                                                        \
  89                 ZFS_ASM_BUG();                                          \
  90         }                                                               \
  91 }
  92
  93 #define XOR(r...)                                                       \
  94 {                                                                       \
  95         switch (REG_CNT(r)) {                                           \
  96         case 8:                                                         \
  97                 __asm(                                                  \
  98                     "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n"      \
  99                     "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n"      \
 100                     "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n"      \
 101                     "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r));         \
 102                 break;                                                  \
 103         case 4:                                                         \
 104                 __asm(                                                  \
 105                     "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n"      \
 106                     "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r));         \
 107                 break;                                                  \
 108         default:                                                        \
 109                 ZFS_ASM_BUG();                                          \
 110         }                                                               \
 111 }
 112
 113 #define ZERO(r...)      XOR(r, r)
 114
 115 #define COPY(r...)                                                      \
 116 {                                                                       \
 117         switch (REG_CNT(r)) {                                           \
 118         case 8:                                                         \
 119                 __asm(                                                  \
 120                     "vmovdqa %" VR0(r) ", %" VR4(r) "\n"                \
 121                     "vmovdqa %" VR1(r) ", %" VR5(r) "\n"                \
 122                     "vmovdqa %" VR2(r) ", %" VR6(r) "\n"                \
 123                     "vmovdqa %" VR3(r) ", %" VR7(r));                   \
 124                 break;                                                  \
 125         case 4:                                                         \
 126                 __asm(                                                  \
 127                     "vmovdqa %" VR0(r) ", %" VR2(r) "\n"                \
 128                     "vmovdqa %" VR1(r) ", %" VR3(r));                   \
 129                 break;                                                  \
 130         default:                                                        \
 131                 ZFS_ASM_BUG();                                          \
 132         }                                                               \
 133 }
 134
 135 #define LOAD(src, r...)                                                 \
 136 {                                                                       \
 137         switch (REG_CNT(r)) {                                           \
 138         case 4:                                                         \
 139                 __asm(                                                  \
 140                     "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n"              \
 141                     "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n"              \
 142                     "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n"              \
 143                     "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n"              \
 144                     : : [SRC] "r" (src));                               \
 145                 break;                                                  \
 146         case 2:                                                         \
 147                 __asm(                                                  \
 148                     "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n"              \
 149                     "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n"              \
 150                     : : [SRC] "r" (src));                               \
 151                 break;                                                  \
 152         default:                                                        \
 153                 ZFS_ASM_BUG();                                          \
 154         }                                                               \
 155 }
 156
 157 #define STORE(dst, r...)                                                \
 158 {                                                                       \
 159         switch (REG_CNT(r)) {                                           \
 160         case 4:                                                         \
 161                 __asm(                                                  \
 162                     "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n"              \
 163                     "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n"              \
 164                     "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n"              \
 165                     "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n"              \
 166                     : : [DST] "r" (dst));                               \
 167                 break;                                                  \
 168         case 2:                                                         \
 169                 __asm(                                                  \
 170                     "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n"              \
 171                     "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n"              \
 172                     : : [DST] "r" (dst));                               \
 173                 break;                                                  \
 174         default:                                                        \
 175                 ZFS_ASM_BUG();                                          \
 176         }                                                               \
 177 }
 178
 179 #define FLUSH()                                                         \
 180 {                                                                       \
 181         __asm("vzeroupper");                                            \
 182 }
 183
 184 #define MUL2_SETUP()                                                    \
 185 {                                                                       \
 186         __asm("vmovq %0,   %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d));        \
 187         __asm("vpbroadcastq %xmm14, %ymm14");                           \
 188         __asm("vpxor        %ymm15, %ymm15 ,%ymm15");                   \
 189 }
 190
 191 #define _MUL2(r...)                                                     \
 192 {                                                                       \
 193         switch  (REG_CNT(r)) {                                          \
 194         case 2:                                                         \
 195                 __asm(                                                  \
 196                     "vpcmpgtb %" VR0(r)", %ymm15,     %ymm12\n"         \
 197                     "vpcmpgtb %" VR1(r)", %ymm15,     %ymm13\n"         \
 198                     "vpaddb   %" VR0(r)", %" VR0(r)", %" VR0(r) "\n"    \
 199                     "vpaddb   %" VR1(r)", %" VR1(r)", %" VR1(r) "\n"    \
 200                     "vpand    %ymm14,     %ymm12,     %ymm12\n"         \
 201                     "vpand    %ymm14,     %ymm13,     %ymm13\n"         \
 202                     "vpxor    %ymm12,     %" VR0(r)", %" VR0(r) "\n"    \
 203                     "vpxor    %ymm13,     %" VR1(r)", %" VR1(r));       \
 204                 break;                                                  \
 205         default:                                                        \
 206                 ZFS_ASM_BUG();                                          \
 207         }                                                               \
 208 }
 209
 210 #define MUL2(r...)                                                      \
 211 {                                                                       \
 212         switch (REG_CNT(r)) {                                           \
 213         case 4:                                                         \
 214             _MUL2(R_01(r));                                             \
 215             _MUL2(R_23(r));                                             \
 216             break;                                                      \
 217         case 2:                                                         \
 218             _MUL2(r);                                                   \
 219             break;                                                      \
 220         default:                                                        \
 221                 ZFS_ASM_BUG();                                          \
 222         }                                                               \
 223 }
 224
 225 #define MUL4(r...)                                                      \
 226 {                                                                       \
 227         MUL2(r);                                                        \
 228         MUL2(r);                                                        \
 229 }
 230
 231 #define _0f             "ymm15"
 232 #define _as             "ymm14"
 233 #define _bs             "ymm13"
 234 #define _ltmod          "ymm12"
 235 #define _ltmul          "ymm11"
 236 #define _ta             "ymm10"
 237 #define _tb             "ymm15"
 238
 239 static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F;
 240
 241 #define _MULx2(c, r...)                                                 \
 242 {                                                                       \
 243         switch (REG_CNT(r)) {                                           \
 244         case 2:                                                         \
 245                 __asm(                                                  \
 246                     "vpbroadcastb (%[mask]), %%" _0f "\n"               \
 247                     /* upper bits */                                    \
 248                     "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n"        \
 249                     "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n"        \
 250                                                                         \
 251                     "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n"             \
 252                     "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n"             \
 253                     "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n"     \
 254                     "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n"     \
 255                     "vpand %%" _0f ", %%" _as ", %%" _as "\n"           \
 256                     "vpand %%" _0f ", %%" _bs ", %%" _bs "\n"           \
 257                                                                         \
 258                     "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n"      \
 259                     "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n"      \
 260                     "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n"      \
 261                     "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n"      \
 262                     /* lower bits */                                    \
 263                     "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n"        \
 264                     "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n"        \
 265                                                                         \
 266                     "vpxor %%" _ta ", %%" _as ", %%" _as "\n"           \
 267                     "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n"           \
 268                                                                         \
 269                     "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n"   \
 270                     "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n"   \
 271                     "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
 272                     "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
 273                                                                         \
 274                     "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n"     \
 275                     "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n"     \
 276                     "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n"     \
 277                     "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n"     \
 278                     : : [mask] "r" (&_mul_mask),                        \
 279                     [lt] "r" (gf_clmul_mod_lt[4*(c)]));                 \
 280                 break;                                                  \
 281         default:                                                        \
 282                 ZFS_ASM_BUG();                                          \
 283         }                                                               \
 284 }
 285
 286 #define MUL(c, r...)                                                    \
 287 {                                                                       \
 288         switch (REG_CNT(r)) {                                           \
 289         case 4:                                                         \
 290                 _MULx2(c, R_01(r));                                     \
 291                 _MULx2(c, R_23(r));                                     \
 292                 break;                                                  \
 293         case 2:                                                         \
 294                 _MULx2(c, R_01(r));                                     \
 295                 break;                                                  \
 296         default:                                                        \
 297                 ZFS_ASM_BUG();                                          \
 298         }                                                               \
 299 }
 300
 301 #define raidz_math_begin()      kfpu_begin()
 302 #define raidz_math_end()                                                \
 303 {                                                                       \
 304         FLUSH();                                                        \
 305         kfpu_end();                                                     \
 306 }
 307
 308
 309 #define SYN_STRIDE              4
 310
 311 #define ZERO_STRIDE             4
 312 #define ZERO_DEFINE()           {}
 313 #define ZERO_D                  0, 1, 2, 3
 314
 315 #define COPY_STRIDE             4
 316 #define COPY_DEFINE()           {}
 317 #define COPY_D                  0, 1, 2, 3
 318
 319 #define ADD_STRIDE              4
 320 #define ADD_DEFINE()            {}
 321 #define ADD_D                   0, 1, 2, 3
 322
 323 #define MUL_STRIDE              4
 324 #define MUL_DEFINE()            {}
 325 #define MUL_D                   0, 1, 2, 3
 326
 327 #define GEN_P_STRIDE            4
 328 #define GEN_P_DEFINE()          {}
 329 #define GEN_P_P                 0, 1, 2, 3
 330
 331 #define GEN_PQ_STRIDE           4
 332 #define GEN_PQ_DEFINE()         {}
 333 #define GEN_PQ_D                0, 1, 2, 3
 334 #define GEN_PQ_C                4, 5, 6, 7
 335
 336 #define GEN_PQR_STRIDE          4
 337 #define GEN_PQR_DEFINE()        {}
 338 #define GEN_PQR_D               0, 1, 2, 3
 339 #define GEN_PQR_C               4, 5, 6, 7
 340
 341 #define SYN_Q_DEFINE()          {}
 342 #define SYN_Q_D                 0, 1, 2, 3
 343 #define SYN_Q_X                 4, 5, 6, 7
 344
 345 #define SYN_R_DEFINE()          {}
 346 #define SYN_R_D                 0, 1, 2, 3
 347 #define SYN_R_X                 4, 5, 6, 7
 348
 349 #define SYN_PQ_DEFINE()         {}
 350 #define SYN_PQ_D                0, 1, 2, 3
 351 #define SYN_PQ_X                4, 5, 6, 7
 352
 353 #define REC_PQ_STRIDE           2
 354 #define REC_PQ_DEFINE()         {}
 355 #define REC_PQ_X                0, 1
 356 #define REC_PQ_Y                2, 3
 357 #define REC_PQ_T                4, 5
 358
 359 #define SYN_PR_DEFINE()         {}
 360 #define SYN_PR_D                0, 1, 2, 3
 361 #define SYN_PR_X                4, 5, 6, 7
 362
 363 #define REC_PR_STRIDE           2
 364 #define REC_PR_DEFINE()         {}
 365 #define REC_PR_X                0, 1
 366 #define REC_PR_Y                2, 3
 367 #define REC_PR_T                4, 5
 368
 369 #define SYN_QR_DEFINE()         {}
 370 #define SYN_QR_D                0, 1, 2, 3
 371 #define SYN_QR_X                4, 5, 6, 7
 372
 373 #define REC_QR_STRIDE           2
 374 #define REC_QR_DEFINE()         {}
 375 #define REC_QR_X                0, 1
 376 #define REC_QR_Y                2, 3
 377 #define REC_QR_T                4, 5
 378
 379 #define SYN_PQR_DEFINE()        {}
 380 #define SYN_PQR_D               0, 1, 2, 3
 381 #define SYN_PQR_X               4, 5, 6, 7
 382
 383 #define REC_PQR_STRIDE          2
 384 #define REC_PQR_DEFINE()        {}
 385 #define REC_PQR_X               0, 1
 386 #define REC_PQR_Y               2, 3
 387 #define REC_PQR_Z               4, 5
 388 #define REC_PQR_XS              6, 7
 389 #define REC_PQR_YS              8, 9
 390
 391
 392 #include <sys/vdev_raidz_impl.h>
 393 #include "vdev_raidz_math_impl.h"
 394
 395 DEFINE_GEN_METHODS(avx2);
 396 DEFINE_REC_METHODS(avx2);
 397
 398 static boolean_t
 399 raidz_will_avx2_work(void)
 400 {
 401         return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available());
 402 }
 403
 404 const raidz_impl_ops_t vdev_raidz_avx2_impl = {
 405         .init = NULL,
 406         .fini = NULL,
 407         .gen = RAIDZ_GEN_METHODS(avx2),
 408         .rec = RAIDZ_REC_METHODS(avx2),
 409         .is_supported = &raidz_will_avx2_work,
 410         .name = "avx2"
 411 };
 412
 413 #endif /* defined(__x86_64) && defined(HAVE_AVX2) */