module/zfs/vdev_raidz_math_avx512bw.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (C) 2016 Romain Dolbeau. All rights reserved.
  23  */
  24
  25 #include <sys/isa_defs.h>
  26
  27 #if defined(__x86_64) && defined(HAVE_AVX512BW)
  28
  29 #include <sys/types.h>
  30 #include <linux/simd_x86.h>
  31
  32 #define __asm __asm__ __volatile__
  33
  34 #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
  35 #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)
  36
  37 #define VR0_(REG, ...) "zmm"#REG
  38 #define VR1_(_1, REG, ...) "zmm"#REG
  39 #define VR2_(_1, _2, REG, ...) "zmm"#REG
  40 #define VR3_(_1, _2, _3, REG, ...) "zmm"#REG
  41 #define VR4_(_1, _2, _3, _4, REG, ...) "zmm"#REG
  42 #define VR5_(_1, _2, _3, _4, _5, REG, ...) "zmm"#REG
  43 #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "zmm"#REG
  44 #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "zmm"#REG
  45
  46 #define VR0(r...) VR0_(r)
  47 #define VR1(r...) VR1_(r)
  48 #define VR2(r...) VR2_(r, 1)
  49 #define VR3(r...) VR3_(r, 1, 2)
  50 #define VR4(r...) VR4_(r, 1, 2)
  51 #define VR5(r...) VR5_(r, 1, 2, 3)
  52 #define VR6(r...) VR6_(r, 1, 2, 3, 4)
  53 #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5)
  54
  55 #define R_01(REG1, REG2, ...) REG1, REG2
  56 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3
  57 #define R_23(REG...) _R_23(REG, 1, 2, 3)
  58
  59 #define ASM_BUG()       ASSERT(0)
  60
  61 extern const uint8_t gf_clmul_mod_lt[4*256][16];
  62
  63 #define ELEM_SIZE 64
  64
  65 typedef struct v {
  66         uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
  67 } v_t;
  68
  69 #define PREFETCHNTA(ptr, offset)                                        \
  70 {                                                                       \
  71         __asm(                                                          \
  72             "prefetchnta " #offset "(%[MEM])\n"                         \
  73             : : [MEM] "r" (ptr));                                       \
  74 }
  75
  76 #define PREFETCH(ptr, offset)                                           \
  77 {                                                                       \
  78         __asm(                                                          \
  79             "prefetcht0 " #offset "(%[MEM])\n"                          \
  80             : : [MEM] "r" (ptr));                                       \
  81 }
  82
  83 #define XOR_ACC(src, r...)                                              \
  84 {                                                                       \
  85         switch (REG_CNT(r)) {                                           \
  86         case 4:                                                         \
  87                 __asm(                                                  \
  88                     "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"  \
  89                     "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"  \
  90                     "vpxorq 0x80(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n"  \
  91                     "vpxorq 0xc0(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n"  \
  92                     : : [SRC] "r" (src));                               \
  93                 break;                                                  \
  94         case 2:                                                         \
  95                 __asm(                                                  \
  96                     "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"  \
  97                     "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"  \
  98                     : : [SRC] "r" (src));                               \
  99                 break;                                                  \
 100         default:                                                        \
 101                 ASM_BUG();                                              \
 102         }                                                               \
 103 }
 104
 105 #define XOR(r...)                                                       \
 106 {                                                                       \
 107         switch (REG_CNT(r)) {                                           \
 108         case 8:                                                         \
 109                 __asm(                                                  \
 110                     "vpxorq %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n"     \
 111                     "vpxorq %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n"     \
 112                     "vpxorq %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n"     \
 113                     "vpxorq %" VR3(r) ", %" VR7(r)", %" VR7(r));        \
 114                 break;                                                  \
 115         case 4:                                                         \
 116                 __asm(                                                  \
 117                     "vpxorq %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n"     \
 118                     "vpxorq %" VR1(r) ", %" VR3(r)", %" VR3(r));        \
 119                 break;                                                  \
 120         default:                                                        \
 121                 ASM_BUG();                                              \
 122         }                                                               \
 123 }
 124
 125 #define ZERO(r...)                                                      \
 126 {                                                                       \
 127         switch (REG_CNT(r)) {                                           \
 128         case 4:                                                         \
 129                 __asm(                                                  \
 130                     "vpxorq %" VR0(r) ", %" VR0(r)", %" VR0(r) "\n"     \
 131                     "vpxorq %" VR1(r) ", %" VR1(r)", %" VR1(r) "\n"     \
 132                     "vpxorq %" VR2(r) ", %" VR2(r)", %" VR2(r) "\n"     \
 133                     "vpxorq %" VR3(r) ", %" VR3(r)", %" VR3(r));        \
 134                 break;                                                  \
 135         case 2:                                                         \
 136                 __asm(                                                  \
 137                     "vpxorq %" VR0(r) ", %" VR0(r)", %" VR0(r) "\n"     \
 138                     "vpxorq %" VR1(r) ", %" VR1(r)", %" VR1(r));        \
 139                 break;                                                  \
 140         default:                                                        \
 141                 ASM_BUG();                                              \
 142         }                                                               \
 143 }
 144
 145 #define COPY(r...)                                                      \
 146 {                                                                       \
 147         switch (REG_CNT(r)) {                                           \
 148         case 8:                                                         \
 149                 __asm(                                                  \
 150                     "vmovdqa64 %" VR0(r) ", %" VR4(r) "\n"              \
 151                     "vmovdqa64 %" VR1(r) ", %" VR5(r) "\n"              \
 152                     "vmovdqa64 %" VR2(r) ", %" VR6(r) "\n"              \
 153                     "vmovdqa64 %" VR3(r) ", %" VR7(r));                 \
 154                 break;                                                  \
 155         case 4:                                                         \
 156                 __asm(                                                  \
 157                     "vmovdqa64 %" VR0(r) ", %" VR2(r) "\n"              \
 158                     "vmovdqa64 %" VR1(r) ", %" VR3(r));                 \
 159                 break;                                                  \
 160         default:                                                        \
 161                 ASM_BUG();                                              \
 162         }                                                               \
 163 }
 164
 165 #define LOAD(src, r...)                                                 \
 166 {                                                                       \
 167         switch (REG_CNT(r)) {                                           \
 168         case 4:                                                         \
 169                 __asm(                                                  \
 170                     "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n"            \
 171                     "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n"            \
 172                     "vmovdqa64 0x80(%[SRC]), %%" VR2(r) "\n"            \
 173                     "vmovdqa64 0xc0(%[SRC]), %%" VR3(r) "\n"            \
 174                     : : [SRC] "r" (src));                               \
 175                 break;                                                  \
 176         case 2:                                                         \
 177                 __asm(                                                  \
 178                     "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n"            \
 179                     "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n"            \
 180                     : : [SRC] "r" (src));                               \
 181                 break;                                                  \
 182         default:                                                        \
 183                 ASM_BUG();                                              \
 184         }                                                               \
 185 }
 186
 187 #define STORE(dst, r...)                                                \
 188 {                                                                       \
 189         switch (REG_CNT(r)) {                                           \
 190         case 4:                                                         \
 191                 __asm(                                                  \
 192                     "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n"            \
 193                     "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n"            \
 194                     "vmovdqa64 %%" VR2(r) ", 0x80(%[DST])\n"            \
 195                     "vmovdqa64 %%" VR3(r) ", 0xc0(%[DST])\n"            \
 196                     : : [DST] "r" (dst));                               \
 197                 break;                                                  \
 198         case 2:                                                         \
 199                 __asm(                                                  \
 200                     "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n"            \
 201                     "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n"            \
 202                     : : [DST] "r" (dst));                               \
 203                 break;                                                  \
 204         default:                                                        \
 205                 ASM_BUG();                                              \
 206         }                                                               \
 207 }
 208
 209 #define FLUSH()                                                         \
 210 {                                                                       \
 211         __asm("vzeroupper");                                            \
 212 }
 213
 214 #define MUL2_SETUP()                                                    \
 215 {                                                                       \
 216         __asm("vmovq %0,   %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d));        \
 217         __asm("vpbroadcastq %xmm14, %zmm14");                           \
 218         __asm("vmovq %0,   %%xmm13" :: "r"(0x8080808080808080));        \
 219         __asm("vpbroadcastq %xmm13, %zmm13");                           \
 220         __asm("vmovq %0,   %%xmm12" :: "r"(0xfefefefefefefefe));        \
 221         __asm("vpbroadcastq %xmm12, %zmm12");                           \
 222         __asm("vpxorq       %zmm15, %zmm15 ,%zmm15");                   \
 223 }
 224
 225 #define _MUL2(r...)                                                     \
 226 {                                                                       \
 227         switch  (REG_CNT(r)) {                                          \
 228         case 2:                                                         \
 229                 __asm(                                                  \
 230                     "vpandq   %" VR0(r)", %zmm13, %zmm10\n"             \
 231                     "vpandq   %" VR1(r)", %zmm13, %zmm11\n"             \
 232                     "vpsrlq   $7, %zmm10, %zmm8\n"                      \
 233                     "vpsrlq   $7, %zmm11, %zmm9\n"                      \
 234                     "vpsllq   $1, %zmm10, %zmm10\n"                     \
 235                     "vpsllq   $1, %zmm11, %zmm11\n"                     \
 236                     "vpsubq   %zmm8, %zmm10, %zmm10\n"                  \
 237                     "vpsubq   %zmm9, %zmm11, %zmm11\n"                  \
 238                     "vpsllq   $1, %" VR0(r)", %" VR0(r) "\n"            \
 239                     "vpsllq   $1, %" VR1(r)", %" VR1(r) "\n"            \
 240                     "vpandq   %zmm10, %zmm14, %zmm10\n"                 \
 241                     "vpandq   %zmm11, %zmm14, %zmm11\n"                 \
 242                     "vpternlogd $0x6c,%zmm12, %zmm10, %" VR0(r) "\n"    \
 243                     "vpternlogd $0x6c,%zmm12, %zmm11, %" VR1(r));       \
 244                 break;                                                  \
 245         default:                                                        \
 246                 ASM_BUG();                                              \
 247         }                                                               \
 248 }
 249
 250 #define MUL2(r...)                                                      \
 251 {                                                                       \
 252         switch (REG_CNT(r)) {                                           \
 253         case 4:                                                         \
 254             _MUL2(R_01(r));                                             \
 255             _MUL2(R_23(r));                                             \
 256             break;                                                      \
 257         case 2:                                                         \
 258             _MUL2(r);                                                   \
 259             break;                                                      \
 260         default:                                                        \
 261                 ASM_BUG();                                              \
 262         }                                                               \
 263 }
 264
 265 #define MUL4(r...)                                                      \
 266 {                                                                       \
 267         MUL2(r);                                                        \
 268         MUL2(r);                                                        \
 269 }
 270
 271 #define _0f             "zmm15"
 272 #define _as             "zmm14"
 273 #define _bs             "zmm13"
 274 #define _ltmod          "zmm12"
 275 #define _ltmul          "zmm11"
 276 #define _ta             "zmm10"
 277 #define _tb             "zmm15"
 278
 279 static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F;
 280
 281 #define _MULx2(c, r...)                                                 \
 282 {                                                                       \
 283         switch (REG_CNT(r)) {                                           \
 284         case 2:                                                         \
 285                 __asm(                                                  \
 286                     "vpbroadcastb (%[mask]), %%" _0f "\n"               \
 287                     /* upper bits */                                    \
 288                     "vbroadcasti32x4 0x00(%[lt]), %%" _ltmod "\n"       \
 289                     "vbroadcasti32x4 0x10(%[lt]), %%" _ltmul "\n"       \
 290                                                                         \
 291                     "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n"             \
 292                     "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n"             \
 293                     "vpandq %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n"    \
 294                     "vpandq %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n"    \
 295                     "vpandq %%" _0f ", %%" _as ", %%" _as "\n"          \
 296                     "vpandq %%" _0f ", %%" _bs ", %%" _bs "\n"          \
 297                                                                         \
 298                     "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n"      \
 299                     "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n"      \
 300                     "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n"      \
 301                     "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n"      \
 302                     /* lower bits */                                    \
 303                     "vbroadcasti32x4 0x20(%[lt]), %%" _ltmod "\n"       \
 304                     "vbroadcasti32x4 0x30(%[lt]), %%" _ltmul "\n"       \
 305                                                                         \
 306                     "vpxorq %%" _ta ", %%" _as ", %%" _as "\n"          \
 307                     "vpxorq %%" _tb ", %%" _bs ", %%" _bs "\n"          \
 308                                                                         \
 309                     "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n"   \
 310                     "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n"   \
 311                     "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
 312                     "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
 313                                                                         \
 314                     "vpxorq %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n"    \
 315                     "vpxorq %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n"    \
 316                     "vpxorq %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n"    \
 317                     "vpxorq %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n"    \
 318                     : : [mask] "r" (&_mul_mask),                        \
 319                     [lt] "r" (gf_clmul_mod_lt[4*(c)]));                 \
 320                 break;                                                  \
 321         default:                                                        \
 322                 ASM_BUG();                                              \
 323         }                                                               \
 324 }
 325
 326 #define MUL(c, r...)                                                    \
 327 {                                                                       \
 328         switch (REG_CNT(r)) {                                           \
 329         case 4:                                                         \
 330                 _MULx2(c, R_01(r));                                     \
 331                 _MULx2(c, R_23(r));                                     \
 332                 break;                                                  \
 333         case 2:                                                         \
 334                 _MULx2(c, R_01(r));                                     \
 335                 break;                                                  \
 336         default:                                                        \
 337                 ASM_BUG();                                              \
 338         }                                                               \
 339 }
 340
 341 #define raidz_math_begin()      kfpu_begin()
 342 #define raidz_math_end()                                                \
 343 {                                                                       \
 344         FLUSH();                                                        \
 345         kfpu_end();                                                     \
 346 }
 347
 348 #define GEN_P_DEFINE()          {}
 349 #define GEN_P_STRIDE            4
 350 #define GEN_P_P                 0, 1, 2, 3
 351
 352 #define GEN_PQ_DEFINE()         {}
 353 #define GEN_PQ_STRIDE           4
 354 #define GEN_PQ_D                0, 1, 2, 3
 355 #define GEN_PQ_P                4, 5, 6, 7
 356 #define GEN_PQ_Q                20, 21, 22, 23
 357
 358 #define GEN_PQR_DEFINE()        {}
 359 #define GEN_PQR_STRIDE          2
 360 #define GEN_PQR_D               0, 1
 361 #define GEN_PQR_P               2, 3
 362 #define GEN_PQR_Q               4, 5
 363 #define GEN_PQR_R               6, 7
 364
 365 #define REC_P_DEFINE()          {}
 366 #define REC_P_STRIDE            4
 367 #define REC_P_X                 0, 1, 2, 3
 368
 369 #define REC_Q_DEFINE()          {}
 370 #define REC_Q_STRIDE            4
 371 #define REC_Q_X                 0, 1, 2, 3
 372
 373 #define REC_R_DEFINE()          {}
 374 #define REC_R_STRIDE            4
 375 #define REC_R_X                 0, 1, 2, 3
 376
 377 #define REC_PQ_DEFINE()         {}
 378 #define REC_PQ_STRIDE           4
 379 #define REC_PQ_X                0, 1, 2, 3
 380 #define REC_PQ_Y                4, 5, 6, 7
 381 #define REC_PQ_D                20, 21, 22, 23
 382
 383 #define REC_PR_DEFINE()         {}
 384 #define REC_PR_STRIDE           4
 385 #define REC_PR_X                0, 1, 2, 3
 386 #define REC_PR_Y                4, 5, 6, 7
 387 #define REC_PR_D                20, 21, 22, 23
 388
 389 #define REC_QR_DEFINE()         {}
 390 #define REC_QR_STRIDE           4
 391 #define REC_QR_X                0, 1, 2, 3
 392 #define REC_QR_Y                4, 5, 6, 7
 393 #define REC_QR_D                20, 21, 22, 23
 394
 395 #define REC_PQR_DEFINE()        {}
 396 #define REC_PQR_STRIDE          2
 397 #define REC_PQR_X               0, 1
 398 #define REC_PQR_Y               2, 3
 399 #define REC_PQR_Z               4, 5
 400 #define REC_PQR_D               6, 7
 401 #define REC_PQR_XS              6, 7
 402 #define REC_PQR_YS              8, 9
 403
 404
 405 #include <sys/vdev_raidz_impl.h>
 406 #include "vdev_raidz_math_impl.h"
 407
 408 DEFINE_GEN_METHODS(avx512bw);
 409 DEFINE_REC_METHODS(avx512bw);
 410
 411 static boolean_t
 412 raidz_will_avx512bw_work(void)
 413 {
 414         return (zfs_avx_available() &&
 415                 zfs_avx512f_available() &&
 416                 zfs_avx512bw_available());
 417 }
 418
 419 const raidz_impl_ops_t vdev_raidz_avx512bw_impl = {
 420         .init = NULL,
 421         .fini = NULL,
 422         .gen = RAIDZ_GEN_METHODS(avx512bw),
 423         .rec = RAIDZ_REC_METHODS(avx512bw),
 424         .is_supported = &raidz_will_avx512bw_work,
 425         .name = "avx512bw"
 426 };
 427
 428 #endif /* defined(__x86_64) && defined(HAVE_AVX512BW) */