X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=module%2Fzfs%2Fvdev_raidz_math_ssse3.c;h=047a48d544f1fed804da46baac3a44d29258c918;hb=a20f43b51ba87d4932f897e23cf9af181d4d1374;hp=81f1b9a077b9f87f4815eb4d7e6ea5c37de5fb89;hpb=a6255b7fce400d485a0e87cbe369aa0ed7dc5dc4;p=mirror_zfs.git diff --git a/module/zfs/vdev_raidz_math_ssse3.c b/module/zfs/vdev_raidz_math_ssse3.c index 81f1b9a07..047a48d54 100644 --- a/module/zfs/vdev_raidz_math_ssse3.c +++ b/module/zfs/vdev_raidz_math_ssse3.c @@ -56,7 +56,7 @@ #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 #define R_23(REG...) _R_23(REG, 1, 2, 3) -#define ASM_BUG() ASSERT(0) +#define ZFS_ASM_BUG() ASSERT(0) const uint8_t gf_clmul_mod_lt[4*256][16]; @@ -66,19 +66,6 @@ typedef struct v { uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); } v_t; -#define PREFETCHNTA(ptr, offset) \ -{ \ - __asm( \ - "prefetchnta " #offset "(%[MEM])\n" \ - : : [MEM] "r" (ptr)); \ -} - -#define PREFETCH(ptr, offset) \ -{ \ - __asm( \ - "prefetcht0 " #offset "(%[MEM])\n" \ - : : [MEM] "r" (ptr)); \ -} #define XOR_ACC(src, r...) \ { \ @@ -98,7 +85,7 @@ typedef struct v { : : [SRC] "r" (src)); \ break; \ default: \ - ASM_BUG(); \ + ZFS_ASM_BUG(); \ } \ } @@ -118,29 +105,11 @@ typedef struct v { "pxor %" VR1(r) ", %" VR3(r)); \ break; \ default: \ - ASM_BUG(); \ + ZFS_ASM_BUG(); \ } \ } -#define ZERO(r...) \ -{ \ - switch (REG_CNT(r)) { \ - case 4: \ - __asm( \ - "pxor %" VR0(r) ", %" VR0(r) "\n" \ - "pxor %" VR1(r) ", %" VR1(r) "\n" \ - "pxor %" VR2(r) ", %" VR2(r) "\n" \ - "pxor %" VR3(r) ", %" VR3(r)); \ - break; \ - case 2: \ - __asm( \ - "pxor %" VR0(r) ", %" VR0(r) "\n" \ - "pxor %" VR1(r) ", %" VR1(r)); \ - break; \ - default: \ - ASM_BUG(); \ - } \ -} +#define ZERO(r...) XOR(r, r) #define COPY(r...) \ { \ @@ -158,7 +127,7 @@ typedef struct v { "movdqa %" VR1(r) ", %" VR3(r)); \ break; \ default: \ - ASM_BUG(); \ + ZFS_ASM_BUG(); \ } \ } @@ -180,7 +149,7 @@ typedef struct v { : : [SRC] "r" (src)); \ break; \ default: \ - ASM_BUG(); \ + ZFS_ASM_BUG(); \ } \ } @@ -202,7 +171,7 @@ typedef struct v { : : [DST] "r" (dst)); \ break; \ default: \ - ASM_BUG(); \ + ZFS_ASM_BUG(); \ } \ } @@ -231,7 +200,7 @@ typedef struct v { "pxor %xmm13, %" VR1(r)); \ break; \ default: \ - ASM_BUG(); \ + ZFS_ASM_BUG(); \ } \ } @@ -246,7 +215,7 @@ typedef struct v { _MUL2_x2(r); \ break; \ default: \ - ASM_BUG(); \ + ZFS_ASM_BUG(); \ } \ } @@ -315,7 +284,7 @@ typedef struct v { [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ break; \ default: \ - ASM_BUG(); \ + ZFS_ASM_BUG(); \ } \ } @@ -330,66 +299,93 @@ typedef struct v { _MULx2(c, R_01(r)); \ break; \ default: \ - ASM_BUG(); \ + ZFS_ASM_BUG(); \ } \ } #define raidz_math_begin() kfpu_begin() #define raidz_math_end() kfpu_end() -#define GEN_P_DEFINE() {} + +#define SYN_STRIDE 4 + +#define ZERO_STRIDE 4 +#define ZERO_DEFINE() {} +#define ZERO_D 0, 1, 2, 3 + +#define COPY_STRIDE 4 +#define COPY_DEFINE() {} +#define COPY_D 0, 1, 2, 3 + +#define ADD_STRIDE 4 +#define ADD_DEFINE() {} +#define ADD_D 0, 1, 2, 3 + +#define MUL_STRIDE 4 +#define MUL_DEFINE() {} +#define MUL_D 0, 1, 2, 3 + #define GEN_P_STRIDE 4 +#define GEN_P_DEFINE() {} #define GEN_P_P 0, 1, 2, 3 -#define GEN_PQ_DEFINE() {} #define GEN_PQ_STRIDE 4 +#define GEN_PQ_DEFINE() {} #define GEN_PQ_D 0, 1, 2, 3 -#define GEN_PQ_P 4, 5, 6, 7 -#define GEN_PQ_Q 8, 9, 10, 11 +#define GEN_PQ_C 4, 5, 6, 7 +#define GEN_PQR_STRIDE 4 #define GEN_PQR_DEFINE() {} -#define GEN_PQR_STRIDE 2 -#define GEN_PQR_D 0, 1 -#define GEN_PQR_P 2, 3 -#define GEN_PQR_Q 4, 5 -#define GEN_PQR_R 6, 7 +#define GEN_PQR_D 0, 1, 2, 3 +#define GEN_PQR_C 4, 5, 6, 7 -#define REC_P_DEFINE() {} -#define REC_P_STRIDE 4 -#define REC_P_X 0, 1, 2, 3 +#define SYN_Q_DEFINE() {} +#define SYN_Q_D 0, 1, 2, 3 +#define SYN_Q_X 4, 5, 6, 7 -#define REC_Q_DEFINE() {} -#define REC_Q_STRIDE 4 -#define REC_Q_X 0, 1, 2, 3 +#define SYN_R_DEFINE() {} +#define SYN_R_D 0, 1, 2, 3 +#define SYN_R_X 4, 5, 6, 7 -#define REC_R_DEFINE() {} -#define REC_R_STRIDE 4 -#define REC_R_X 0, 1, 2, 3 +#define SYN_PQ_DEFINE() {} +#define SYN_PQ_D 0, 1, 2, 3 +#define SYN_PQ_X 4, 5, 6, 7 -#define REC_PQ_DEFINE() {} #define REC_PQ_STRIDE 2 +#define REC_PQ_DEFINE() {} #define REC_PQ_X 0, 1 #define REC_PQ_Y 2, 3 -#define REC_PQ_D 4, 5 +#define REC_PQ_T 4, 5 + +#define SYN_PR_DEFINE() {} +#define SYN_PR_D 0, 1, 2, 3 +#define SYN_PR_X 4, 5, 6, 7 -#define REC_PR_DEFINE() {} #define REC_PR_STRIDE 2 +#define REC_PR_DEFINE() {} #define REC_PR_X 0, 1 #define REC_PR_Y 2, 3 -#define REC_PR_D 4, 5 +#define REC_PR_T 4, 5 + +#define SYN_QR_DEFINE() {} +#define SYN_QR_D 0, 1, 2, 3 +#define SYN_QR_X 4, 5, 6, 7 -#define REC_QR_DEFINE() {} #define REC_QR_STRIDE 2 +#define REC_QR_DEFINE() {} #define REC_QR_X 0, 1 #define REC_QR_Y 2, 3 -#define REC_QR_D 4, 5 +#define REC_QR_T 4, 5 + +#define SYN_PQR_DEFINE() {} +#define SYN_PQR_D 0, 1, 2, 3 +#define SYN_PQR_X 4, 5, 6, 7 -#define REC_PQR_DEFINE() {} #define REC_PQR_STRIDE 2 +#define REC_PQR_DEFINE() {} #define REC_PQR_X 0, 1 #define REC_PQR_Y 2, 3 #define REC_PQR_Z 4, 5 -#define REC_PQR_D 6, 7 #define REC_PQR_XS 6, 7 #define REC_PQR_YS 8, 9 @@ -403,13 +399,8 @@ DEFINE_REC_METHODS(ssse3); static boolean_t raidz_will_ssse3_work(void) { -/* ABD Bringup -- vector code not ready */ -#if 1 - return (B_FALSE); -#else return (zfs_sse_available() && zfs_sse2_available() && zfs_ssse3_available()); -#endif } const raidz_impl_ops_t vdev_raidz_ssse3_impl = { @@ -424,10 +415,12 @@ const raidz_impl_ops_t vdev_raidz_ssse3_impl = { #endif /* defined(__x86_64) && defined(HAVE_SSSE3) */ -#if defined(__x86_64) && (defined(HAVE_SSSE3) || defined(HAVE_AVX2)) - +#if defined(__x86_64) +#if defined(HAVE_SSSE3) || defined(HAVE_AVX2) || defined(HAVE_AVX512BW) +/* BEGIN CSTYLED */ const uint8_t -__attribute__((aligned(256))) gf_clmul_mod_lt[4*256][16] = { +__attribute__((aligned(256))) gf_clmul_mod_lt[4*256][16] = +{ { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -2477,5 +2470,6 @@ __attribute__((aligned(256))) gf_clmul_mod_lt[4*256][16] = { { 0x00, 0xff, 0xfe, 0x01, 0xfc, 0x03, 0x02, 0xfd, 0xf8, 0x07, 0x06, 0xf9, 0x04, 0xfb, 0xfa, 0x05 } }; - -#endif /* defined(__x86_64) && (defined(HAVE_SSSE3) || defined(HAVE_AVX2)) */ +/* END CSTYLED */ +#endif /* defined(HAVE_SSSE3) || defined(HAVE_AVX2) || defined(HAVE_AVX512BW) */ +#endif /* defined(__x86_64) */