#define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3
#define R_23(REG...) _R_23(REG, 1, 2, 3)
-#define ASM_BUG() ASSERT(0)
+#define ZFS_ASM_BUG() ASSERT(0)
const uint8_t gf_clmul_mod_lt[4*256][16];
uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
} v_t;
-#define PREFETCHNTA(ptr, offset) \
-{ \
- __asm( \
- "prefetchnta " #offset "(%[MEM])\n" \
- : : [MEM] "r" (ptr)); \
-}
-
-#define PREFETCH(ptr, offset) \
-{ \
- __asm( \
- "prefetcht0 " #offset "(%[MEM])\n" \
- : : [MEM] "r" (ptr)); \
-}
#define XOR_ACC(src, r...) \
{ \
: : [SRC] "r" (src)); \
break; \
default: \
- ASM_BUG(); \
+ ZFS_ASM_BUG(); \
} \
}
"pxor %" VR1(r) ", %" VR3(r)); \
break; \
default: \
- ASM_BUG(); \
+ ZFS_ASM_BUG(); \
} \
}
-#define ZERO(r...) \
-{ \
- switch (REG_CNT(r)) { \
- case 4: \
- __asm( \
- "pxor %" VR0(r) ", %" VR0(r) "\n" \
- "pxor %" VR1(r) ", %" VR1(r) "\n" \
- "pxor %" VR2(r) ", %" VR2(r) "\n" \
- "pxor %" VR3(r) ", %" VR3(r)); \
- break; \
- case 2: \
- __asm( \
- "pxor %" VR0(r) ", %" VR0(r) "\n" \
- "pxor %" VR1(r) ", %" VR1(r)); \
- break; \
- default: \
- ASM_BUG(); \
- } \
-}
+#define ZERO(r...) XOR(r, r)
#define COPY(r...) \
{ \
"movdqa %" VR1(r) ", %" VR3(r)); \
break; \
default: \
- ASM_BUG(); \
+ ZFS_ASM_BUG(); \
} \
}
: : [SRC] "r" (src)); \
break; \
default: \
- ASM_BUG(); \
+ ZFS_ASM_BUG(); \
} \
}
: : [DST] "r" (dst)); \
break; \
default: \
- ASM_BUG(); \
+ ZFS_ASM_BUG(); \
} \
}
"pxor %xmm13, %" VR1(r)); \
break; \
default: \
- ASM_BUG(); \
+ ZFS_ASM_BUG(); \
} \
}
_MUL2_x2(r); \
break; \
default: \
- ASM_BUG(); \
+ ZFS_ASM_BUG(); \
} \
}
[lt] "r" (gf_clmul_mod_lt[4*(c)])); \
break; \
default: \
- ASM_BUG(); \
+ ZFS_ASM_BUG(); \
} \
}
_MULx2(c, R_01(r)); \
break; \
default: \
- ASM_BUG(); \
+ ZFS_ASM_BUG(); \
} \
}
#define raidz_math_begin() kfpu_begin()
#define raidz_math_end() kfpu_end()
-#define GEN_P_DEFINE() {}
+
+#define SYN_STRIDE 4
+
+#define ZERO_STRIDE 4
+#define ZERO_DEFINE() {}
+#define ZERO_D 0, 1, 2, 3
+
+#define COPY_STRIDE 4
+#define COPY_DEFINE() {}
+#define COPY_D 0, 1, 2, 3
+
+#define ADD_STRIDE 4
+#define ADD_DEFINE() {}
+#define ADD_D 0, 1, 2, 3
+
+#define MUL_STRIDE 4
+#define MUL_DEFINE() {}
+#define MUL_D 0, 1, 2, 3
+
#define GEN_P_STRIDE 4
+#define GEN_P_DEFINE() {}
#define GEN_P_P 0, 1, 2, 3
-#define GEN_PQ_DEFINE() {}
#define GEN_PQ_STRIDE 4
+#define GEN_PQ_DEFINE() {}
#define GEN_PQ_D 0, 1, 2, 3
-#define GEN_PQ_P 4, 5, 6, 7
-#define GEN_PQ_Q 8, 9, 10, 11
+#define GEN_PQ_C 4, 5, 6, 7
+#define GEN_PQR_STRIDE 4
#define GEN_PQR_DEFINE() {}
-#define GEN_PQR_STRIDE 2
-#define GEN_PQR_D 0, 1
-#define GEN_PQR_P 2, 3
-#define GEN_PQR_Q 4, 5
-#define GEN_PQR_R 6, 7
+#define GEN_PQR_D 0, 1, 2, 3
+#define GEN_PQR_C 4, 5, 6, 7
-#define REC_P_DEFINE() {}
-#define REC_P_STRIDE 4
-#define REC_P_X 0, 1, 2, 3
+#define SYN_Q_DEFINE() {}
+#define SYN_Q_D 0, 1, 2, 3
+#define SYN_Q_X 4, 5, 6, 7
-#define REC_Q_DEFINE() {}
-#define REC_Q_STRIDE 4
-#define REC_Q_X 0, 1, 2, 3
+#define SYN_R_DEFINE() {}
+#define SYN_R_D 0, 1, 2, 3
+#define SYN_R_X 4, 5, 6, 7
-#define REC_R_DEFINE() {}
-#define REC_R_STRIDE 4
-#define REC_R_X 0, 1, 2, 3
+#define SYN_PQ_DEFINE() {}
+#define SYN_PQ_D 0, 1, 2, 3
+#define SYN_PQ_X 4, 5, 6, 7
-#define REC_PQ_DEFINE() {}
#define REC_PQ_STRIDE 2
+#define REC_PQ_DEFINE() {}
#define REC_PQ_X 0, 1
#define REC_PQ_Y 2, 3
-#define REC_PQ_D 4, 5
+#define REC_PQ_T 4, 5
+
+#define SYN_PR_DEFINE() {}
+#define SYN_PR_D 0, 1, 2, 3
+#define SYN_PR_X 4, 5, 6, 7
-#define REC_PR_DEFINE() {}
#define REC_PR_STRIDE 2
+#define REC_PR_DEFINE() {}
#define REC_PR_X 0, 1
#define REC_PR_Y 2, 3
-#define REC_PR_D 4, 5
+#define REC_PR_T 4, 5
+
+#define SYN_QR_DEFINE() {}
+#define SYN_QR_D 0, 1, 2, 3
+#define SYN_QR_X 4, 5, 6, 7
-#define REC_QR_DEFINE() {}
#define REC_QR_STRIDE 2
+#define REC_QR_DEFINE() {}
#define REC_QR_X 0, 1
#define REC_QR_Y 2, 3
-#define REC_QR_D 4, 5
+#define REC_QR_T 4, 5
+
+#define SYN_PQR_DEFINE() {}
+#define SYN_PQR_D 0, 1, 2, 3
+#define SYN_PQR_X 4, 5, 6, 7
-#define REC_PQR_DEFINE() {}
#define REC_PQR_STRIDE 2
+#define REC_PQR_DEFINE() {}
#define REC_PQR_X 0, 1
#define REC_PQR_Y 2, 3
#define REC_PQR_Z 4, 5
-#define REC_PQR_D 6, 7
#define REC_PQR_XS 6, 7
#define REC_PQR_YS 8, 9
static boolean_t
raidz_will_ssse3_work(void)
{
-/* ABD Bringup -- vector code not ready */
-#if 1
- return (B_FALSE);
-#else
return (zfs_sse_available() && zfs_sse2_available() &&
zfs_ssse3_available());
-#endif
}
const raidz_impl_ops_t vdev_raidz_ssse3_impl = {
#endif /* defined(__x86_64) && defined(HAVE_SSSE3) */
-#if defined(__x86_64) && (defined(HAVE_SSSE3) || defined(HAVE_AVX2))
-
+#if defined(__x86_64)
+#if defined(HAVE_SSSE3) || defined(HAVE_AVX2) || defined(HAVE_AVX512BW)
+/* BEGIN CSTYLED */
const uint8_t
-__attribute__((aligned(256))) gf_clmul_mod_lt[4*256][16] = {
+__attribute__((aligned(256))) gf_clmul_mod_lt[4*256][16] =
+{
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
{ 0x00, 0xff, 0xfe, 0x01, 0xfc, 0x03, 0x02, 0xfd,
0xf8, 0x07, 0x06, 0xf9, 0x04, 0xfb, 0xfa, 0x05 }
};
-
-#endif /* defined(__x86_64) && (defined(HAVE_SSSE3) || defined(HAVE_AVX2)) */
+/* END CSTYLED */
+#endif /* defined(HAVE_SSSE3) || defined(HAVE_AVX2) || defined(HAVE_AVX512BW) */
+#endif /* defined(__x86_64) */
projects / mirror_zfs.git / blobdiff