[mirror_zfs.git] / include / linux / simd_x86.h

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (C) 2016 Gvozden Neskovic <neskovic@compeng.uni-frankfurt.de>.
 */

/*
 * USER API:
 *
 * Kernel fpu methods:
 * 	kfpu_begin()
 * 	kfpu_end()
 *
 * SIMD support:
 *
 * Following functions should be called to determine whether CPU feature
 * is supported. All functions are usable in kernel and user space.
 * If a SIMD algorithm is using more than one instruction set
 * all relevant feature test functions should be called.
 *
 * Supported features:
 * 	zfs_sse_available()
 * 	zfs_sse2_available()
 * 	zfs_sse3_available()
 * 	zfs_ssse3_available()
 * 	zfs_sse4_1_available()
 * 	zfs_sse4_2_available()
 *
 * 	zfs_avx_available()
 * 	zfs_avx2_available()
 *
 * 	zfs_bmi1_available()
 * 	zfs_bmi2_available()
 *
 * 	zfs_avx512f_available()
 * 	zfs_avx512cd_available()
 * 	zfs_avx512er_available()
 * 	zfs_avx512pf_available()
 * 	zfs_avx512bw_available()
 * 	zfs_avx512dq_available()
 * 	zfs_avx512vl_available()
 * 	zfs_avx512ifma_available()
 * 	zfs_avx512vbmi_available()
 *
 * NOTE(AVX-512VL):	If using AVX-512 instructions with 128Bit registers
 * 			also add zfs_avx512vl_available() to feature check.
 */

#ifndef _SIMD_X86_H
#define	_SIMD_X86_H

#include <sys/isa_defs.h>

/* only for __x86 */
#if defined(__x86)

#include <sys/types.h>

#if defined(_KERNEL)
#include <asm/cpufeature.h>
#else
#include <cpuid.h>
#endif

#if defined(_KERNEL)
#if defined(HAVE_UNDERSCORE_KERNEL_FPU)
#include <asm/fpu/api.h>
#include <asm/fpu/internal.h>
#define	kfpu_begin()		\
{							\
	preempt_disable();		\
	__kernel_fpu_begin();	\
}
#define	kfpu_end()			\
{							\
	__kernel_fpu_end();		\
	preempt_enable();		\
}
#elif defined(HAVE_KERNEL_FPU)
#include <asm/i387.h>
#include <asm/xcr.h>
#define	kfpu_begin()	kernel_fpu_begin()
#define	kfpu_end()		kernel_fpu_end()
#else
/* Kernel doesn't export any kernel_fpu_* functions */
#include <asm/fpu/internal.h>	/* For kernel xgetbv() */
#define	kfpu_begin() 	panic("This code should never run")
#define	kfpu_end() 	panic("This code should never run")
#endif /* defined(HAVE_KERNEL_FPU) */

#else
/*
 * fpu dummy methods for userspace
 */
#define	kfpu_begin() 	do {} while (0)
#define	kfpu_end() 		do {} while (0)
#endif /* defined(_KERNEL) */

/*
 * CPUID feature tests for user-space. Linux kernel provides an interface for
 * CPU feature testing.
 */
#if !defined(_KERNEL)

/*
 * x86 registers used implicitly by CPUID
 */
typedef enum cpuid_regs {
	EAX = 0,
	EBX,
	ECX,
	EDX,
	CPUID_REG_CNT = 4
} cpuid_regs_t;

/*
 * List of instruction sets identified by CPUID
 */
typedef enum cpuid_inst_sets {
	SSE = 0,
	SSE2,
	SSE3,
	SSSE3,
	SSE4_1,
	SSE4_2,
	OSXSAVE,
	AVX,
	AVX2,
	BMI1,
	BMI2,
	AVX512F,
	AVX512CD,
	AVX512DQ,
	AVX512BW,
	AVX512IFMA,
	AVX512VBMI,
	AVX512PF,
	AVX512ER,
	AVX512VL,
	AES,
	PCLMULQDQ
} cpuid_inst_sets_t;

/*
 * Instruction set descriptor.
 */
typedef struct cpuid_feature_desc {
	uint32_t leaf;		/* CPUID leaf */
	uint32_t subleaf;	/* CPUID sub-leaf */
	uint32_t flag;		/* bit mask of the feature */
	cpuid_regs_t reg;	/* which CPUID return register to test */
} cpuid_feature_desc_t;

#define	_AVX512F_BIT		(1U << 16)
#define	_AVX512CD_BIT		(_AVX512F_BIT | (1U << 28))
#define	_AVX512DQ_BIT		(_AVX512F_BIT | (1U << 17))
#define	_AVX512BW_BIT		(_AVX512F_BIT | (1U << 30))
#define	_AVX512IFMA_BIT		(_AVX512F_BIT | (1U << 21))
#define	_AVX512VBMI_BIT		(1U << 1) /* AVX512F_BIT is on another leaf  */
#define	_AVX512PF_BIT		(_AVX512F_BIT | (1U << 26))
#define	_AVX512ER_BIT		(_AVX512F_BIT | (1U << 27))
#define	_AVX512VL_BIT		(1U << 31) /* if used also check other levels */
#define	_AES_BIT		(1U << 25)
#define	_PCLMULQDQ_BIT		(1U << 1)

/*
 * Descriptions of supported instruction sets
 */
static const cpuid_feature_desc_t cpuid_features[] = {
	[SSE]		= {1U, 0U,	1U << 25, 	EDX	},
	[SSE2]		= {1U, 0U,	1U << 26,	EDX	},
	[SSE3]		= {1U, 0U,	1U << 0,	ECX	},
	[SSSE3]		= {1U, 0U,	1U << 9,	ECX	},
	[SSE4_1]	= {1U, 0U,	1U << 19,	ECX	},
	[SSE4_2]	= {1U, 0U,	1U << 20,	ECX	},
	[OSXSAVE]	= {1U, 0U,	1U << 27,	ECX	},
	[AVX]		= {1U, 0U,	1U << 28,	ECX	},
	[AVX2]		= {7U, 0U,	1U << 5,	EBX	},
	[BMI1]		= {7U, 0U,	1U << 3,	EBX	},
	[BMI2]		= {7U, 0U,	1U << 8,	EBX	},
	[AVX512F]	= {7U, 0U, _AVX512F_BIT,	EBX	},
	[AVX512CD]	= {7U, 0U, _AVX512CD_BIT,	EBX	},
	[AVX512DQ]	= {7U, 0U, _AVX512DQ_BIT,	EBX	},
	[AVX512BW]	= {7U, 0U, _AVX512BW_BIT,	EBX	},
	[AVX512IFMA]	= {7U, 0U, _AVX512IFMA_BIT,	EBX	},
	[AVX512VBMI]	= {7U, 0U, _AVX512VBMI_BIT,	ECX	},
	[AVX512PF]	= {7U, 0U, _AVX512PF_BIT,	EBX	},
	[AVX512ER]	= {7U, 0U, _AVX512ER_BIT,	EBX	},
	[AVX512VL]	= {7U, 0U, _AVX512ER_BIT,	EBX	},
	[AES]		= {1U, 0U, _AES_BIT,		ECX	},
	[PCLMULQDQ]	= {1U, 0U, _PCLMULQDQ_BIT,	ECX	},
};

/*
 * Check if OS supports AVX and AVX2 by checking XCR0
 * Only call this function if CPUID indicates that AVX feature is
 * supported by the CPU, otherwise it might be an illegal instruction.
 */
static inline uint64_t
xgetbv(uint32_t index)
{
	uint32_t eax, edx;
	/* xgetbv - instruction byte code */
	__asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0"
	    : "=a" (eax), "=d" (edx)
	    : "c" (index));

	return ((((uint64_t)edx)<<32) | (uint64_t)eax);
}

/*
 * Check if CPU supports a feature
 */
static inline boolean_t
__cpuid_check_feature(const cpuid_feature_desc_t *desc)
{
	uint32_t r[CPUID_REG_CNT];

	if (__get_cpuid_max(0, NULL) >= desc->leaf) {
		/*
		 * __cpuid_count is needed to properly check
		 * for AVX2. It is a macro, so return parameters
		 * are passed by value.
		 */
		__cpuid_count(desc->leaf, desc->subleaf,
		    r[EAX], r[EBX], r[ECX], r[EDX]);
		return ((r[desc->reg] & desc->flag) == desc->flag);
	}
	return (B_FALSE);
}

#define	CPUID_FEATURE_CHECK(name, id)				\
static inline boolean_t						\
__cpuid_has_ ## name(void)					\
{								\
	return (__cpuid_check_feature(&cpuid_features[id]));	\
}

/*
 * Define functions for user-space CPUID features testing
 */
CPUID_FEATURE_CHECK(sse, SSE);
CPUID_FEATURE_CHECK(sse2, SSE2);
CPUID_FEATURE_CHECK(sse3, SSE3);
CPUID_FEATURE_CHECK(ssse3, SSSE3);
CPUID_FEATURE_CHECK(sse4_1, SSE4_1);
CPUID_FEATURE_CHECK(sse4_2, SSE4_2);
CPUID_FEATURE_CHECK(avx, AVX);
CPUID_FEATURE_CHECK(avx2, AVX2);
CPUID_FEATURE_CHECK(osxsave, OSXSAVE);
CPUID_FEATURE_CHECK(bmi1, BMI1);
CPUID_FEATURE_CHECK(bmi2, BMI2);
CPUID_FEATURE_CHECK(avx512f, AVX512F);
CPUID_FEATURE_CHECK(avx512cd, AVX512CD);
CPUID_FEATURE_CHECK(avx512dq, AVX512DQ);
CPUID_FEATURE_CHECK(avx512bw, AVX512BW);
CPUID_FEATURE_CHECK(avx512ifma, AVX512IFMA);
CPUID_FEATURE_CHECK(avx512vbmi, AVX512VBMI);
CPUID_FEATURE_CHECK(avx512pf, AVX512PF);
CPUID_FEATURE_CHECK(avx512er, AVX512ER);
CPUID_FEATURE_CHECK(avx512vl, AVX512VL);
CPUID_FEATURE_CHECK(aes, AES);
CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ);

#endif /* !defined(_KERNEL) */


/*
 * Detect register set support
 */
static inline boolean_t
__simd_state_enabled(const uint64_t state)
{
	boolean_t has_osxsave;
	uint64_t xcr0;

#if defined(_KERNEL)
#if defined(X86_FEATURE_OSXSAVE) && defined(KERNEL_EXPORTS_X86_FPU)
	has_osxsave = !!boot_cpu_has(X86_FEATURE_OSXSAVE);
#else
	has_osxsave = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_osxsave = __cpuid_has_osxsave();
#endif

	if (!has_osxsave)
		return (B_FALSE);

	xcr0 = xgetbv(0);
	return ((xcr0 & state) == state);
}

#define	_XSTATE_SSE_AVX		(0x2 | 0x4)
#define	_XSTATE_AVX512		(0xE0 | _XSTATE_SSE_AVX)

#define	__ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX)
#define	__zmm_enabled() __simd_state_enabled(_XSTATE_AVX512)


/*
 * Check if SSE instruction set is available
 */
static inline boolean_t
zfs_sse_available(void)
{
#if defined(_KERNEL)
#if defined(KERNEL_EXPORTS_X86_FPU)
	return (!!boot_cpu_has(X86_FEATURE_XMM));
#else
	return (B_FALSE);
#endif
#elif !defined(_KERNEL)
	return (__cpuid_has_sse());
#endif
}

/*
 * Check if SSE2 instruction set is available
 */
static inline boolean_t
zfs_sse2_available(void)
{
#if defined(_KERNEL)
#if defined(KERNEL_EXPORTS_X86_FPU)
	return (!!boot_cpu_has(X86_FEATURE_XMM2));
#else
	return (B_FALSE);
#endif
#elif !defined(_KERNEL)
	return (__cpuid_has_sse2());
#endif
}

/*
 * Check if SSE3 instruction set is available
 */
static inline boolean_t
zfs_sse3_available(void)
{
#if defined(_KERNEL)
#if defined(KERNEL_EXPORTS_X86_FPU)
	return (!!boot_cpu_has(X86_FEATURE_XMM3));
#else
	return (B_FALSE);
#endif
#elif !defined(_KERNEL)
	return (__cpuid_has_sse3());
#endif
}

/*
 * Check if SSSE3 instruction set is available
 */
static inline boolean_t
zfs_ssse3_available(void)
{
#if defined(_KERNEL)
#if defined(KERNEL_EXPORTS_X86_FPU)
	return (!!boot_cpu_has(X86_FEATURE_SSSE3));
#else
	return (B_FALSE);
#endif
#elif !defined(_KERNEL)
	return (__cpuid_has_ssse3());
#endif
}

/*
 * Check if SSE4.1 instruction set is available
 */
static inline boolean_t
zfs_sse4_1_available(void)
{
#if defined(_KERNEL)
#if defined(KERNEL_EXPORTS_X86_FPU)
	return (!!boot_cpu_has(X86_FEATURE_XMM4_1));
#else
	return (B_FALSE);
#endif
#elif !defined(_KERNEL)
	return (__cpuid_has_sse4_1());
#endif
}

/*
 * Check if SSE4.2 instruction set is available
 */
static inline boolean_t
zfs_sse4_2_available(void)
{
#if defined(_KERNEL)
#if defined(KERNEL_EXPORTS_X86_FPU)
	return (!!boot_cpu_has(X86_FEATURE_XMM4_2));
#else
	return (B_FALSE);
#endif
#elif !defined(_KERNEL)
	return (__cpuid_has_sse4_2());
#endif
}

/*
 * Check if AVX instruction set is available
 */
static inline boolean_t
zfs_avx_available(void)
{
	boolean_t has_avx;
#if defined(_KERNEL)
#if defined(KERNEL_EXPORTS_X86_FPU)
	has_avx = !!boot_cpu_has(X86_FEATURE_AVX);
#else
	has_avx = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx = __cpuid_has_avx();
#endif

	return (has_avx && __ymm_enabled());
}

/*
 * Check if AVX2 instruction set is available
 */
static inline boolean_t
zfs_avx2_available(void)
{
	boolean_t has_avx2;
#if defined(_KERNEL)
#if defined(X86_FEATURE_AVX2) && defined(KERNEL_EXPORTS_X86_FPU)
	has_avx2 = !!boot_cpu_has(X86_FEATURE_AVX2);
#else
	has_avx2 = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx2 = __cpuid_has_avx2();
#endif

	return (has_avx2 && __ymm_enabled());
}

/*
 * Check if BMI1 instruction set is available
 */
static inline boolean_t
zfs_bmi1_available(void)
{
#if defined(_KERNEL)
#if defined(X86_FEATURE_BMI1) && defined(KERNEL_EXPORTS_X86_FPU)
	return (!!boot_cpu_has(X86_FEATURE_BMI1));
#else
	return (B_FALSE);
#endif
#elif !defined(_KERNEL)
	return (__cpuid_has_bmi1());
#endif
}

/*
 * Check if BMI2 instruction set is available
 */
static inline boolean_t
zfs_bmi2_available(void)
{
#if defined(_KERNEL)
#if defined(X86_FEATURE_BMI2) && defined(KERNEL_EXPORTS_X86_FPU)
	return (!!boot_cpu_has(X86_FEATURE_BMI2));
#else
	return (B_FALSE);
#endif
#elif !defined(_KERNEL)
	return (__cpuid_has_bmi2());
#endif
}

/*
 * Check if AES instruction set is available
 */
static inline boolean_t
zfs_aes_available(void)
{
#if defined(_KERNEL)
#if defined(X86_FEATURE_AES) && defined(KERNEL_EXPORTS_X86_FPU)
	return (!!boot_cpu_has(X86_FEATURE_AES));
#else
	return (B_FALSE);
#endif
#elif !defined(_KERNEL)
	return (__cpuid_has_aes());
#endif
}

/*
 * Check if PCLMULQDQ instruction set is available
 */
static inline boolean_t
zfs_pclmulqdq_available(void)
{
#if defined(_KERNEL)
#if defined(X86_FEATURE_PCLMULQDQ) && defined(KERNEL_EXPORTS_X86_FPU)
	return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ));
#else
	return (B_FALSE);
#endif
#elif !defined(_KERNEL)
	return (__cpuid_has_pclmulqdq());
#endif
}

/*
 * AVX-512 family of instruction sets:
 *
 * AVX512F	Foundation
 * AVX512CD	Conflict Detection Instructions
 * AVX512ER	Exponential and Reciprocal Instructions
 * AVX512PF	Prefetch Instructions
 *
 * AVX512BW	Byte and Word Instructions
 * AVX512DQ	Double-word and Quadword Instructions
 * AVX512VL	Vector Length Extensions
 *
 * AVX512IFMA	Integer Fused Multiply Add (Not supported by kernel 4.4)
 * AVX512VBMI	Vector Byte Manipulation Instructions
 */


/* Check if AVX512F instruction set is available */
static inline boolean_t
zfs_avx512f_available(void)
{
	boolean_t has_avx512 = B_FALSE;

#if defined(_KERNEL)
#if defined(X86_FEATURE_AVX512F) && defined(KERNEL_EXPORTS_X86_FPU)
	has_avx512 = !!boot_cpu_has(X86_FEATURE_AVX512F);
#else
	has_avx512 = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx512 = __cpuid_has_avx512f();
#endif

	return (has_avx512 && __zmm_enabled());
}

/* Check if AVX512CD instruction set is available */
static inline boolean_t
zfs_avx512cd_available(void)
{
	boolean_t has_avx512 = B_FALSE;

#if defined(_KERNEL)
#if defined(X86_FEATURE_AVX512CD) && defined(KERNEL_EXPORTS_X86_FPU)
	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
	    boot_cpu_has(X86_FEATURE_AVX512CD);
#else
	has_avx512 = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx512 = __cpuid_has_avx512cd();
#endif

	return (has_avx512 && __zmm_enabled());
}

/* Check if AVX512ER instruction set is available */
static inline boolean_t
zfs_avx512er_available(void)
{
	boolean_t has_avx512 = B_FALSE;

#if defined(_KERNEL)
#if defined(X86_FEATURE_AVX512ER) && defined(KERNEL_EXPORTS_X86_FPU)
	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
	    boot_cpu_has(X86_FEATURE_AVX512ER);
#else
	has_avx512 = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx512 = __cpuid_has_avx512er();
#endif

	return (has_avx512 && __zmm_enabled());
}

/* Check if AVX512PF instruction set is available */
static inline boolean_t
zfs_avx512pf_available(void)
{
	boolean_t has_avx512 = B_FALSE;

#if defined(_KERNEL)
#if defined(X86_FEATURE_AVX512PF) && defined(KERNEL_EXPORTS_X86_FPU)
	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
	    boot_cpu_has(X86_FEATURE_AVX512PF);
#else
	has_avx512 = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx512 = __cpuid_has_avx512pf();
#endif

	return (has_avx512 && __zmm_enabled());
}

/* Check if AVX512BW instruction set is available */
static inline boolean_t
zfs_avx512bw_available(void)
{
	boolean_t has_avx512 = B_FALSE;

#if defined(_KERNEL)
#if defined(X86_FEATURE_AVX512BW) && defined(KERNEL_EXPORTS_X86_FPU)
	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
	    boot_cpu_has(X86_FEATURE_AVX512BW);
#else
	has_avx512 = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx512 = __cpuid_has_avx512bw();
#endif

	return (has_avx512 && __zmm_enabled());
}

/* Check if AVX512DQ instruction set is available */
static inline boolean_t
zfs_avx512dq_available(void)
{
	boolean_t has_avx512 = B_FALSE;

#if defined(_KERNEL)
#if defined(X86_FEATURE_AVX512DQ) && defined(KERNEL_EXPORTS_X86_FPU)
	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
	    boot_cpu_has(X86_FEATURE_AVX512DQ);
#else
	has_avx512 = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx512 = __cpuid_has_avx512dq();
#endif

	return (has_avx512 && __zmm_enabled());
}

/* Check if AVX512VL instruction set is available */
static inline boolean_t
zfs_avx512vl_available(void)
{
	boolean_t has_avx512 = B_FALSE;

#if defined(_KERNEL)
#if defined(X86_FEATURE_AVX512VL) && defined(KERNEL_EXPORTS_X86_FPU)
	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
	    boot_cpu_has(X86_FEATURE_AVX512VL);
#else
	has_avx512 = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx512 = __cpuid_has_avx512vl();
#endif

	return (has_avx512 && __zmm_enabled());
}

/* Check if AVX512IFMA instruction set is available */
static inline boolean_t
zfs_avx512ifma_available(void)
{
	boolean_t has_avx512 = B_FALSE;

#if defined(_KERNEL)
#if defined(X86_FEATURE_AVX512IFMA) && defined(KERNEL_EXPORTS_X86_FPU)
	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
	    boot_cpu_has(X86_FEATURE_AVX512IFMA);
#else
	has_avx512 = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx512 = __cpuid_has_avx512ifma();
#endif

	return (has_avx512 && __zmm_enabled());
}

/* Check if AVX512VBMI instruction set is available */
static inline boolean_t
zfs_avx512vbmi_available(void)
{
	boolean_t has_avx512 = B_FALSE;

#if defined(_KERNEL)
#if defined(X86_FEATURE_AVX512VBMI) && defined(KERNEL_EXPORTS_X86_FPU)
	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
	    boot_cpu_has(X86_FEATURE_AVX512VBMI);
#else
	has_avx512 = B_FALSE;
#endif
#elif !defined(_KERNEL)
	has_avx512 = __cpuid_has_avx512f() &&
	    __cpuid_has_avx512vbmi();
#endif

	return (has_avx512 && __zmm_enabled());
}

#endif /* defined(__x86) */

#endif /* _SIMD_X86_H */
Commit	Line	Data
fc0c72b1 GN	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
	22	* Copyright (C) 2016 Gvozden Neskovic <neskovic@compeng.uni-frankfurt.de>.
	23	*/
	24
	25	/*
	26	* USER API:
	27	*
	28	* Kernel fpu methods:
	29	* kfpu_begin()
	30	* kfpu_end()
	31	*
	32	* SIMD support:
	33	*
	34	* Following functions should be called to determine whether CPU feature
	35	* is supported. All functions are usable in kernel and user space.
	36	* If a SIMD algorithm is using more than one instruction set
	37	* all relevant feature test functions should be called.
	38	*
	39	* Supported features:
	40	* zfs_sse_available()
	41	* zfs_sse2_available()
	42	* zfs_sse3_available()
	43	* zfs_ssse3_available()
	44	* zfs_sse4_1_available()
	45	* zfs_sse4_2_available()
32ffaa3d	46	*
fc0c72b1 GN	47	* zfs_avx_available()
fc0c72b1 GN	48	* zfs_avx2_available()
32ffaa3d	49	*
fc0c72b1 GN	50	* zfs_bmi1_available()
fc0c72b1 GN	51	* zfs_bmi2_available()
32ffaa3d GN	52	*
	53	* zfs_avx512f_available()
	54	* zfs_avx512cd_available()
	55	* zfs_avx512er_available()
	56	* zfs_avx512pf_available()
	57	* zfs_avx512bw_available()
	58	* zfs_avx512dq_available()
	59	* zfs_avx512vl_available()
	60	* zfs_avx512ifma_available()
	61	* zfs_avx512vbmi_available()
	62	*
	63	* NOTE(AVX-512VL): If using AVX-512 instructions with 128Bit registers
	64	* also add zfs_avx512vl_available() to feature check.
fc0c72b1 GN	65	*/
	66
	67	#ifndef _SIMD_X86_H
	68	#define _SIMD_X86_H
	69
	70	#include <sys/isa_defs.h>
	71
	72	/* only for __x86 */
	73	#if defined(__x86)
	74
	75	#include <sys/types.h>
	76
	77	#if defined(_KERNEL)
	78	#include <asm/cpufeature.h>
	79	#else
	80	#include <cpuid.h>
	81	#endif
	82
	83	#if defined(_KERNEL)
0c593296	84	#if defined(HAVE_UNDERSCORE_KERNEL_FPU)
fc0c72b1 GN	85	#include <asm/fpu/api.h>
	86	#include <asm/fpu/internal.h>
	87	#define kfpu_begin() \
	88	{ \
	89	preempt_disable(); \
	90	__kernel_fpu_begin(); \
	91	}
	92	#define kfpu_end() \
	93	{ \
	94	__kernel_fpu_end(); \
	95	preempt_enable(); \
	96	}
0c593296	97	#elif defined(HAVE_KERNEL_FPU)
fc0c72b1 GN	98	#include <asm/i387.h>
	99	#include <asm/xcr.h>
	100	#define kfpu_begin() kernel_fpu_begin()
	101	#define kfpu_end() kernel_fpu_end()
0c593296 TH	102	#else
	103	/* Kernel doesn't export any kernel_fpu_* functions */
	104	#include <asm/fpu/internal.h> /* For kernel xgetbv() */
	105	#define kfpu_begin() panic("This code should never run")
	106	#define kfpu_end() panic("This code should never run")
	107	#endif /* defined(HAVE_KERNEL_FPU) */
	108
fc0c72b1 GN	109	#else
	110	/*
	111	* fpu dummy methods for userspace
	112	*/
	113	#define kfpu_begin() do {} while (0)
	114	#define kfpu_end() do {} while (0)
	115	#endif /* defined(_KERNEL) */
	116
	117	/*
	118	* CPUID feature tests for user-space. Linux kernel provides an interface for
	119	* CPU feature testing.
	120	*/
	121	#if !defined(_KERNEL)
	122
	123	/*
	124	* x86 registers used implicitly by CPUID
	125	*/
	126	typedef enum cpuid_regs {
	127	EAX = 0,
	128	EBX,
	129	ECX,
	130	EDX,
	131	CPUID_REG_CNT = 4
	132	} cpuid_regs_t;
	133
	134	/*
	135	* List of instruction sets identified by CPUID
	136	*/
	137	typedef enum cpuid_inst_sets {
	138	SSE = 0,
	139	SSE2,
	140	SSE3,
	141	SSSE3,
	142	SSE4_1,
	143	SSE4_2,
	144	OSXSAVE,
	145	AVX,
	146	AVX2,
	147	BMI1,
32ffaa3d GN	148	BMI2,
	149	AVX512F,
	150	AVX512CD,
	151	AVX512DQ,
	152	AVX512BW,
	153	AVX512IFMA,
	154	AVX512VBMI,
	155	AVX512PF,
	156	AVX512ER,
010d1247 NL	157	AVX512VL,
	158	AES,
	159	PCLMULQDQ
fc0c72b1 GN	160	} cpuid_inst_sets_t;
	161
	162	/*
	163	* Instruction set descriptor.
	164	*/
	165	typedef struct cpuid_feature_desc {
	166	uint32_t leaf; /* CPUID leaf */
32ffaa3d	167	uint32_t subleaf; /* CPUID sub-leaf */
fc0c72b1 GN	168	uint32_t flag; /* bit mask of the feature */
	169	cpuid_regs_t reg; /* which CPUID return register to test */
	170	} cpuid_feature_desc_t;
	171
32ffaa3d GN	172	#define _AVX512F_BIT (1U << 16)
	173	#define _AVX512CD_BIT (_AVX512F_BIT \| (1U << 28))
	174	#define _AVX512DQ_BIT (_AVX512F_BIT \| (1U << 17))
	175	#define _AVX512BW_BIT (_AVX512F_BIT \| (1U << 30))
	176	#define _AVX512IFMA_BIT (_AVX512F_BIT \| (1U << 21))
	177	#define _AVX512VBMI_BIT (1U << 1) /* AVX512F_BIT is on another leaf */
	178	#define _AVX512PF_BIT (_AVX512F_BIT \| (1U << 26))
	179	#define _AVX512ER_BIT (_AVX512F_BIT \| (1U << 27))
	180	#define _AVX512VL_BIT (1U << 31) /* if used also check other levels */
010d1247 NL	181	#define _AES_BIT (1U << 25)
010d1247 NL	182	#define _PCLMULQDQ_BIT (1U << 1)
32ffaa3d	183
fc0c72b1 GN	184	/*
	185	* Descriptions of supported instruction sets
	186	*/
	187	static const cpuid_feature_desc_t cpuid_features[] = {
	188	[SSE] = {1U, 0U, 1U << 25, EDX },
	189	[SSE2] = {1U, 0U, 1U << 26, EDX },
	190	[SSE3] = {1U, 0U, 1U << 0, ECX },
	191	[SSSE3] = {1U, 0U, 1U << 9, ECX },
	192	[SSE4_1] = {1U, 0U, 1U << 19, ECX },
	193	[SSE4_2] = {1U, 0U, 1U << 20, ECX },
	194	[OSXSAVE] = {1U, 0U, 1U << 27, ECX },
	195	[AVX] = {1U, 0U, 1U << 28, ECX },
	196	[AVX2] = {7U, 0U, 1U << 5, EBX },
	197	[BMI1] = {7U, 0U, 1U << 3, EBX },
32ffaa3d GN	198	[BMI2] = {7U, 0U, 1U << 8, EBX },
	199	[AVX512F] = {7U, 0U, _AVX512F_BIT, EBX },
	200	[AVX512CD] = {7U, 0U, _AVX512CD_BIT, EBX },
	201	[AVX512DQ] = {7U, 0U, _AVX512DQ_BIT, EBX },
	202	[AVX512BW] = {7U, 0U, _AVX512BW_BIT, EBX },
	203	[AVX512IFMA] = {7U, 0U, _AVX512IFMA_BIT, EBX },
	204	[AVX512VBMI] = {7U, 0U, _AVX512VBMI_BIT, ECX },
	205	[AVX512PF] = {7U, 0U, _AVX512PF_BIT, EBX },
	206	[AVX512ER] = {7U, 0U, _AVX512ER_BIT, EBX },
010d1247 NL	207	[AVX512VL] = {7U, 0U, _AVX512ER_BIT, EBX },
	208	[AES] = {1U, 0U, _AES_BIT, ECX },
	209	[PCLMULQDQ] = {1U, 0U, _PCLMULQDQ_BIT, ECX },
fc0c72b1 GN	210	};
	211
	212	/*
	213	* Check if OS supports AVX and AVX2 by checking XCR0
	214	* Only call this function if CPUID indicates that AVX feature is
	215	* supported by the CPU, otherwise it might be an illegal instruction.
	216	*/
	217	static inline uint64_t
	218	xgetbv(uint32_t index)
	219	{
	220	uint32_t eax, edx;
	221	/* xgetbv - instruction byte code */
	222	__asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0"
02730c33 BB	223	: "=a" (eax), "=d" (edx)
02730c33 BB	224	: "c" (index));
fc0c72b1 GN	225
	226	return ((((uint64_t)edx)<<32) \| (uint64_t)eax);
	227	}
	228
	229	/*
	230	* Check if CPU supports a feature
	231	*/
	232	static inline boolean_t
	233	__cpuid_check_feature(const cpuid_feature_desc_t *desc)
	234	{
	235	uint32_t r[CPUID_REG_CNT];
	236
	237	if (__get_cpuid_max(0, NULL) >= desc->leaf) {
	238	/*
	239	* __cpuid_count is needed to properly check
	240	* for AVX2. It is a macro, so return parameters
	241	* are passed by value.
	242	*/
	243	__cpuid_count(desc->leaf, desc->subleaf,
02730c33	244	r[EAX], r[EBX], r[ECX], r[EDX]);
32ffaa3d	245	return ((r[desc->reg] & desc->flag) == desc->flag);
fc0c72b1 GN	246	}
	247	return (B_FALSE);
	248	}
	249
02730c33	250	#define CPUID_FEATURE_CHECK(name, id) \
32ffaa3d GN	251	static inline boolean_t \
	252	__cpuid_has_ ## name(void) \
	253	{ \
fc0c72b1 GN	254	return (__cpuid_check_feature(&cpuid_features[id])); \
	255	}
	256
	257	/*
	258	* Define functions for user-space CPUID features testing
	259	*/
	260	CPUID_FEATURE_CHECK(sse, SSE);
	261	CPUID_FEATURE_CHECK(sse2, SSE2);
	262	CPUID_FEATURE_CHECK(sse3, SSE3);
	263	CPUID_FEATURE_CHECK(ssse3, SSSE3);
	264	CPUID_FEATURE_CHECK(sse4_1, SSE4_1);
	265	CPUID_FEATURE_CHECK(sse4_2, SSE4_2);
	266	CPUID_FEATURE_CHECK(avx, AVX);
	267	CPUID_FEATURE_CHECK(avx2, AVX2);
	268	CPUID_FEATURE_CHECK(osxsave, OSXSAVE);
	269	CPUID_FEATURE_CHECK(bmi1, BMI1);
	270	CPUID_FEATURE_CHECK(bmi2, BMI2);
32ffaa3d GN	271	CPUID_FEATURE_CHECK(avx512f, AVX512F);
	272	CPUID_FEATURE_CHECK(avx512cd, AVX512CD);
	273	CPUID_FEATURE_CHECK(avx512dq, AVX512DQ);
	274	CPUID_FEATURE_CHECK(avx512bw, AVX512BW);
	275	CPUID_FEATURE_CHECK(avx512ifma, AVX512IFMA);
	276	CPUID_FEATURE_CHECK(avx512vbmi, AVX512VBMI);
	277	CPUID_FEATURE_CHECK(avx512pf, AVX512PF);
	278	CPUID_FEATURE_CHECK(avx512er, AVX512ER);
	279	CPUID_FEATURE_CHECK(avx512vl, AVX512VL);
010d1247 NL	280	CPUID_FEATURE_CHECK(aes, AES);
010d1247 NL	281	CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ);
fc0c72b1 GN	282
	283	#endif /* !defined(_KERNEL) */
	284
32ffaa3d	285
fc0c72b1	286	/*
32ffaa3d	287	* Detect register set support
fc0c72b1 GN	288	*/
fc0c72b1 GN	289	static inline boolean_t
32ffaa3d	290	__simd_state_enabled(const uint64_t state)
fc0c72b1	291	{
fc0c72b1 GN	292	boolean_t has_osxsave;
	293	uint64_t xcr0;
	294
0c593296 TH	295	#if defined(_KERNEL)
0c593296 TH	296	#if defined(X86_FEATURE_OSXSAVE) && defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1	297	has_osxsave = !!boot_cpu_has(X86_FEATURE_OSXSAVE);
fc0c72b1	298	#else
0c593296 TH	299	has_osxsave = B_FALSE;
	300	#endif
	301	#elif !defined(_KERNEL)
fc0c72b1 GN	302	has_osxsave = __cpuid_has_osxsave();
	303	#endif
	304
	305	if (!has_osxsave)
	306	return (B_FALSE);
	307
	308	xcr0 = xgetbv(0);
32ffaa3d	309	return ((xcr0 & state) == state);
fc0c72b1 GN	310	}
fc0c72b1 GN	311
32ffaa3d GN	312	#define _XSTATE_SSE_AVX (0x2 \| 0x4)
	313	#define _XSTATE_AVX512 (0xE0 \| _XSTATE_SSE_AVX)
	314
	315	#define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX)
	316	#define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512)
	317
	318
fc0c72b1 GN	319	/*
	320	* Check if SSE instruction set is available
	321	*/
	322	static inline boolean_t
	323	zfs_sse_available(void)
	324	{
	325	#if defined(_KERNEL)
0c593296	326	#if defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1 GN	327	return (!!boot_cpu_has(X86_FEATURE_XMM));
fc0c72b1 GN	328	#else
0c593296 TH	329	return (B_FALSE);
	330	#endif
	331	#elif !defined(_KERNEL)
fc0c72b1 GN	332	return (__cpuid_has_sse());
	333	#endif
	334	}
	335
	336	/*
	337	* Check if SSE2 instruction set is available
	338	*/
	339	static inline boolean_t
	340	zfs_sse2_available(void)
	341	{
	342	#if defined(_KERNEL)
0c593296	343	#if defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1 GN	344	return (!!boot_cpu_has(X86_FEATURE_XMM2));
fc0c72b1 GN	345	#else
0c593296 TH	346	return (B_FALSE);
	347	#endif
	348	#elif !defined(_KERNEL)
fc0c72b1 GN	349	return (__cpuid_has_sse2());
	350	#endif
	351	}
	352
	353	/*
	354	* Check if SSE3 instruction set is available
	355	*/
	356	static inline boolean_t
	357	zfs_sse3_available(void)
	358	{
	359	#if defined(_KERNEL)
0c593296	360	#if defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1 GN	361	return (!!boot_cpu_has(X86_FEATURE_XMM3));
fc0c72b1 GN	362	#else
0c593296 TH	363	return (B_FALSE);
	364	#endif
	365	#elif !defined(_KERNEL)
fc0c72b1 GN	366	return (__cpuid_has_sse3());
	367	#endif
	368	}
	369
	370	/*
	371	* Check if SSSE3 instruction set is available
	372	*/
	373	static inline boolean_t
	374	zfs_ssse3_available(void)
	375	{
	376	#if defined(_KERNEL)
0c593296	377	#if defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1 GN	378	return (!!boot_cpu_has(X86_FEATURE_SSSE3));
fc0c72b1 GN	379	#else
0c593296 TH	380	return (B_FALSE);
	381	#endif
	382	#elif !defined(_KERNEL)
fc0c72b1 GN	383	return (__cpuid_has_ssse3());
	384	#endif
	385	}
	386
	387	/*
	388	* Check if SSE4.1 instruction set is available
	389	*/
	390	static inline boolean_t
	391	zfs_sse4_1_available(void)
	392	{
	393	#if defined(_KERNEL)
0c593296	394	#if defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1 GN	395	return (!!boot_cpu_has(X86_FEATURE_XMM4_1));
fc0c72b1 GN	396	#else
0c593296 TH	397	return (B_FALSE);
	398	#endif
	399	#elif !defined(_KERNEL)
fc0c72b1 GN	400	return (__cpuid_has_sse4_1());
	401	#endif
	402	}
	403
	404	/*
	405	* Check if SSE4.2 instruction set is available
	406	*/
	407	static inline boolean_t
	408	zfs_sse4_2_available(void)
	409	{
	410	#if defined(_KERNEL)
0c593296	411	#if defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1 GN	412	return (!!boot_cpu_has(X86_FEATURE_XMM4_2));
fc0c72b1 GN	413	#else
0c593296 TH	414	return (B_FALSE);
	415	#endif
	416	#elif !defined(_KERNEL)
fc0c72b1 GN	417	return (__cpuid_has_sse4_2());
	418	#endif
	419	}
	420
	421	/*
	422	* Check if AVX instruction set is available
	423	*/
	424	static inline boolean_t
	425	zfs_avx_available(void)
	426	{
	427	boolean_t has_avx;
	428	#if defined(_KERNEL)
0c593296	429	#if defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1 GN	430	has_avx = !!boot_cpu_has(X86_FEATURE_AVX);
fc0c72b1 GN	431	#else
0c593296 TH	432	has_avx = B_FALSE;
	433	#endif
	434	#elif !defined(_KERNEL)
fc0c72b1 GN	435	has_avx = __cpuid_has_avx();
	436	#endif
	437
	438	return (has_avx && __ymm_enabled());
	439	}
	440
	441	/*
	442	* Check if AVX2 instruction set is available
	443	*/
	444	static inline boolean_t
	445	zfs_avx2_available(void)
	446	{
	447	boolean_t has_avx2;
0c593296 TH	448	#if defined(_KERNEL)
0c593296 TH	449	#if defined(X86_FEATURE_AVX2) && defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1	450	has_avx2 = !!boot_cpu_has(X86_FEATURE_AVX2);
fc0c72b1	451	#else
0c593296 TH	452	has_avx2 = B_FALSE;
	453	#endif
	454	#elif !defined(_KERNEL)
fc0c72b1 GN	455	has_avx2 = __cpuid_has_avx2();
	456	#endif
	457
	458	return (has_avx2 && __ymm_enabled());
	459	}
	460
	461	/*
	462	* Check if BMI1 instruction set is available
	463	*/
	464	static inline boolean_t
	465	zfs_bmi1_available(void)
	466	{
0c593296 TH	467	#if defined(_KERNEL)
0c593296 TH	468	#if defined(X86_FEATURE_BMI1) && defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1	469	return (!!boot_cpu_has(X86_FEATURE_BMI1));
fc0c72b1	470	#else
0c593296 TH	471	return (B_FALSE);
	472	#endif
	473	#elif !defined(_KERNEL)
fc0c72b1 GN	474	return (__cpuid_has_bmi1());
	475	#endif
	476	}
	477
	478	/*
	479	* Check if BMI2 instruction set is available
	480	*/
	481	static inline boolean_t
	482	zfs_bmi2_available(void)
	483	{
0c593296 TH	484	#if defined(_KERNEL)
0c593296 TH	485	#if defined(X86_FEATURE_BMI2) && defined(KERNEL_EXPORTS_X86_FPU)
fc0c72b1	486	return (!!boot_cpu_has(X86_FEATURE_BMI2));
fc0c72b1	487	#else
0c593296 TH	488	return (B_FALSE);
	489	#endif
	490	#elif !defined(_KERNEL)
fc0c72b1 GN	491	return (__cpuid_has_bmi2());
	492	#endif
	493	}
	494
010d1247 NL	495	/*
	496	* Check if AES instruction set is available
	497	*/
	498	static inline boolean_t
	499	zfs_aes_available(void)
	500	{
0c593296 TH	501	#if defined(_KERNEL)
0c593296 TH	502	#if defined(X86_FEATURE_AES) && defined(KERNEL_EXPORTS_X86_FPU)
010d1247	503	return (!!boot_cpu_has(X86_FEATURE_AES));
010d1247	504	#else
0c593296 TH	505	return (B_FALSE);
	506	#endif
	507	#elif !defined(_KERNEL)
010d1247 NL	508	return (__cpuid_has_aes());
	509	#endif
	510	}
	511
	512	/*
	513	* Check if PCLMULQDQ instruction set is available
	514	*/
	515	static inline boolean_t
	516	zfs_pclmulqdq_available(void)
	517	{
0c593296 TH	518	#if defined(_KERNEL)
0c593296 TH	519	#if defined(X86_FEATURE_PCLMULQDQ) && defined(KERNEL_EXPORTS_X86_FPU)
010d1247	520	return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ));
010d1247	521	#else
0c593296 TH	522	return (B_FALSE);
	523	#endif
	524	#elif !defined(_KERNEL)
010d1247 NL	525	return (__cpuid_has_pclmulqdq());
	526	#endif
	527	}
32ffaa3d GN	528
	529	/*
	530	* AVX-512 family of instruction sets:
	531	*
	532	* AVX512F Foundation
	533	* AVX512CD Conflict Detection Instructions
	534	* AVX512ER Exponential and Reciprocal Instructions
	535	* AVX512PF Prefetch Instructions
	536	*
	537	* AVX512BW Byte and Word Instructions
	538	* AVX512DQ Double-word and Quadword Instructions
	539	* AVX512VL Vector Length Extensions
	540	*
	541	* AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4)
	542	* AVX512VBMI Vector Byte Manipulation Instructions
	543	*/
	544
	545
	546	/* Check if AVX512F instruction set is available */
	547	static inline boolean_t
	548	zfs_avx512f_available(void)
	549	{
	550	boolean_t has_avx512 = B_FALSE;
	551
0c593296 TH	552	#if defined(_KERNEL)
0c593296 TH	553	#if defined(X86_FEATURE_AVX512F) && defined(KERNEL_EXPORTS_X86_FPU)
32ffaa3d	554	has_avx512 = !!boot_cpu_has(X86_FEATURE_AVX512F);
0c593296 TH	555	#else
	556	has_avx512 = B_FALSE;
	557	#endif
32ffaa3d GN	558	#elif !defined(_KERNEL)
	559	has_avx512 = __cpuid_has_avx512f();
	560	#endif
	561
	562	return (has_avx512 && __zmm_enabled());
	563	}
	564
	565	/* Check if AVX512CD instruction set is available */
	566	static inline boolean_t
	567	zfs_avx512cd_available(void)
	568	{
	569	boolean_t has_avx512 = B_FALSE;
	570
0c593296 TH	571	#if defined(_KERNEL)
0c593296 TH	572	#if defined(X86_FEATURE_AVX512CD) && defined(KERNEL_EXPORTS_X86_FPU)
32ffaa3d GN	573	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
32ffaa3d GN	574	boot_cpu_has(X86_FEATURE_AVX512CD);
0c593296 TH	575	#else
	576	has_avx512 = B_FALSE;
	577	#endif
32ffaa3d GN	578	#elif !defined(_KERNEL)
	579	has_avx512 = __cpuid_has_avx512cd();
	580	#endif
	581
	582	return (has_avx512 && __zmm_enabled());
	583	}
	584
	585	/* Check if AVX512ER instruction set is available */
	586	static inline boolean_t
	587	zfs_avx512er_available(void)
	588	{
	589	boolean_t has_avx512 = B_FALSE;
	590
0c593296 TH	591	#if defined(_KERNEL)
0c593296 TH	592	#if defined(X86_FEATURE_AVX512ER) && defined(KERNEL_EXPORTS_X86_FPU)
32ffaa3d GN	593	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
32ffaa3d GN	594	boot_cpu_has(X86_FEATURE_AVX512ER);
0c593296 TH	595	#else
	596	has_avx512 = B_FALSE;
	597	#endif
32ffaa3d GN	598	#elif !defined(_KERNEL)
	599	has_avx512 = __cpuid_has_avx512er();
	600	#endif
	601
	602	return (has_avx512 && __zmm_enabled());
	603	}
	604
	605	/* Check if AVX512PF instruction set is available */
	606	static inline boolean_t
	607	zfs_avx512pf_available(void)
	608	{
	609	boolean_t has_avx512 = B_FALSE;
	610
0c593296 TH	611	#if defined(_KERNEL)
0c593296 TH	612	#if defined(X86_FEATURE_AVX512PF) && defined(KERNEL_EXPORTS_X86_FPU)
32ffaa3d GN	613	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
32ffaa3d GN	614	boot_cpu_has(X86_FEATURE_AVX512PF);
0c593296 TH	615	#else
	616	has_avx512 = B_FALSE;
	617	#endif
32ffaa3d GN	618	#elif !defined(_KERNEL)
	619	has_avx512 = __cpuid_has_avx512pf();
	620	#endif
	621
	622	return (has_avx512 && __zmm_enabled());
	623	}
	624
	625	/* Check if AVX512BW instruction set is available */
	626	static inline boolean_t
	627	zfs_avx512bw_available(void)
	628	{
	629	boolean_t has_avx512 = B_FALSE;
	630
0c593296 TH	631	#if defined(_KERNEL)
0c593296 TH	632	#if defined(X86_FEATURE_AVX512BW) && defined(KERNEL_EXPORTS_X86_FPU)
32ffaa3d GN	633	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
32ffaa3d GN	634	boot_cpu_has(X86_FEATURE_AVX512BW);
0c593296 TH	635	#else
	636	has_avx512 = B_FALSE;
	637	#endif
32ffaa3d GN	638	#elif !defined(_KERNEL)
	639	has_avx512 = __cpuid_has_avx512bw();
	640	#endif
	641
	642	return (has_avx512 && __zmm_enabled());
	643	}
	644
	645	/* Check if AVX512DQ instruction set is available */
	646	static inline boolean_t
	647	zfs_avx512dq_available(void)
	648	{
	649	boolean_t has_avx512 = B_FALSE;
	650
0c593296 TH	651	#if defined(_KERNEL)
0c593296 TH	652	#if defined(X86_FEATURE_AVX512DQ) && defined(KERNEL_EXPORTS_X86_FPU)
32ffaa3d GN	653	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
32ffaa3d GN	654	boot_cpu_has(X86_FEATURE_AVX512DQ);
0c593296 TH	655	#else
	656	has_avx512 = B_FALSE;
	657	#endif
32ffaa3d GN	658	#elif !defined(_KERNEL)
	659	has_avx512 = __cpuid_has_avx512dq();
	660	#endif
	661
	662	return (has_avx512 && __zmm_enabled());
	663	}
	664
	665	/* Check if AVX512VL instruction set is available */
	666	static inline boolean_t
	667	zfs_avx512vl_available(void)
	668	{
	669	boolean_t has_avx512 = B_FALSE;
	670
0c593296 TH	671	#if defined(_KERNEL)
0c593296 TH	672	#if defined(X86_FEATURE_AVX512VL) && defined(KERNEL_EXPORTS_X86_FPU)
32ffaa3d GN	673	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
32ffaa3d GN	674	boot_cpu_has(X86_FEATURE_AVX512VL);
0c593296 TH	675	#else
	676	has_avx512 = B_FALSE;
	677	#endif
32ffaa3d GN	678	#elif !defined(_KERNEL)
	679	has_avx512 = __cpuid_has_avx512vl();
	680	#endif
	681
	682	return (has_avx512 && __zmm_enabled());
	683	}
	684
	685	/* Check if AVX512IFMA instruction set is available */
	686	static inline boolean_t
	687	zfs_avx512ifma_available(void)
	688	{
	689	boolean_t has_avx512 = B_FALSE;
	690
0c593296 TH	691	#if defined(_KERNEL)
0c593296 TH	692	#if defined(X86_FEATURE_AVX512IFMA) && defined(KERNEL_EXPORTS_X86_FPU)
32ffaa3d GN	693	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
32ffaa3d GN	694	boot_cpu_has(X86_FEATURE_AVX512IFMA);
0c593296 TH	695	#else
	696	has_avx512 = B_FALSE;
	697	#endif
32ffaa3d GN	698	#elif !defined(_KERNEL)
	699	has_avx512 = __cpuid_has_avx512ifma();
	700	#endif
	701
	702	return (has_avx512 && __zmm_enabled());
	703	}
	704
	705	/* Check if AVX512VBMI instruction set is available */
	706	static inline boolean_t
	707	zfs_avx512vbmi_available(void)
	708	{
	709	boolean_t has_avx512 = B_FALSE;
	710
0c593296 TH	711	#if defined(_KERNEL)
0c593296 TH	712	#if defined(X86_FEATURE_AVX512VBMI) && defined(KERNEL_EXPORTS_X86_FPU)
32ffaa3d GN	713	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
32ffaa3d GN	714	boot_cpu_has(X86_FEATURE_AVX512VBMI);
0c593296 TH	715	#else
	716	has_avx512 = B_FALSE;
	717	#endif
32ffaa3d GN	718	#elif !defined(_KERNEL)
	719	has_avx512 = __cpuid_has_avx512f() &&
	720	__cpuid_has_avx512vbmi();
	721	#endif
	722
	723	return (has_avx512 && __zmm_enabled());
	724	}
	725
fc0c72b1 GN	726	#endif /* defined(__x86) */
	727
	728	#endif /* _SIMD_X86_H */