[ceph.git] / ceph / src / pmdk / src / libpmem2 / x86_64 / init.c

// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2014-2020, Intel Corporation */

#include <string.h>
#include <xmmintrin.h>

#include "auto_flush.h"
#include "cpu.h"
#include "flush.h"
#include "memcpy_memset.h"
#include "os.h"
#include "out.h"
#include "pmem2_arch.h"
#include "valgrind_internal.h"

#define MOVNT_THRESHOLD	256

size_t Movnt_threshold = MOVNT_THRESHOLD;

/*
 * memory_barrier -- (internal) issue the fence instruction
 */
static void
memory_barrier(void)
{
	LOG(15, NULL);
	_mm_sfence();	/* ensure CLWB or CLFLUSHOPT completes */
}

/*
 * flush_clflush -- (internal) flush the CPU cache, using clflush
 */
static void
flush_clflush(const void *addr, size_t len)
{
	LOG(15, "addr %p len %zu", addr, len);

	flush_clflush_nolog(addr, len);
}

/*
 * flush_clflushopt -- (internal) flush the CPU cache, using clflushopt
 */
static void
flush_clflushopt(const void *addr, size_t len)
{
	LOG(15, "addr %p len %zu", addr, len);

	flush_clflushopt_nolog(addr, len);
}

/*
 * flush_clwb -- (internal) flush the CPU cache, using clwb
 */
static void
flush_clwb(const void *addr, size_t len)
{
	LOG(15, "addr %p len %zu", addr, len);

	flush_clwb_nolog(addr, len);
}

#if SSE2_AVAILABLE || AVX_AVAILABLE || AVX512F_AVAILABLE
#define PMEM2_F_MEM_MOVNT (PMEM2_F_MEM_WC | PMEM2_F_MEM_NONTEMPORAL)
#define PMEM2_F_MEM_MOV   (PMEM2_F_MEM_WB | PMEM2_F_MEM_TEMPORAL)

#define MEMCPY_TEMPLATE(isa, flush, perfbarrier) \
static void *\
memmove_nodrain_##isa##_##flush##perfbarrier(void *dest, const void *src, \
		size_t len, unsigned flags, flush_func flushf)\
{\
	if (len == 0 || src == dest)\
		return dest;\
\
	if (flags & PMEM2_F_MEM_NOFLUSH) \
		memmove_mov_##isa##_noflush(dest, src, len); \
	else if (flags & PMEM2_F_MEM_MOVNT)\
		memmove_movnt_##isa ##_##flush##perfbarrier(dest, src, len);\
	else if (flags & PMEM2_F_MEM_MOV)\
		memmove_mov_##isa##_##flush(dest, src, len);\
	else if (len < Movnt_threshold)\
		memmove_mov_##isa##_##flush(dest, src, len);\
	else\
		memmove_movnt_##isa##_##flush##perfbarrier(dest, src, len);\
\
	return dest;\
}

#define MEMCPY_TEMPLATE_EADR(isa, perfbarrier) \
static void *\
memmove_nodrain_##isa##_eadr##perfbarrier(void *dest, const void *src, \
		size_t len, unsigned flags, flush_func flushf)\
{\
	if (len == 0 || src == dest)\
		return dest;\
\
	if (flags & PMEM2_F_MEM_NOFLUSH)\
		memmove_mov_##isa##_noflush(dest, src, len);\
	else if (flags & PMEM2_F_MEM_NONTEMPORAL)\
		memmove_movnt_##isa##_empty##perfbarrier(dest, src, len);\
	else\
		memmove_mov_##isa##_empty(dest, src, len);\
\
	return dest;\
}

#define MEMSET_TEMPLATE(isa, flush, perfbarrier)\
static void *\
memset_nodrain_##isa##_##flush##perfbarrier(void *dest, int c, size_t len, \
		unsigned flags, flush_func flushf)\
{\
	if (len == 0)\
		return dest;\
\
	if (flags & PMEM2_F_MEM_NOFLUSH) \
		memset_mov_##isa##_noflush(dest, c, len); \
	else if (flags & PMEM2_F_MEM_MOVNT)\
		memset_movnt_##isa##_##flush##perfbarrier(dest, c, len);\
	else if (flags & PMEM2_F_MEM_MOV)\
		memset_mov_##isa##_##flush(dest, c, len);\
	else if (len < Movnt_threshold)\
		memset_mov_##isa##_##flush(dest, c, len);\
	else\
		memset_movnt_##isa##_##flush##perfbarrier(dest, c, len);\
\
	return dest;\
}

#define MEMSET_TEMPLATE_EADR(isa, perfbarrier) \
static void *\
memset_nodrain_##isa##_eadr##perfbarrier(void *dest, int c, size_t len, \
		unsigned flags, flush_func flushf)\
{\
	if (len == 0)\
		return dest;\
\
	if (flags & PMEM2_F_MEM_NOFLUSH)\
		memset_mov_##isa##_noflush(dest, c, len);\
	else if (flags & PMEM2_F_MEM_NONTEMPORAL)\
		memset_movnt_##isa##_empty##perfbarrier(dest, c, len);\
	else\
		memset_mov_##isa##_empty(dest, c, len);\
\
	return dest;\
}
#endif

#if SSE2_AVAILABLE
MEMCPY_TEMPLATE(sse2, clflush, _nobarrier)
MEMCPY_TEMPLATE(sse2, clflushopt, _nobarrier)
MEMCPY_TEMPLATE(sse2, clwb, _nobarrier)
MEMCPY_TEMPLATE_EADR(sse2, _nobarrier)

MEMSET_TEMPLATE(sse2, clflush, _nobarrier)
MEMSET_TEMPLATE(sse2, clflushopt, _nobarrier)
MEMSET_TEMPLATE(sse2, clwb, _nobarrier)
MEMSET_TEMPLATE_EADR(sse2, _nobarrier)

MEMCPY_TEMPLATE(sse2, clflush, _wcbarrier)
MEMCPY_TEMPLATE(sse2, clflushopt, _wcbarrier)
MEMCPY_TEMPLATE(sse2, clwb, _wcbarrier)
MEMCPY_TEMPLATE_EADR(sse2, _wcbarrier)

MEMSET_TEMPLATE(sse2, clflush, _wcbarrier)
MEMSET_TEMPLATE(sse2, clflushopt, _wcbarrier)
MEMSET_TEMPLATE(sse2, clwb, _wcbarrier)
MEMSET_TEMPLATE_EADR(sse2, _wcbarrier)
#endif

#if AVX_AVAILABLE
MEMCPY_TEMPLATE(avx, clflush, _nobarrier)
MEMCPY_TEMPLATE(avx, clflushopt, _nobarrier)
MEMCPY_TEMPLATE(avx, clwb, _nobarrier)
MEMCPY_TEMPLATE_EADR(avx, _nobarrier)

MEMSET_TEMPLATE(avx, clflush, _nobarrier)
MEMSET_TEMPLATE(avx, clflushopt, _nobarrier)
MEMSET_TEMPLATE(avx, clwb, _nobarrier)
MEMSET_TEMPLATE_EADR(avx, _nobarrier)

MEMCPY_TEMPLATE(avx, clflush, _wcbarrier)
MEMCPY_TEMPLATE(avx, clflushopt, _wcbarrier)
MEMCPY_TEMPLATE(avx, clwb, _wcbarrier)
MEMCPY_TEMPLATE_EADR(avx, _wcbarrier)

MEMSET_TEMPLATE(avx, clflush, _wcbarrier)
MEMSET_TEMPLATE(avx, clflushopt, _wcbarrier)
MEMSET_TEMPLATE(avx, clwb, _wcbarrier)
MEMSET_TEMPLATE_EADR(avx, _wcbarrier)
#endif

#if AVX512F_AVAILABLE
MEMCPY_TEMPLATE(avx512f, clflush, /* cstyle wa */)
MEMCPY_TEMPLATE(avx512f, clflushopt, /* */)
MEMCPY_TEMPLATE(avx512f, clwb, /* */)
MEMCPY_TEMPLATE_EADR(avx512f, /* */)

MEMSET_TEMPLATE(avx512f, clflush, /* */)
MEMSET_TEMPLATE(avx512f, clflushopt, /* */)
MEMSET_TEMPLATE(avx512f, clwb, /* */)
MEMSET_TEMPLATE_EADR(avx512f, /* */)
#endif

enum memcpy_impl {
	MEMCPY_INVALID,
	MEMCPY_SSE2,
	MEMCPY_AVX,
	MEMCPY_AVX512F
};

/*
 * use_sse2_memcpy_memset -- (internal) SSE2 detected, use it if possible
 */
static void
use_sse2_memcpy_memset(struct pmem2_arch_info *info, enum memcpy_impl *impl,
		int wc_workaround)
{
#if SSE2_AVAILABLE
	*impl = MEMCPY_SSE2;
	if (wc_workaround) {
		info->memmove_nodrain_eadr =
				memmove_nodrain_sse2_eadr_wcbarrier;
		if (info->flush == flush_clflush)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clflush_wcbarrier;
		else if (info->flush == flush_clflushopt)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clflushopt_wcbarrier;
		else if (info->flush == flush_clwb)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clwb_wcbarrier;
		else
			ASSERT(0);

		info->memset_nodrain_eadr = memset_nodrain_sse2_eadr_wcbarrier;
		if (info->flush == flush_clflush)
			info->memset_nodrain =
				memset_nodrain_sse2_clflush_wcbarrier;
		else if (info->flush == flush_clflushopt)
			info->memset_nodrain =
				memset_nodrain_sse2_clflushopt_wcbarrier;
		else if (info->flush == flush_clwb)
			info->memset_nodrain =
				memset_nodrain_sse2_clwb_wcbarrier;
		else
			ASSERT(0);
	} else {
		info->memmove_nodrain_eadr =
				memmove_nodrain_sse2_eadr_nobarrier;
		if (info->flush == flush_clflush)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clflush_nobarrier;
		else if (info->flush == flush_clflushopt)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clflushopt_nobarrier;
		else if (info->flush == flush_clwb)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clwb_nobarrier;
		else
			ASSERT(0);

		info->memset_nodrain_eadr =
				memset_nodrain_sse2_eadr_nobarrier;
		if (info->flush == flush_clflush)
			info->memset_nodrain =
				memset_nodrain_sse2_clflush_nobarrier;
		else if (info->flush == flush_clflushopt)
			info->memset_nodrain =
				memset_nodrain_sse2_clflushopt_nobarrier;
		else if (info->flush == flush_clwb)
			info->memset_nodrain =
				memset_nodrain_sse2_clwb_nobarrier;
		else
			ASSERT(0);
	}

#else
	LOG(3, "sse2 disabled at build time");
#endif

}

/*
 * use_avx_memcpy_memset -- (internal) AVX detected, use it if possible
 */
static void
use_avx_memcpy_memset(struct pmem2_arch_info *info, enum memcpy_impl *impl,
		int wc_workaround)
{
#if AVX_AVAILABLE
	LOG(3, "avx supported");

	char *e = os_getenv("PMEM_AVX");
	if (e != NULL && strcmp(e, "0") == 0) {
		LOG(3, "PMEM_AVX set to 0");
		return;
	}

	LOG(3, "PMEM_AVX enabled");
	*impl = MEMCPY_AVX;

	if (wc_workaround) {
		info->memmove_nodrain_eadr =
				memmove_nodrain_avx_eadr_wcbarrier;
		if (info->flush == flush_clflush)
			info->memmove_nodrain =
				memmove_nodrain_avx_clflush_wcbarrier;
		else if (info->flush == flush_clflushopt)
			info->memmove_nodrain =
				memmove_nodrain_avx_clflushopt_wcbarrier;
		else if (info->flush == flush_clwb)
			info->memmove_nodrain =
				memmove_nodrain_avx_clwb_wcbarrier;
		else
			ASSERT(0);

		info->memset_nodrain_eadr =
				memset_nodrain_avx_eadr_wcbarrier;
		if (info->flush == flush_clflush)
			info->memset_nodrain =
				memset_nodrain_avx_clflush_wcbarrier;
		else if (info->flush == flush_clflushopt)
			info->memset_nodrain =
				memset_nodrain_avx_clflushopt_wcbarrier;
		else if (info->flush == flush_clwb)
			info->memset_nodrain =
				memset_nodrain_avx_clwb_wcbarrier;
		else
			ASSERT(0);
	} else {
		info->memmove_nodrain_eadr =
				memmove_nodrain_avx_eadr_nobarrier;
		if (info->flush == flush_clflush)
			info->memmove_nodrain =
				memmove_nodrain_avx_clflush_nobarrier;
		else if (info->flush == flush_clflushopt)
			info->memmove_nodrain =
				memmove_nodrain_avx_clflushopt_nobarrier;
		else if (info->flush == flush_clwb)
			info->memmove_nodrain =
				memmove_nodrain_avx_clwb_nobarrier;
		else
			ASSERT(0);

		info->memset_nodrain_eadr =
				memset_nodrain_avx_eadr_nobarrier;
		if (info->flush == flush_clflush)
			info->memset_nodrain =
				memset_nodrain_avx_clflush_nobarrier;
		else if (info->flush == flush_clflushopt)
			info->memset_nodrain =
				memset_nodrain_avx_clflushopt_nobarrier;
		else if (info->flush == flush_clwb)
			info->memset_nodrain =
				memset_nodrain_avx_clwb_nobarrier;
		else
			ASSERT(0);
	}
#else
	LOG(3, "avx supported, but disabled at build time");
#endif
}

/*
 * use_avx512f_memcpy_memset -- (internal) AVX512F detected, use it if possible
 */
static void
use_avx512f_memcpy_memset(struct pmem2_arch_info *info,
		enum memcpy_impl *impl)
{
#if AVX512F_AVAILABLE
	LOG(3, "avx512f supported");

	char *e = os_getenv("PMEM_AVX512F");
	if (e != NULL && strcmp(e, "0") == 0) {
		LOG(3, "PMEM_AVX512F set to 0");
		return;
	}

	LOG(3, "PMEM_AVX512F enabled");
	*impl = MEMCPY_AVX512F;

	info->memmove_nodrain_eadr = memmove_nodrain_avx512f_eadr;
	if (info->flush == flush_clflush)
		info->memmove_nodrain = memmove_nodrain_avx512f_clflush;
	else if (info->flush == flush_clflushopt)
		info->memmove_nodrain = memmove_nodrain_avx512f_clflushopt;
	else if (info->flush == flush_clwb)
		info->memmove_nodrain = memmove_nodrain_avx512f_clwb;
	else
		ASSERT(0);

	info->memset_nodrain_eadr = memset_nodrain_avx512f_eadr;
	if (info->flush == flush_clflush)
		info->memset_nodrain = memset_nodrain_avx512f_clflush;
	else if (info->flush == flush_clflushopt)
		info->memset_nodrain = memset_nodrain_avx512f_clflushopt;
	else if (info->flush == flush_clwb)
		info->memset_nodrain = memset_nodrain_avx512f_clwb;
	else
		ASSERT(0);
#else
	LOG(3, "avx512f supported, but disabled at build time");
#endif
}

/*
 * pmem_get_cpuinfo -- configure libpmem based on CPUID
 */
static void
pmem_cpuinfo_to_funcs(struct pmem2_arch_info *info, enum memcpy_impl *impl)
{
	LOG(3, NULL);

	if (is_cpu_clflush_present()) {
		LOG(3, "clflush supported");

		info->flush = flush_clflush;
		info->flush_has_builtin_fence = 1;
		info->fence = memory_barrier;
	}

	if (is_cpu_clflushopt_present()) {
		LOG(3, "clflushopt supported");

		char *e = os_getenv("PMEM_NO_CLFLUSHOPT");
		if (e && strcmp(e, "1") == 0) {
			LOG(3, "PMEM_NO_CLFLUSHOPT forced no clflushopt");
		} else {
			info->flush = flush_clflushopt;
			info->flush_has_builtin_fence = 0;
			info->fence = memory_barrier;
		}
	}

	if (is_cpu_clwb_present()) {
		LOG(3, "clwb supported");

		char *e = os_getenv("PMEM_NO_CLWB");
		if (e && strcmp(e, "1") == 0) {
			LOG(3, "PMEM_NO_CLWB forced no clwb");
		} else {
			info->flush = flush_clwb;
			info->flush_has_builtin_fence = 0;
			info->fence = memory_barrier;
		}
	}

	/*
	 * XXX Disable this work around for Intel CPUs with optimized
	 * WC eviction.
	 */
	int wc_workaround = is_cpu_genuine_intel();

	char *ptr = os_getenv("PMEM_WC_WORKAROUND");
	if (ptr) {
		if (strcmp(ptr, "1") == 0) {
			LOG(3, "WC workaround forced to 1");
			wc_workaround = 1;
		} else if (strcmp(ptr, "0") == 0) {
			LOG(3, "WC workaround forced to 0");
			wc_workaround = 0;
		} else {
			LOG(3, "incorrect value of PMEM_WC_WORKAROUND (%s)",
				ptr);
		}
	}
	LOG(3, "WC workaround = %d", wc_workaround);

	ptr = os_getenv("PMEM_NO_MOVNT");
	if (ptr && strcmp(ptr, "1") == 0) {
		LOG(3, "PMEM_NO_MOVNT forced no movnt");
	} else {
		use_sse2_memcpy_memset(info, impl, wc_workaround);

		if (is_cpu_avx_present())
			use_avx_memcpy_memset(info, impl, wc_workaround);

		if (is_cpu_avx512f_present())
			use_avx512f_memcpy_memset(info, impl);
	}
}

/*
 * pmem2_arch_init -- initialize architecture-specific list of pmem operations
 */
void
pmem2_arch_init(struct pmem2_arch_info *info)
{
	LOG(3, NULL);
	enum memcpy_impl impl = MEMCPY_INVALID;

	pmem_cpuinfo_to_funcs(info, &impl);

	/*
	 * For testing, allow overriding the default threshold
	 * for using non-temporal stores in pmem_memcpy_*(), pmem_memmove_*()
	 * and pmem_memset_*().
	 * It has no effect if movnt is not supported or disabled.
	 */
	const char *ptr = os_getenv("PMEM_MOVNT_THRESHOLD");
	if (ptr) {
		long long val = atoll(ptr);

		if (val < 0) {
			LOG(3, "Invalid PMEM_MOVNT_THRESHOLD");
		} else {
			LOG(3, "PMEM_MOVNT_THRESHOLD set to %zu", (size_t)val);
			Movnt_threshold = (size_t)val;
		}
	}

	if (info->flush == flush_clwb)
		LOG(3, "using clwb");
	else if (info->flush == flush_clflushopt)
		LOG(3, "using clflushopt");
	else if (info->flush == flush_clflush)
		LOG(3, "using clflush");
	else
		FATAL("invalid deep flush function address");

	if (impl == MEMCPY_AVX512F)
		LOG(3, "using movnt AVX512F");
	else if (impl == MEMCPY_AVX)
		LOG(3, "using movnt AVX");
	else if (impl == MEMCPY_SSE2)
		LOG(3, "using movnt SSE2");
}
Commit	Line	Data
a4b75251 TL	1	// SPDX-License-Identifier: BSD-3-Clause
	2	/* Copyright 2014-2020, Intel Corporation */
	3
	4	#include <string.h>
	5	#include <xmmintrin.h>
	6
	7	#include "auto_flush.h"
	8	#include "cpu.h"
	9	#include "flush.h"
	10	#include "memcpy_memset.h"
	11	#include "os.h"
	12	#include "out.h"
	13	#include "pmem2_arch.h"
	14	#include "valgrind_internal.h"
	15
	16	#define MOVNT_THRESHOLD 256
	17
	18	size_t Movnt_threshold = MOVNT_THRESHOLD;
	19
	20	/*
	21	* memory_barrier -- (internal) issue the fence instruction
	22	*/
	23	static void
	24	memory_barrier(void)
	25	{
	26	LOG(15, NULL);
	27	_mm_sfence(); /* ensure CLWB or CLFLUSHOPT completes */
	28	}
	29
	30	/*
	31	* flush_clflush -- (internal) flush the CPU cache, using clflush
	32	*/
	33	static void
	34	flush_clflush(const void *addr, size_t len)
	35	{
	36	LOG(15, "addr %p len %zu", addr, len);
	37
	38	flush_clflush_nolog(addr, len);
	39	}
	40
	41	/*
	42	* flush_clflushopt -- (internal) flush the CPU cache, using clflushopt
	43	*/
	44	static void
	45	flush_clflushopt(const void *addr, size_t len)
	46	{
	47	LOG(15, "addr %p len %zu", addr, len);
	48
	49	flush_clflushopt_nolog(addr, len);
	50	}
	51
	52	/*
	53	* flush_clwb -- (internal) flush the CPU cache, using clwb
	54	*/
	55	static void
	56	flush_clwb(const void *addr, size_t len)
	57	{
	58	LOG(15, "addr %p len %zu", addr, len);
	59
	60	flush_clwb_nolog(addr, len);
	61	}
	62
	63	#if SSE2_AVAILABLE \|\| AVX_AVAILABLE \|\| AVX512F_AVAILABLE
	64	#define PMEM2_F_MEM_MOVNT (PMEM2_F_MEM_WC \| PMEM2_F_MEM_NONTEMPORAL)
65	#define PMEM2_F_MEM_MOV (PMEM2_F_MEM_WB \| PMEM2_F_MEM_TEMPORAL)
66
67	#define MEMCPY_TEMPLATE(isa, flush, perfbarrier) \
68	static void *\
69	memmove_nodrain_##isa##_##flush##perfbarrier(void dest, const void src, \
70	size_t len, unsigned flags, flush_func flushf)\
71	{\
72	if (len == 0 \|\| src == dest)\
73	return dest;\
74	\
75	if (flags & PMEM2_F_MEM_NOFLUSH) \
76	memmove_mov_##isa##_noflush(dest, src, len); \
77	else if (flags & PMEM2_F_MEM_MOVNT)\
78	memmove_movnt_##isa ##_##flush##perfbarrier(dest, src, len);\
79	else if (flags & PMEM2_F_MEM_MOV)\
80	memmove_mov_##isa##_##flush(dest, src, len);\
81	else if (len < Movnt_threshold)\
82	memmove_mov_##isa##_##flush(dest, src, len);\
83	else\
84	memmove_movnt_##isa##_##flush##perfbarrier(dest, src, len);\
85	\
86	return dest;\
87	}
88
89	#define MEMCPY_TEMPLATE_EADR(isa, perfbarrier) \
90	static void *\
91	memmove_nodrain_##isa##_eadr##perfbarrier(void dest, const void src, \
92	size_t len, unsigned flags, flush_func flushf)\
93	{\
94	if (len == 0 \|\| src == dest)\
95	return dest;\
96	\
97	if (flags & PMEM2_F_MEM_NOFLUSH)\
98	memmove_mov_##isa##_noflush(dest, src, len);\
99	else if (flags & PMEM2_F_MEM_NONTEMPORAL)\
100	memmove_movnt_##isa##_empty##perfbarrier(dest, src, len);\
101	else\
102	memmove_mov_##isa##_empty(dest, src, len);\
103	\
104	return dest;\
105	}
106
107	#define MEMSET_TEMPLATE(isa, flush, perfbarrier)\
108	static void *\
109	memset_nodrain_##isa##_##flush##perfbarrier(void *dest, int c, size_t len, \
110	unsigned flags, flush_func flushf)\
111	{\
112	if (len == 0)\
113	return dest;\
114	\
115	if (flags & PMEM2_F_MEM_NOFLUSH) \
116	memset_mov_##isa##_noflush(dest, c, len); \
117	else if (flags & PMEM2_F_MEM_MOVNT)\
118	memset_movnt_##isa##_##flush##perfbarrier(dest, c, len);\
119	else if (flags & PMEM2_F_MEM_MOV)\
120	memset_mov_##isa##_##flush(dest, c, len);\
121	else if (len < Movnt_threshold)\
122	memset_mov_##isa##_##flush(dest, c, len);\
123	else\
124	memset_movnt_##isa##_##flush##perfbarrier(dest, c, len);\
125	\
126	return dest;\
127	}
128
129	#define MEMSET_TEMPLATE_EADR(isa, perfbarrier) \
130	static void *\
131	memset_nodrain_##isa##_eadr##perfbarrier(void *dest, int c, size_t len, \
132	unsigned flags, flush_func flushf)\
133	{\
134	if (len == 0)\
135	return dest;\
136	\
137	if (flags & PMEM2_F_MEM_NOFLUSH)\
138	memset_mov_##isa##_noflush(dest, c, len);\
139	else if (flags & PMEM2_F_MEM_NONTEMPORAL)\
140	memset_movnt_##isa##_empty##perfbarrier(dest, c, len);\
141	else\
142	memset_mov_##isa##_empty(dest, c, len);\
143	\
144	return dest;\
145	}
146	#endif
147
148	#if SSE2_AVAILABLE
149	MEMCPY_TEMPLATE(sse2, clflush, _nobarrier)
150	MEMCPY_TEMPLATE(sse2, clflushopt, _nobarrier)
151	MEMCPY_TEMPLATE(sse2, clwb, _nobarrier)
152	MEMCPY_TEMPLATE_EADR(sse2, _nobarrier)
153
154	MEMSET_TEMPLATE(sse2, clflush, _nobarrier)
155	MEMSET_TEMPLATE(sse2, clflushopt, _nobarrier)
156	MEMSET_TEMPLATE(sse2, clwb, _nobarrier)
157	MEMSET_TEMPLATE_EADR(sse2, _nobarrier)
158
159	MEMCPY_TEMPLATE(sse2, clflush, _wcbarrier)
160	MEMCPY_TEMPLATE(sse2, clflushopt, _wcbarrier)
161	MEMCPY_TEMPLATE(sse2, clwb, _wcbarrier)
162	MEMCPY_TEMPLATE_EADR(sse2, _wcbarrier)
163
164	MEMSET_TEMPLATE(sse2, clflush, _wcbarrier)
165	MEMSET_TEMPLATE(sse2, clflushopt, _wcbarrier)
166	MEMSET_TEMPLATE(sse2, clwb, _wcbarrier)
167	MEMSET_TEMPLATE_EADR(sse2, _wcbarrier)
168	#endif
169
170	#if AVX_AVAILABLE
171	MEMCPY_TEMPLATE(avx, clflush, _nobarrier)
172	MEMCPY_TEMPLATE(avx, clflushopt, _nobarrier)
173	MEMCPY_TEMPLATE(avx, clwb, _nobarrier)
174	MEMCPY_TEMPLATE_EADR(avx, _nobarrier)
175
176	MEMSET_TEMPLATE(avx, clflush, _nobarrier)
177	MEMSET_TEMPLATE(avx, clflushopt, _nobarrier)
178	MEMSET_TEMPLATE(avx, clwb, _nobarrier)
179	MEMSET_TEMPLATE_EADR(avx, _nobarrier)
180
181	MEMCPY_TEMPLATE(avx, clflush, _wcbarrier)
182	MEMCPY_TEMPLATE(avx, clflushopt, _wcbarrier)
183	MEMCPY_TEMPLATE(avx, clwb, _wcbarrier)
184	MEMCPY_TEMPLATE_EADR(avx, _wcbarrier)
185
186	MEMSET_TEMPLATE(avx, clflush, _wcbarrier)
187	MEMSET_TEMPLATE(avx, clflushopt, _wcbarrier)
188	MEMSET_TEMPLATE(avx, clwb, _wcbarrier)
189	MEMSET_TEMPLATE_EADR(avx, _wcbarrier)
190	#endif
191
192	#if AVX512F_AVAILABLE
193	MEMCPY_TEMPLATE(avx512f, clflush, /* cstyle wa */)
194	MEMCPY_TEMPLATE(avx512f, clflushopt, /* */)
195	MEMCPY_TEMPLATE(avx512f, clwb, /* */)
196	MEMCPY_TEMPLATE_EADR(avx512f, /* */)
197
198	MEMSET_TEMPLATE(avx512f, clflush, /* */)
199	MEMSET_TEMPLATE(avx512f, clflushopt, /* */)
200	MEMSET_TEMPLATE(avx512f, clwb, /* */)
201	MEMSET_TEMPLATE_EADR(avx512f, /* */)
202	#endif
203
204	enum memcpy_impl {
205	MEMCPY_INVALID,
206	MEMCPY_SSE2,
207	MEMCPY_AVX,
208	MEMCPY_AVX512F
209	};
210
211	/*
212	* use_sse2_memcpy_memset -- (internal) SSE2 detected, use it if possible
213	*/
214	static void
215	use_sse2_memcpy_memset(struct pmem2_arch_info info, enum memcpy_impl impl,
216	int wc_workaround)
217	{
218	#if SSE2_AVAILABLE
219	*impl = MEMCPY_SSE2;
220	if (wc_workaround) {
221	info->memmove_nodrain_eadr =
222	memmove_nodrain_sse2_eadr_wcbarrier;
223	if (info->flush == flush_clflush)
224	info->memmove_nodrain =
225	memmove_nodrain_sse2_clflush_wcbarrier;
226	else if (info->flush == flush_clflushopt)
227	info->memmove_nodrain =
228	memmove_nodrain_sse2_clflushopt_wcbarrier;
229	else if (info->flush == flush_clwb)
230	info->memmove_nodrain =
231	memmove_nodrain_sse2_clwb_wcbarrier;
232	else
233	ASSERT(0);
234
235	info->memset_nodrain_eadr = memset_nodrain_sse2_eadr_wcbarrier;
236	if (info->flush == flush_clflush)
237	info->memset_nodrain =
238	memset_nodrain_sse2_clflush_wcbarrier;
239	else if (info->flush == flush_clflushopt)
240	info->memset_nodrain =
241	memset_nodrain_sse2_clflushopt_wcbarrier;
242	else if (info->flush == flush_clwb)
243	info->memset_nodrain =
244	memset_nodrain_sse2_clwb_wcbarrier;
245	else
246	ASSERT(0);
247	} else {
248	info->memmove_nodrain_eadr =
249	memmove_nodrain_sse2_eadr_nobarrier;
250	if (info->flush == flush_clflush)
251	info->memmove_nodrain =
252	memmove_nodrain_sse2_clflush_nobarrier;
253	else if (info->flush == flush_clflushopt)
254	info->memmove_nodrain =
255	memmove_nodrain_sse2_clflushopt_nobarrier;
256	else if (info->flush == flush_clwb)
257	info->memmove_nodrain =
258	memmove_nodrain_sse2_clwb_nobarrier;
259	else
260	ASSERT(0);
261
262	info->memset_nodrain_eadr =
263	memset_nodrain_sse2_eadr_nobarrier;
264	if (info->flush == flush_clflush)
265	info->memset_nodrain =
266	memset_nodrain_sse2_clflush_nobarrier;
267	else if (info->flush == flush_clflushopt)
268	info->memset_nodrain =
269	memset_nodrain_sse2_clflushopt_nobarrier;
270	else if (info->flush == flush_clwb)
271	info->memset_nodrain =
272	memset_nodrain_sse2_clwb_nobarrier;
273	else
274	ASSERT(0);
275	}
276
277	#else
278	LOG(3, "sse2 disabled at build time");
279	#endif
280
281	}
282
283	/*
284	* use_avx_memcpy_memset -- (internal) AVX detected, use it if possible
285	*/
286	static void
287	use_avx_memcpy_memset(struct pmem2_arch_info info, enum memcpy_impl impl,
288	int wc_workaround)
289	{
290	#if AVX_AVAILABLE
291	LOG(3, "avx supported");
292
293	char *e = os_getenv("PMEM_AVX");
294	if (e != NULL && strcmp(e, "0") == 0) {
295	LOG(3, "PMEM_AVX set to 0");
296	return;
297	}
298
299	LOG(3, "PMEM_AVX enabled");
300	*impl = MEMCPY_AVX;
301
302	if (wc_workaround) {
303	info->memmove_nodrain_eadr =
304	memmove_nodrain_avx_eadr_wcbarrier;
305	if (info->flush == flush_clflush)
306	info->memmove_nodrain =
307	memmove_nodrain_avx_clflush_wcbarrier;
308	else if (info->flush == flush_clflushopt)
309	info->memmove_nodrain =
310	memmove_nodrain_avx_clflushopt_wcbarrier;
311	else if (info->flush == flush_clwb)
312	info->memmove_nodrain =
313	memmove_nodrain_avx_clwb_wcbarrier;
314	else
315	ASSERT(0);
316
317	info->memset_nodrain_eadr =
318	memset_nodrain_avx_eadr_wcbarrier;
319	if (info->flush == flush_clflush)
320	info->memset_nodrain =
321	memset_nodrain_avx_clflush_wcbarrier;
322	else if (info->flush == flush_clflushopt)
323	info->memset_nodrain =
324	memset_nodrain_avx_clflushopt_wcbarrier;
325	else if (info->flush == flush_clwb)
326	info->memset_nodrain =
327	memset_nodrain_avx_clwb_wcbarrier;
328	else
329	ASSERT(0);
330	} else {
331	info->memmove_nodrain_eadr =
332	memmove_nodrain_avx_eadr_nobarrier;
333	if (info->flush == flush_clflush)
334	info->memmove_nodrain =
335	memmove_nodrain_avx_clflush_nobarrier;
336	else if (info->flush == flush_clflushopt)
337	info->memmove_nodrain =
338	memmove_nodrain_avx_clflushopt_nobarrier;
339	else if (info->flush == flush_clwb)
340	info->memmove_nodrain =
341	memmove_nodrain_avx_clwb_nobarrier;
342	else
343	ASSERT(0);
344
345	info->memset_nodrain_eadr =
346	memset_nodrain_avx_eadr_nobarrier;
347	if (info->flush == flush_clflush)
348	info->memset_nodrain =
349	memset_nodrain_avx_clflush_nobarrier;
350	else if (info->flush == flush_clflushopt)
351	info->memset_nodrain =
352	memset_nodrain_avx_clflushopt_nobarrier;
353	else if (info->flush == flush_clwb)
354	info->memset_nodrain =
355	memset_nodrain_avx_clwb_nobarrier;
356	else
357	ASSERT(0);
358	}
359	#else
360	LOG(3, "avx supported, but disabled at build time");
361	#endif
362	}
363
364	/*
365	* use_avx512f_memcpy_memset -- (internal) AVX512F detected, use it if possible
366	*/
367	static void
368	use_avx512f_memcpy_memset(struct pmem2_arch_info *info,
369	enum memcpy_impl *impl)
370	{
371	#if AVX512F_AVAILABLE
372	LOG(3, "avx512f supported");
373
374	char *e = os_getenv("PMEM_AVX512F");
375	if (e != NULL && strcmp(e, "0") == 0) {
376	LOG(3, "PMEM_AVX512F set to 0");
377	return;
378	}
379
380	LOG(3, "PMEM_AVX512F enabled");
381	*impl = MEMCPY_AVX512F;
382
383	info->memmove_nodrain_eadr = memmove_nodrain_avx512f_eadr;
384	if (info->flush == flush_clflush)
385	info->memmove_nodrain = memmove_nodrain_avx512f_clflush;
386	else if (info->flush == flush_clflushopt)
387	info->memmove_nodrain = memmove_nodrain_avx512f_clflushopt;
388	else if (info->flush == flush_clwb)
389	info->memmove_nodrain = memmove_nodrain_avx512f_clwb;
390	else
391	ASSERT(0);
392
393	info->memset_nodrain_eadr = memset_nodrain_avx512f_eadr;
394	if (info->flush == flush_clflush)
395	info->memset_nodrain = memset_nodrain_avx512f_clflush;
396	else if (info->flush == flush_clflushopt)
397	info->memset_nodrain = memset_nodrain_avx512f_clflushopt;
398	else if (info->flush == flush_clwb)
399	info->memset_nodrain = memset_nodrain_avx512f_clwb;
400	else
401	ASSERT(0);
402	#else
403	LOG(3, "avx512f supported, but disabled at build time");
404	#endif
405	}
406
407	/*
408	* pmem_get_cpuinfo -- configure libpmem based on CPUID
409	*/
410	static void
411	pmem_cpuinfo_to_funcs(struct pmem2_arch_info info, enum memcpy_impl impl)
412	{
413	LOG(3, NULL);
414
415	if (is_cpu_clflush_present()) {
416	LOG(3, "clflush supported");
417
418	info->flush = flush_clflush;
419	info->flush_has_builtin_fence = 1;
420	info->fence = memory_barrier;
421	}
422
423	if (is_cpu_clflushopt_present()) {
424	LOG(3, "clflushopt supported");
425
426	char *e = os_getenv("PMEM_NO_CLFLUSHOPT");
427	if (e && strcmp(e, "1") == 0) {
428	LOG(3, "PMEM_NO_CLFLUSHOPT forced no clflushopt");
429	} else {
430	info->flush = flush_clflushopt;
431	info->flush_has_builtin_fence = 0;
432	info->fence = memory_barrier;
433	}
434	}
435
436	if (is_cpu_clwb_present()) {
437	LOG(3, "clwb supported");
438
439	char *e = os_getenv("PMEM_NO_CLWB");
440	if (e && strcmp(e, "1") == 0) {
441	LOG(3, "PMEM_NO_CLWB forced no clwb");
442	} else {
443	info->flush = flush_clwb;
444	info->flush_has_builtin_fence = 0;
445	info->fence = memory_barrier;
446	}
447	}
448
449	/*
450	* XXX Disable this work around for Intel CPUs with optimized
451	* WC eviction.
452	*/
453	int wc_workaround = is_cpu_genuine_intel();
454
455	char *ptr = os_getenv("PMEM_WC_WORKAROUND");
456	if (ptr) {
457	if (strcmp(ptr, "1") == 0) {
458	LOG(3, "WC workaround forced to 1");
459	wc_workaround = 1;
460	} else if (strcmp(ptr, "0") == 0) {
461	LOG(3, "WC workaround forced to 0");
462	wc_workaround = 0;
463	} else {
464	LOG(3, "incorrect value of PMEM_WC_WORKAROUND (%s)",
465	ptr);
466	}
467	}
468	LOG(3, "WC workaround = %d", wc_workaround);
469
470	ptr = os_getenv("PMEM_NO_MOVNT");
471	if (ptr && strcmp(ptr, "1") == 0) {
472	LOG(3, "PMEM_NO_MOVNT forced no movnt");
473	} else {
474	use_sse2_memcpy_memset(info, impl, wc_workaround);
475
476	if (is_cpu_avx_present())
477	use_avx_memcpy_memset(info, impl, wc_workaround);
478
479	if (is_cpu_avx512f_present())
480	use_avx512f_memcpy_memset(info, impl);
481	}
482	}
483
484	/*
485	* pmem2_arch_init -- initialize architecture-specific list of pmem operations
486	*/
487	void
488	pmem2_arch_init(struct pmem2_arch_info *info)
489	{
490	LOG(3, NULL);
491	enum memcpy_impl impl = MEMCPY_INVALID;
492
493	pmem_cpuinfo_to_funcs(info, &impl);
494
495	/*
496	* For testing, allow overriding the default threshold
497	* for using non-temporal stores in pmem_memcpy_(), pmem_memmove_()
498	* and pmem_memset_*().
499	* It has no effect if movnt is not supported or disabled.
500	*/
501	const char *ptr = os_getenv("PMEM_MOVNT_THRESHOLD");
502	if (ptr) {
503	long long val = atoll(ptr);
504
505	if (val < 0) {
506	LOG(3, "Invalid PMEM_MOVNT_THRESHOLD");
507	} else {
508	LOG(3, "PMEM_MOVNT_THRESHOLD set to %zu", (size_t)val);
509	Movnt_threshold = (size_t)val;
510	}
511	}
512
513	if (info->flush == flush_clwb)
514	LOG(3, "using clwb");
515	else if (info->flush == flush_clflushopt)
516	LOG(3, "using clflushopt");
517	else if (info->flush == flush_clflush)
518	LOG(3, "using clflush");
519	else
520	FATAL("invalid deep flush function address");
521
522	if (impl == MEMCPY_AVX512F)
523	LOG(3, "using movnt AVX512F");
524	else if (impl == MEMCPY_AVX)
525	LOG(3, "using movnt AVX");
526	else if (impl == MEMCPY_SSE2)
527	LOG(3, "using movnt SSE2");
528	}