[mirror_zfs.git] / module / zfs / vdev_raidz_math.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
 */

#include <sys/zfs_context.h>
#include <sys/types.h>
#include <sys/zio.h>
#include <sys/debug.h>
#include <sys/zfs_debug.h>

#include <sys/vdev_raidz.h>
#include <sys/vdev_raidz_impl.h>

extern boolean_t raidz_will_scalar_work(void);

/* Opaque implementation with NULL methods to represent original methods */
static const raidz_impl_ops_t vdev_raidz_original_impl = {
	.name = "original",
	.is_supported = raidz_will_scalar_work,
};

/* RAIDZ parity op that contain the fastest methods */
static raidz_impl_ops_t vdev_raidz_fastest_impl = {
	.name = "fastest"
};

/* All compiled in implementations */
const raidz_impl_ops_t *raidz_all_maths[] = {
	&vdev_raidz_original_impl,
	&vdev_raidz_scalar_impl,
#if defined(__x86_64) && defined(HAVE_SSE2)	/* only x86_64 for now */
	&vdev_raidz_sse2_impl,
#endif
#if defined(__x86_64) && defined(HAVE_SSSE3)	/* only x86_64 for now */
	&vdev_raidz_ssse3_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX2)	/* only x86_64 for now */
	&vdev_raidz_avx2_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX512F)	/* only x86_64 for now */
	&vdev_raidz_avx512f_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX512BW)	/* only x86_64 for now */
	&vdev_raidz_avx512bw_impl,
#endif
#if defined(__aarch64__)
	&vdev_raidz_aarch64_neon_impl,
	&vdev_raidz_aarch64_neonx2_impl,
#endif
};

/* Indicate that benchmark has been completed */
static boolean_t raidz_math_initialized = B_FALSE;

/* Select raidz implementation */
#define	IMPL_FASTEST	(UINT32_MAX)
#define	IMPL_CYCLE	(UINT32_MAX - 1)
#define	IMPL_ORIGINAL	(0)
#define	IMPL_SCALAR	(1)

#define	RAIDZ_IMPL_READ(i)	(*(volatile uint32_t *) &(i))

static uint32_t zfs_vdev_raidz_impl = IMPL_SCALAR;
static uint32_t user_sel_impl = IMPL_FASTEST;

/* Hold all supported implementations */
static size_t raidz_supp_impl_cnt = 0;
static raidz_impl_ops_t *raidz_supp_impl[ARRAY_SIZE(raidz_all_maths)];

/*
 * kstats values for supported implementations
 * Values represent per disk throughput of 8 disk+parity raidz vdev [B/s]
 */
static raidz_impl_kstat_t raidz_impl_kstats[ARRAY_SIZE(raidz_all_maths) + 1];

/* kstat for benchmarked implementations */
static kstat_t *raidz_math_kstat = NULL;

/*
 * Selects the raidz operation for raidz_map
 * If rm_ops is set to NULL original raidz implementation will be used
 */
raidz_impl_ops_t *
vdev_raidz_math_get_ops()
{
	raidz_impl_ops_t *ops = NULL;
	const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl);

	switch (impl) {
	case IMPL_FASTEST:
		ASSERT(raidz_math_initialized);
		ops = &vdev_raidz_fastest_impl;
		break;
#if !defined(_KERNEL)
	case IMPL_CYCLE:
	{
		ASSERT(raidz_math_initialized);
		ASSERT3U(raidz_supp_impl_cnt, >, 0);
		/* Cycle through all supported implementations */
		static size_t cycle_impl_idx = 0;
		size_t idx = (++cycle_impl_idx) % raidz_supp_impl_cnt;
		ops = raidz_supp_impl[idx];
	}
	break;
#endif
	case IMPL_ORIGINAL:
		ops = (raidz_impl_ops_t *)&vdev_raidz_original_impl;
		break;
	case IMPL_SCALAR:
		ops = (raidz_impl_ops_t *)&vdev_raidz_scalar_impl;
		break;
	default:
		ASSERT3U(impl, <, raidz_supp_impl_cnt);
		ASSERT3U(raidz_supp_impl_cnt, >, 0);
		if (impl < ARRAY_SIZE(raidz_all_maths))
			ops = raidz_supp_impl[impl];
		break;
	}

	ASSERT3P(ops, !=, NULL);

	return (ops);
}

/*
 * Select parity generation method for raidz_map
 */
int
vdev_raidz_math_generate(raidz_map_t *rm)
{
	raidz_gen_f gen_parity = NULL;

	switch (raidz_parity(rm)) {
		case 1:
			gen_parity = rm->rm_ops->gen[RAIDZ_GEN_P];
			break;
		case 2:
			gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQ];
			break;
		case 3:
			gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQR];
			break;
		default:
			gen_parity = NULL;
			cmn_err(CE_PANIC, "invalid RAID-Z configuration %d",
			    raidz_parity(rm));
			break;
	}

	/* if method is NULL execute the original implementation */
	if (gen_parity == NULL)
		return (RAIDZ_ORIGINAL_IMPL);

	gen_parity(rm);

	return (0);
}

static raidz_rec_f
reconstruct_fun_p_sel(raidz_map_t *rm, const int *parity_valid,
    const int nbaddata)
{
	if (nbaddata == 1 && parity_valid[CODE_P]) {
		return (rm->rm_ops->rec[RAIDZ_REC_P]);
	}
	return ((raidz_rec_f) NULL);
}

static raidz_rec_f
reconstruct_fun_pq_sel(raidz_map_t *rm, const int *parity_valid,
    const int nbaddata)
{
	if (nbaddata == 1) {
		if (parity_valid[CODE_P]) {
			return (rm->rm_ops->rec[RAIDZ_REC_P]);
		} else if (parity_valid[CODE_Q]) {
			return (rm->rm_ops->rec[RAIDZ_REC_Q]);
		}
	} else if (nbaddata == 2 &&
	    parity_valid[CODE_P] && parity_valid[CODE_Q]) {
		return (rm->rm_ops->rec[RAIDZ_REC_PQ]);
	}
	return ((raidz_rec_f) NULL);
}

static raidz_rec_f
reconstruct_fun_pqr_sel(raidz_map_t *rm, const int *parity_valid,
    const int nbaddata)
{
	if (nbaddata == 1) {
		if (parity_valid[CODE_P]) {
			return (rm->rm_ops->rec[RAIDZ_REC_P]);
		} else if (parity_valid[CODE_Q]) {
			return (rm->rm_ops->rec[RAIDZ_REC_Q]);
		} else if (parity_valid[CODE_R]) {
			return (rm->rm_ops->rec[RAIDZ_REC_R]);
		}
	} else if (nbaddata == 2) {
		if (parity_valid[CODE_P] && parity_valid[CODE_Q]) {
			return (rm->rm_ops->rec[RAIDZ_REC_PQ]);
		} else if (parity_valid[CODE_P] && parity_valid[CODE_R]) {
			return (rm->rm_ops->rec[RAIDZ_REC_PR]);
		} else if (parity_valid[CODE_Q] && parity_valid[CODE_R]) {
			return (rm->rm_ops->rec[RAIDZ_REC_QR]);
		}
	} else if (nbaddata == 3 &&
	    parity_valid[CODE_P] && parity_valid[CODE_Q] &&
	    parity_valid[CODE_R]) {
		return (rm->rm_ops->rec[RAIDZ_REC_PQR]);
	}
	return ((raidz_rec_f) NULL);
}

/*
 * Select data reconstruction method for raidz_map
 * @parity_valid - Parity validity flag
 * @dt           - Failed data index array
 * @nbaddata     - Number of failed data columns
 */
int
vdev_raidz_math_reconstruct(raidz_map_t *rm, const int *parity_valid,
    const int *dt, const int nbaddata)
{
	raidz_rec_f rec_fn = NULL;

	switch (raidz_parity(rm)) {
	case PARITY_P:
		rec_fn = reconstruct_fun_p_sel(rm, parity_valid, nbaddata);
		break;
	case PARITY_PQ:
		rec_fn = reconstruct_fun_pq_sel(rm, parity_valid, nbaddata);
		break;
	case PARITY_PQR:
		rec_fn = reconstruct_fun_pqr_sel(rm, parity_valid, nbaddata);
		break;
	default:
		cmn_err(CE_PANIC, "invalid RAID-Z configuration %d",
		    raidz_parity(rm));
		break;
	}

	if (rec_fn == NULL)
		return (RAIDZ_ORIGINAL_IMPL);
	else
		return (rec_fn(rm, dt));
}

const char *raidz_gen_name[] = {
	"gen_p", "gen_pq", "gen_pqr"
};
const char *raidz_rec_name[] = {
	"rec_p", "rec_q", "rec_r",
	"rec_pq", "rec_pr", "rec_qr", "rec_pqr"
};

#define	RAIDZ_KSTAT_LINE_LEN	(17 + 10*12 + 1)

static int
raidz_math_kstat_headers(char *buf, size_t size)
{
	int i;
	ssize_t off;

	ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN);

	off = snprintf(buf, size, "%-17s", "implementation");

	for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++)
		off += snprintf(buf + off, size - off, "%-16s",
		    raidz_gen_name[i]);

	for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++)
		off += snprintf(buf + off, size - off, "%-16s",
		    raidz_rec_name[i]);

	(void) snprintf(buf + off, size - off, "\n");

	return (0);
}

static int
raidz_math_kstat_data(char *buf, size_t size, void *data)
{
	raidz_impl_kstat_t *fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
	raidz_impl_kstat_t *cstat = (raidz_impl_kstat_t *)data;
	ssize_t off = 0;
	int i;

	ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN);

	if (cstat == fstat) {
		off += snprintf(buf + off, size - off, "%-17s", "fastest");

		for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) {
			int id = fstat->gen[i];
			off += snprintf(buf + off, size - off, "%-16s",
			    raidz_supp_impl[id]->name);
		}
		for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) {
			int id = fstat->rec[i];
			off += snprintf(buf + off, size - off, "%-16s",
			    raidz_supp_impl[id]->name);
		}
	} else {
		ptrdiff_t id = cstat - raidz_impl_kstats;

		off += snprintf(buf + off, size - off, "%-17s",
		    raidz_supp_impl[id]->name);

		for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++)
			off += snprintf(buf + off, size - off, "%-16llu",
			    (u_longlong_t)cstat->gen[i]);

		for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++)
			off += snprintf(buf + off, size - off, "%-16llu",
			    (u_longlong_t)cstat->rec[i]);
	}

	(void) snprintf(buf + off, size - off, "\n");

	return (0);
}

static void *
raidz_math_kstat_addr(kstat_t *ksp, loff_t n)
{
	if (n <= raidz_supp_impl_cnt)
		ksp->ks_private = (void *) (raidz_impl_kstats + n);
	else
		ksp->ks_private = NULL;

	return (ksp->ks_private);
}

#define	BENCH_D_COLS	(8ULL)
#define	BENCH_COLS	(BENCH_D_COLS + PARITY_PQR)
#define	BENCH_ZIO_SIZE	(1ULL << SPA_OLD_MAXBLOCKSHIFT)	/* 128 kiB */
#define	BENCH_NS	MSEC2NSEC(25)			/* 25ms */

typedef void (*benchmark_fn)(raidz_map_t *rm, const int fn);

static void
benchmark_gen_impl(raidz_map_t *rm, const int fn)
{
	(void) fn;
	vdev_raidz_generate_parity(rm);
}

static void
benchmark_rec_impl(raidz_map_t *rm, const int fn)
{
	static const int rec_tgt[7][3] = {
		{1, 2, 3},	/* rec_p:   bad QR & D[0]	*/
		{0, 2, 3},	/* rec_q:   bad PR & D[0]	*/
		{0, 1, 3},	/* rec_r:   bad PQ & D[0]	*/
		{2, 3, 4},	/* rec_pq:  bad R  & D[0][1]	*/
		{1, 3, 4},	/* rec_pr:  bad Q  & D[0][1]	*/
		{0, 3, 4},	/* rec_qr:  bad P  & D[0][1]	*/
		{3, 4, 5}	/* rec_pqr: bad    & D[0][1][2] */
	};

	vdev_raidz_reconstruct(rm, rec_tgt[fn], 3);
}

/*
 * Benchmarking of all supported implementations (raidz_supp_impl_cnt)
 * is performed by setting the rm_ops pointer and calling the top level
 * generate/reconstruct methods of bench_rm.
 */
static void
benchmark_raidz_impl(raidz_map_t *bench_rm, const int fn, benchmark_fn bench_fn)
{
	uint64_t run_cnt, speed, best_speed = 0;
	hrtime_t t_start, t_diff;
	raidz_impl_ops_t *curr_impl;
	raidz_impl_kstat_t *fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
	int impl, i;

	for (impl = 0; impl < raidz_supp_impl_cnt; impl++) {
		/* set an implementation to benchmark */
		curr_impl = raidz_supp_impl[impl];
		bench_rm->rm_ops = curr_impl;

		run_cnt = 0;
		t_start = gethrtime();

		do {
			for (i = 0; i < 25; i++, run_cnt++)
				bench_fn(bench_rm, fn);

			t_diff = gethrtime() - t_start;
		} while (t_diff < BENCH_NS);

		speed = run_cnt * BENCH_ZIO_SIZE * NANOSEC;
		speed /= (t_diff * BENCH_COLS);

		if (bench_fn == benchmark_gen_impl)
			raidz_impl_kstats[impl].gen[fn] = speed;
		else
			raidz_impl_kstats[impl].rec[fn] = speed;

		/* Update fastest implementation method */
		if (speed > best_speed) {
			best_speed = speed;

			if (bench_fn == benchmark_gen_impl) {
				fstat->gen[fn] = impl;
				vdev_raidz_fastest_impl.gen[fn] =
				    curr_impl->gen[fn];
			} else {
				fstat->rec[fn] = impl;
				vdev_raidz_fastest_impl.rec[fn] =
				    curr_impl->rec[fn];
			}
		}
	}
}

void
vdev_raidz_math_init(void)
{
	raidz_impl_ops_t *curr_impl;
	zio_t *bench_zio = NULL;
	raidz_map_t *bench_rm = NULL;
	uint64_t bench_parity;
	int i, c, fn;

	/* move supported impl into raidz_supp_impl */
	for (i = 0, c = 0; i < ARRAY_SIZE(raidz_all_maths); i++) {
		curr_impl = (raidz_impl_ops_t *)raidz_all_maths[i];

		/* initialize impl */
		if (curr_impl->init)
			curr_impl->init();

		if (curr_impl->is_supported())
			raidz_supp_impl[c++] = (raidz_impl_ops_t *)curr_impl;
	}
	membar_producer();		/* complete raidz_supp_impl[] init */
	raidz_supp_impl_cnt = c;	/* number of supported impl */

#if !defined(_KERNEL)
	/* Skip benchmarking and use last implementation as fastest */
	memcpy(&vdev_raidz_fastest_impl, raidz_supp_impl[raidz_supp_impl_cnt-1],
	    sizeof (vdev_raidz_fastest_impl));
	strcpy(vdev_raidz_fastest_impl.name, "fastest");

	raidz_math_initialized = B_TRUE;

	/* Use 'cycle' math selection method for userspace */
	VERIFY0(vdev_raidz_impl_set("cycle"));
	return;
#endif

	/* Fake an zio and run the benchmark on a warmed up buffer */
	bench_zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP);
	bench_zio->io_offset = 0;
	bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */
	bench_zio->io_abd = abd_alloc_linear(BENCH_ZIO_SIZE, B_TRUE);
	memset(abd_to_buf(bench_zio->io_abd), 0xAA, BENCH_ZIO_SIZE);

	/* Benchmark parity generation methods */
	for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) {
		bench_parity = fn + 1;
		/* New raidz_map is needed for each generate_p/q/r */
		bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT,
		    BENCH_D_COLS + bench_parity, bench_parity);

		benchmark_raidz_impl(bench_rm, fn, benchmark_gen_impl);

		vdev_raidz_map_free(bench_rm);
	}

	/* Benchmark data reconstruction methods */
	bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT,
	    BENCH_COLS, PARITY_PQR);

	for (fn = 0; fn < RAIDZ_REC_NUM; fn++)
		benchmark_raidz_impl(bench_rm, fn, benchmark_rec_impl);

	vdev_raidz_map_free(bench_rm);

	/* cleanup the bench zio */
	abd_free(bench_zio->io_abd);
	kmem_free(bench_zio, sizeof (zio_t));

	/* install kstats for all impl */
	raidz_math_kstat = kstat_create("zfs", 0, "vdev_raidz_bench", "misc",
	    KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);

	if (raidz_math_kstat != NULL) {
		raidz_math_kstat->ks_data = NULL;
		raidz_math_kstat->ks_ndata = UINT32_MAX;
		kstat_set_raw_ops(raidz_math_kstat,
		    raidz_math_kstat_headers,
		    raidz_math_kstat_data,
		    raidz_math_kstat_addr);
		kstat_install(raidz_math_kstat);
	}

	/* Finish initialization */
	atomic_swap_32(&zfs_vdev_raidz_impl, user_sel_impl);
	raidz_math_initialized = B_TRUE;
}

void
vdev_raidz_math_fini(void)
{
	raidz_impl_ops_t const *curr_impl;
	int i;

	if (raidz_math_kstat != NULL) {
		kstat_delete(raidz_math_kstat);
		raidz_math_kstat = NULL;
	}

	/* fini impl */
	for (i = 0; i < ARRAY_SIZE(raidz_all_maths); i++) {
		curr_impl = raidz_all_maths[i];
		if (curr_impl->fini)
			curr_impl->fini();
	}
}

static const struct {
	char *name;
	uint32_t sel;
} math_impl_opts[] = {
#if !defined(_KERNEL)
		{ "cycle",	IMPL_CYCLE },
#endif
		{ "fastest",	IMPL_FASTEST },
		{ "original",	IMPL_ORIGINAL },
		{ "scalar",	IMPL_SCALAR }
};

/*
 * Function sets desired raidz implementation.
 *
 * If we are called before init(), user preference will be saved in
 * user_sel_impl, and applied in later init() call. This occurs when module
 * parameter is specified on module load. Otherwise, directly update
 * zfs_vdev_raidz_impl.
 *
 * @val		Name of raidz implementation to use
 * @param	Unused.
 */
int
vdev_raidz_impl_set(const char *val)
{
	int err = -EINVAL;
	char req_name[RAIDZ_IMPL_NAME_MAX];
	uint32_t impl = RAIDZ_IMPL_READ(user_sel_impl);
	size_t i;

	/* sanitize input */
	i = strnlen(val, RAIDZ_IMPL_NAME_MAX);
	if (i == 0 || i == RAIDZ_IMPL_NAME_MAX)
		return (err);

	strlcpy(req_name, val, RAIDZ_IMPL_NAME_MAX);
	while (i > 0 && !!isspace(req_name[i-1]))
		i--;
	req_name[i] = '\0';

	/* Check mandatory options */
	for (i = 0; i < ARRAY_SIZE(math_impl_opts); i++) {
		if (strcmp(req_name, math_impl_opts[i].name) == 0) {
			impl = math_impl_opts[i].sel;
			err = 0;
			break;
		}
	}

	/* check all supported impl if init() was already called */
	if (err != 0 && raidz_math_initialized) {
		/* check all supported implementations */
		for (i = 0; i < raidz_supp_impl_cnt; i++) {
			if (strcmp(req_name, raidz_supp_impl[i]->name) == 0) {
				impl = i;
				err = 0;
				break;
			}
		}
	}

	if (err == 0) {
		if (raidz_math_initialized)
			atomic_swap_32(&zfs_vdev_raidz_impl, impl);
		else
			atomic_swap_32(&user_sel_impl, impl);
	}

	return (err);
}

#if defined(_KERNEL)
#include <linux/mod_compat.h>

static int
zfs_vdev_raidz_impl_set(const char *val, zfs_kernel_param_t *kp)
{
	return (vdev_raidz_impl_set(val));
}

static int
zfs_vdev_raidz_impl_get(char *buffer, zfs_kernel_param_t *kp)
{
	int i, cnt = 0;
	char *fmt;
	const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl);

	ASSERT(raidz_math_initialized);

	/* list mandatory options */
	for (i = 0; i < ARRAY_SIZE(math_impl_opts) - 2; i++) {
		fmt = (impl == math_impl_opts[i].sel) ? "[%s] " : "%s ";
		cnt += sprintf(buffer + cnt, fmt, math_impl_opts[i].name);
	}

	/* list all supported implementations */
	for (i = 0; i < raidz_supp_impl_cnt; i++) {
		fmt = (i == impl) ? "[%s] " : "%s ";
		cnt += sprintf(buffer + cnt, fmt, raidz_supp_impl[i]->name);
	}

	return (cnt);
}

module_param_call(zfs_vdev_raidz_impl, zfs_vdev_raidz_impl_set,
    zfs_vdev_raidz_impl_get, NULL, 0644);
MODULE_PARM_DESC(zfs_vdev_raidz_impl, "Select raidz implementation.");
#endif
Commit	Line	Data
ab9f4b0b GN	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
	22	* Copyright (C) 2016 Gvozden Nešković. All rights reserved.
	23	*/
	24
	25	#include <sys/zfs_context.h>
	26	#include <sys/types.h>
	27	#include <sys/zio.h>
	28	#include <sys/debug.h>
	29	#include <sys/zfs_debug.h>
	30
	31	#include <sys/vdev_raidz.h>
	32	#include <sys/vdev_raidz_impl.h>
	33
c9187d86 GN	34	extern boolean_t raidz_will_scalar_work(void);
	35
	36	/* Opaque implementation with NULL methods to represent original methods */
	37	static const raidz_impl_ops_t vdev_raidz_original_impl = {
	38	.name = "original",
	39	.is_supported = raidz_will_scalar_work,
	40	};
	41
	42	/* RAIDZ parity op that contain the fastest methods */
	43	static raidz_impl_ops_t vdev_raidz_fastest_impl = {
	44	.name = "fastest"
	45	};
	46
ab9f4b0b GN	47	/* All compiled in implementations */
ab9f4b0b GN	48	const raidz_impl_ops_t *raidz_all_maths[] = {
c9187d86	49	&vdev_raidz_original_impl,
ab9f4b0b	50	&vdev_raidz_scalar_impl,
ae25d222 GN	51	#if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */
	52	&vdev_raidz_sse2_impl,
	53	#endif
ab9f4b0b	54	#if defined(__x86_64) && defined(HAVE_SSSE3) /* only x86_64 for now */
ae25d222	55	&vdev_raidz_ssse3_impl,
ab9f4b0b GN	56	#endif
ab9f4b0b GN	57	#if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */
62a65a65 RD	58	&vdev_raidz_avx2_impl,
62a65a65 RD	59	#endif
7f547f85	60	#if defined(__x86_64) && defined(HAVE_AVX512F) /* only x86_64 for now */
65d71d42	61	&vdev_raidz_avx512f_impl,
7f547f85 RD	62	#endif
7f547f85 RD	63	#if defined(__x86_64) && defined(HAVE_AVX512BW) /* only x86_64 for now */
01017962	64	&vdev_raidz_avx512bw_impl,
7f547f85	65	#endif
62a65a65	66	#if defined(__aarch64__)
88cc2352 RD	67	&vdev_raidz_aarch64_neon_impl,
88cc2352 RD	68	&vdev_raidz_aarch64_neonx2_impl,
ab9f4b0b GN	69	#endif
	70	};
	71
	72	/* Indicate that benchmark has been completed */
	73	static boolean_t raidz_math_initialized = B_FALSE;
	74
	75	/* Select raidz implementation */
c9187d86 GN	76	#define IMPL_FASTEST (UINT32_MAX)
	77	#define IMPL_CYCLE (UINT32_MAX - 1)
	78	#define IMPL_ORIGINAL (0)
	79	#define IMPL_SCALAR (1)
	80
	81	#define RAIDZ_IMPL_READ(i) ((volatile uint32_t ) &(i))
	82
	83	static uint32_t zfs_vdev_raidz_impl = IMPL_SCALAR;
	84	static uint32_t user_sel_impl = IMPL_FASTEST;
ab9f4b0b GN	85
ab9f4b0b GN	86	/* Hold all supported implementations */
c9187d86 GN	87	static size_t raidz_supp_impl_cnt = 0;
c9187d86 GN	88	static raidz_impl_ops_t *raidz_supp_impl[ARRAY_SIZE(raidz_all_maths)];
ab9f4b0b GN	89
ab9f4b0b GN	90	/*
26a08b5c GN	91	* kstats values for supported implementations
26a08b5c GN	92	* Values represent per disk throughput of 8 disk+parity raidz vdev [B/s]
ab9f4b0b GN	93	*/
	94	static raidz_impl_kstat_t raidz_impl_kstats[ARRAY_SIZE(raidz_all_maths) + 1];
	95
	96	/* kstat for benchmarked implementations */
	97	static kstat_t *raidz_math_kstat = NULL;
	98
	99	/*
	100	* Selects the raidz operation for raidz_map
	101	* If rm_ops is set to NULL original raidz implementation will be used
	102	*/
c9187d86 GN	103	raidz_impl_ops_t *
c9187d86 GN	104	vdev_raidz_math_get_ops()
ab9f4b0b	105	{
c9187d86 GN	106	raidz_impl_ops_t *ops = NULL;
	107	const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl);
	108
	109	switch (impl) {
	110	case IMPL_FASTEST:
	111	ASSERT(raidz_math_initialized);
	112	ops = &vdev_raidz_fastest_impl;
	113	break;
ab9f4b0b	114	#if !defined(_KERNEL)
c9187d86 GN	115	case IMPL_CYCLE:
	116	{
	117	ASSERT(raidz_math_initialized);
	118	ASSERT3U(raidz_supp_impl_cnt, >, 0);
	119	/* Cycle through all supported implementations */
ab9f4b0b	120	static size_t cycle_impl_idx = 0;
c9187d86 GN	121	size_t idx = (++cycle_impl_idx) % raidz_supp_impl_cnt;
c9187d86 GN	122	ops = raidz_supp_impl[idx];
ab9f4b0b	123	}
c9187d86	124	break;
ab9f4b0b	125	#endif
c9187d86	126	case IMPL_ORIGINAL:
02730c33	127	ops = (raidz_impl_ops_t *)&vdev_raidz_original_impl;
c9187d86 GN	128	break;
c9187d86 GN	129	case IMPL_SCALAR:
02730c33	130	ops = (raidz_impl_ops_t *)&vdev_raidz_scalar_impl;
c9187d86 GN	131	break;
c9187d86 GN	132	default:
c9187d86 GN	133	ASSERT3U(impl, <, raidz_supp_impl_cnt);
c9187d86 GN	134	ASSERT3U(raidz_supp_impl_cnt, >, 0);
d99a0153 CW	135	if (impl < ARRAY_SIZE(raidz_all_maths))
d99a0153 CW	136	ops = raidz_supp_impl[impl];
c9187d86 GN	137	break;
c9187d86 GN	138	}
ab9f4b0b	139
c9187d86 GN	140	ASSERT3P(ops, !=, NULL);
	141
	142	return (ops);
ab9f4b0b GN	143	}
	144
	145	/*
	146	* Select parity generation method for raidz_map
	147	*/
c9187d86	148	int
ab9f4b0b GN	149	vdev_raidz_math_generate(raidz_map_t *rm)
	150	{
	151	raidz_gen_f gen_parity = NULL;
	152
	153	switch (raidz_parity(rm)) {
	154	case 1:
	155	gen_parity = rm->rm_ops->gen[RAIDZ_GEN_P];
	156	break;
	157	case 2:
	158	gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQ];
	159	break;
	160	case 3:
	161	gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQR];
	162	break;
	163	default:
	164	gen_parity = NULL;
	165	cmn_err(CE_PANIC, "invalid RAID-Z configuration %d",
02730c33	166	raidz_parity(rm));
ab9f4b0b GN	167	break;
	168	}
	169
c9187d86 GN	170	/* if method is NULL execute the original implementation */
	171	if (gen_parity == NULL)
	172	return (RAIDZ_ORIGINAL_IMPL);
ab9f4b0b GN	173
ab9f4b0b GN	174	gen_parity(rm);
c9187d86 GN	175
c9187d86 GN	176	return (0);
ab9f4b0b GN	177	}
	178
	179	static raidz_rec_f
c9187d86	180	reconstruct_fun_p_sel(raidz_map_t rm, const int parity_valid,
4ea3f864	181	const int nbaddata)
ab9f4b0b GN	182	{
	183	if (nbaddata == 1 && parity_valid[CODE_P]) {
	184	return (rm->rm_ops->rec[RAIDZ_REC_P]);
	185	}
	186	return ((raidz_rec_f) NULL);
	187	}
	188
	189	static raidz_rec_f
c9187d86	190	reconstruct_fun_pq_sel(raidz_map_t rm, const int parity_valid,
4ea3f864	191	const int nbaddata)
ab9f4b0b GN	192	{
	193	if (nbaddata == 1) {
	194	if (parity_valid[CODE_P]) {
	195	return (rm->rm_ops->rec[RAIDZ_REC_P]);
	196	} else if (parity_valid[CODE_Q]) {
	197	return (rm->rm_ops->rec[RAIDZ_REC_Q]);
	198	}
	199	} else if (nbaddata == 2 &&
02730c33	200	parity_valid[CODE_P] && parity_valid[CODE_Q]) {
ab9f4b0b GN	201	return (rm->rm_ops->rec[RAIDZ_REC_PQ]);
	202	}
	203	return ((raidz_rec_f) NULL);
	204	}
	205
	206	static raidz_rec_f
c9187d86	207	reconstruct_fun_pqr_sel(raidz_map_t rm, const int parity_valid,
4ea3f864	208	const int nbaddata)
ab9f4b0b GN	209	{
	210	if (nbaddata == 1) {
	211	if (parity_valid[CODE_P]) {
	212	return (rm->rm_ops->rec[RAIDZ_REC_P]);
	213	} else if (parity_valid[CODE_Q]) {
	214	return (rm->rm_ops->rec[RAIDZ_REC_Q]);
	215	} else if (parity_valid[CODE_R]) {
	216	return (rm->rm_ops->rec[RAIDZ_REC_R]);
	217	}
	218	} else if (nbaddata == 2) {
	219	if (parity_valid[CODE_P] && parity_valid[CODE_Q]) {
	220	return (rm->rm_ops->rec[RAIDZ_REC_PQ]);
	221	} else if (parity_valid[CODE_P] && parity_valid[CODE_R]) {
	222	return (rm->rm_ops->rec[RAIDZ_REC_PR]);
	223	} else if (parity_valid[CODE_Q] && parity_valid[CODE_R]) {
	224	return (rm->rm_ops->rec[RAIDZ_REC_QR]);
	225	}
	226	} else if (nbaddata == 3 &&
02730c33 BB	227	parity_valid[CODE_P] && parity_valid[CODE_Q] &&
02730c33 BB	228	parity_valid[CODE_R]) {
ab9f4b0b GN	229	return (rm->rm_ops->rec[RAIDZ_REC_PQR]);
	230	}
	231	return ((raidz_rec_f) NULL);
	232	}
	233
	234	/*
	235	* Select data reconstruction method for raidz_map
	236	* @parity_valid - Parity validity flag
	237	* @dt - Failed data index array
	238	* @nbaddata - Number of failed data columns
	239	*/
	240	int
	241	vdev_raidz_math_reconstruct(raidz_map_t rm, const int parity_valid,
4ea3f864	242	const int *dt, const int nbaddata)
ab9f4b0b	243	{
cbf484f8	244	raidz_rec_f rec_fn = NULL;
ab9f4b0b GN	245
ab9f4b0b GN	246	switch (raidz_parity(rm)) {
c9187d86	247	case PARITY_P:
cbf484f8	248	rec_fn = reconstruct_fun_p_sel(rm, parity_valid, nbaddata);
c9187d86 GN	249	break;
c9187d86 GN	250	case PARITY_PQ:
cbf484f8	251	rec_fn = reconstruct_fun_pq_sel(rm, parity_valid, nbaddata);
c9187d86 GN	252	break;
c9187d86 GN	253	case PARITY_PQR:
cbf484f8	254	rec_fn = reconstruct_fun_pqr_sel(rm, parity_valid, nbaddata);
c9187d86 GN	255	break;
	256	default:
	257	cmn_err(CE_PANIC, "invalid RAID-Z configuration %d",
	258	raidz_parity(rm));
	259	break;
ab9f4b0b GN	260	}
ab9f4b0b GN	261
cbf484f8	262	if (rec_fn == NULL)
c9187d86 GN	263	return (RAIDZ_ORIGINAL_IMPL);
c9187d86 GN	264	else
cbf484f8	265	return (rec_fn(rm, dt));
ab9f4b0b GN	266	}
	267
	268	const char *raidz_gen_name[] = {
	269	"gen_p", "gen_pq", "gen_pqr"
	270	};
	271	const char *raidz_rec_name[] = {
	272	"rec_p", "rec_q", "rec_r",
	273	"rec_pq", "rec_pr", "rec_qr", "rec_pqr"
	274	};
	275
26a08b5c GN	276	#define RAIDZ_KSTAT_LINE_LEN (17 + 10*12 + 1)
	277
	278	static int
	279	raidz_math_kstat_headers(char *buf, size_t size)
ab9f4b0b GN	280	{
ab9f4b0b GN	281	int i;
26a08b5c GN	282	ssize_t off;
	283
	284	ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN);
	285
	286	off = snprintf(buf, size, "%-17s", "implementation");
	287
	288	for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++)
62a65a65	289	off += snprintf(buf + off, size - off, "%-16s",
26a08b5c GN	290	raidz_gen_name[i]);
	291
	292	for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++)
62a65a65	293	off += snprintf(buf + off, size - off, "%-16s",
26a08b5c GN	294	raidz_rec_name[i]);
	295
	296	(void) snprintf(buf + off, size - off, "\n");
	297
	298	return (0);
	299	}
	300
	301	static int
	302	raidz_math_kstat_data(char buf, size_t size, void data)
	303	{
02730c33 BB	304	raidz_impl_kstat_t *fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
02730c33 BB	305	raidz_impl_kstat_t cstat = (raidz_impl_kstat_t )data;
26a08b5c GN	306	ssize_t off = 0;
	307	int i;
	308
	309	ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN);
ab9f4b0b	310
26a08b5c GN	311	if (cstat == fstat) {
26a08b5c GN	312	off += snprintf(buf + off, size - off, "%-17s", "fastest");
ab9f4b0b	313
26a08b5c GN	314	for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) {
26a08b5c GN	315	int id = fstat->gen[i];
62a65a65	316	off += snprintf(buf + off, size - off, "%-16s",
26a08b5c GN	317	raidz_supp_impl[id]->name);
	318	}
	319	for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) {
	320	int id = fstat->rec[i];
62a65a65	321	off += snprintf(buf + off, size - off, "%-16s",
26a08b5c GN	322	raidz_supp_impl[id]->name);
	323	}
	324	} else {
	325	ptrdiff_t id = cstat - raidz_impl_kstats;
	326
	327	off += snprintf(buf + off, size - off, "%-17s",
	328	raidz_supp_impl[id]->name);
	329
	330	for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++)
62a65a65	331	off += snprintf(buf + off, size - off, "%-16llu",
02730c33	332	(u_longlong_t)cstat->gen[i]);
26a08b5c GN	333
26a08b5c GN	334	for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++)
62a65a65	335	off += snprintf(buf + off, size - off, "%-16llu",
02730c33	336	(u_longlong_t)cstat->rec[i]);
ab9f4b0b	337	}
26a08b5c GN	338
	339	(void) snprintf(buf + off, size - off, "\n");
	340
	341	return (0);
	342	}
	343
	344	static void *
	345	raidz_math_kstat_addr(kstat_t *ksp, loff_t n)
	346	{
	347	if (n <= raidz_supp_impl_cnt)
	348	ksp->ks_private = (void *) (raidz_impl_kstats + n);
	349	else
	350	ksp->ks_private = NULL;
	351
	352	return (ksp->ks_private);
ab9f4b0b GN	353	}
	354
	355	#define BENCH_D_COLS (8ULL)
	356	#define BENCH_COLS (BENCH_D_COLS + PARITY_PQR)
590c9a09	357	#define BENCH_ZIO_SIZE (1ULL << SPA_OLD_MAXBLOCKSHIFT) /* 128 kiB */
ab9f4b0b GN	358	#define BENCH_NS MSEC2NSEC(25) /* 25ms */
	359
	360	typedef void (benchmark_fn)(raidz_map_t rm, const int fn);
	361
	362	static void
	363	benchmark_gen_impl(raidz_map_t *rm, const int fn)
	364	{
	365	(void) fn;
	366	vdev_raidz_generate_parity(rm);
	367	}
	368
	369	static void
	370	benchmark_rec_impl(raidz_map_t *rm, const int fn)
	371	{
	372	static const int rec_tgt[7][3] = {
	373	{1, 2, 3}, /* rec_p: bad QR & D[0] */
	374	{0, 2, 3}, /* rec_q: bad PR & D[0] */
	375	{0, 1, 3}, /* rec_r: bad PQ & D[0] */
	376	{2, 3, 4}, /* rec_pq: bad R & D[0][1] */
	377	{1, 3, 4}, /* rec_pr: bad Q & D[0][1] */
	378	{0, 3, 4}, /* rec_qr: bad P & D[0][1] */
	379	{3, 4, 5} /* rec_pqr: bad & D[0][1][2] */
	380	};
	381
	382	vdev_raidz_reconstruct(rm, rec_tgt[fn], 3);
	383	}
	384
	385	/*
	386	* Benchmarking of all supported implementations (raidz_supp_impl_cnt)
	387	* is performed by setting the rm_ops pointer and calling the top level
	388	* generate/reconstruct methods of bench_rm.
	389	*/
	390	static void
	391	benchmark_raidz_impl(raidz_map_t *bench_rm, const int fn, benchmark_fn bench_fn)
	392	{
	393	uint64_t run_cnt, speed, best_speed = 0;
	394	hrtime_t t_start, t_diff;
	395	raidz_impl_ops_t *curr_impl;
02730c33	396	raidz_impl_kstat_t *fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
ab9f4b0b GN	397	int impl, i;
ab9f4b0b GN	398
c9187d86	399	for (impl = 0; impl < raidz_supp_impl_cnt; impl++) {
ab9f4b0b GN	400	/* set an implementation to benchmark */
	401	curr_impl = raidz_supp_impl[impl];
	402	bench_rm->rm_ops = curr_impl;
	403
	404	run_cnt = 0;
	405	t_start = gethrtime();
	406
	407	do {
	408	for (i = 0; i < 25; i++, run_cnt++)
	409	bench_fn(bench_rm, fn);
	410
	411	t_diff = gethrtime() - t_start;
	412	} while (t_diff < BENCH_NS);
	413
	414	speed = run_cnt * BENCH_ZIO_SIZE * NANOSEC;
	415	speed /= (t_diff * BENCH_COLS);
	416
	417	if (bench_fn == benchmark_gen_impl)
26a08b5c	418	raidz_impl_kstats[impl].gen[fn] = speed;
ab9f4b0b	419	else
26a08b5c	420	raidz_impl_kstats[impl].rec[fn] = speed;
ab9f4b0b	421
c9187d86 GN	422	/* Update fastest implementation method */
c9187d86 GN	423	if (speed > best_speed) {
ab9f4b0b GN	424	best_speed = speed;
ab9f4b0b GN	425
c9187d86	426	if (bench_fn == benchmark_gen_impl) {
26a08b5c	427	fstat->gen[fn] = impl;
ab9f4b0b GN	428	vdev_raidz_fastest_impl.gen[fn] =
ab9f4b0b GN	429	curr_impl->gen[fn];
c9187d86	430	} else {
26a08b5c	431	fstat->rec[fn] = impl;
ab9f4b0b GN	432	vdev_raidz_fastest_impl.rec[fn] =
ab9f4b0b GN	433	curr_impl->rec[fn];
c9187d86	434	}
ab9f4b0b GN	435	}
	436	}
	437	}
	438
	439	void
	440	vdev_raidz_math_init(void)
	441	{
	442	raidz_impl_ops_t *curr_impl;
	443	zio_t *bench_zio = NULL;
	444	raidz_map_t *bench_rm = NULL;
	445	uint64_t bench_parity;
	446	int i, c, fn;
	447
ab9f4b0b GN	448	/* move supported impl into raidz_supp_impl */
ab9f4b0b GN	449	for (i = 0, c = 0; i < ARRAY_SIZE(raidz_all_maths); i++) {
02730c33	450	curr_impl = (raidz_impl_ops_t *)raidz_all_maths[i];
ab9f4b0b GN	451
	452	/* initialize impl */
	453	if (curr_impl->init)
	454	curr_impl->init();
	455
26a08b5c	456	if (curr_impl->is_supported())
02730c33	457	raidz_supp_impl[c++] = (raidz_impl_ops_t *)curr_impl;
ab9f4b0b	458	}
c9187d86	459	membar_producer(); /* complete raidz_supp_impl[] init */
ab9f4b0b	460	raidz_supp_impl_cnt = c; /* number of supported impl */
ab9f4b0b	461
ab9f4b0b	462	#if !defined(_KERNEL)
c9187d86	463	/* Skip benchmarking and use last implementation as fastest */
ab9f4b0b GN	464	memcpy(&vdev_raidz_fastest_impl, raidz_supp_impl[raidz_supp_impl_cnt-1],
ab9f4b0b GN	465	sizeof (vdev_raidz_fastest_impl));
c9187d86	466	strcpy(vdev_raidz_fastest_impl.name, "fastest");
ab9f4b0b GN	467
	468	raidz_math_initialized = B_TRUE;
	469
	470	/* Use 'cycle' math selection method for userspace */
	471	VERIFY0(vdev_raidz_impl_set("cycle"));
	472	return;
	473	#endif
	474
a6255b7f	475	/* Fake an zio and run the benchmark on a warmed up buffer */
ab9f4b0b GN	476	bench_zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP);
	477	bench_zio->io_offset = 0;
	478	bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */
a6255b7f DQ	479	bench_zio->io_abd = abd_alloc_linear(BENCH_ZIO_SIZE, B_TRUE);
a6255b7f DQ	480	memset(abd_to_buf(bench_zio->io_abd), 0xAA, BENCH_ZIO_SIZE);
ab9f4b0b GN	481
	482	/* Benchmark parity generation methods */
	483	for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) {
	484	bench_parity = fn + 1;
	485	/* New raidz_map is needed for each generate_p/q/r */
c9187d86	486	bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT,
ab9f4b0b GN	487	BENCH_D_COLS + bench_parity, bench_parity);
	488
	489	benchmark_raidz_impl(bench_rm, fn, benchmark_gen_impl);
	490
	491	vdev_raidz_map_free(bench_rm);
	492	}
	493
	494	/* Benchmark data reconstruction methods */
c9187d86 GN	495	bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT,
c9187d86 GN	496	BENCH_COLS, PARITY_PQR);
ab9f4b0b GN	497
	498	for (fn = 0; fn < RAIDZ_REC_NUM; fn++)
	499	benchmark_raidz_impl(bench_rm, fn, benchmark_rec_impl);
	500
	501	vdev_raidz_map_free(bench_rm);
	502
	503	/* cleanup the bench zio */
a6255b7f	504	abd_free(bench_zio->io_abd);
ab9f4b0b GN	505	kmem_free(bench_zio, sizeof (zio_t));
	506
	507	/* install kstats for all impl */
26a08b5c	508	raidz_math_kstat = kstat_create("zfs", 0, "vdev_raidz_bench", "misc",
02730c33	509	KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
ab9f4b0b GN	510
ab9f4b0b GN	511	if (raidz_math_kstat != NULL) {
26a08b5c GN	512	raidz_math_kstat->ks_data = NULL;
	513	raidz_math_kstat->ks_ndata = UINT32_MAX;
	514	kstat_set_raw_ops(raidz_math_kstat,
	515	raidz_math_kstat_headers,
	516	raidz_math_kstat_data,
	517	raidz_math_kstat_addr);
ab9f4b0b GN	518	kstat_install(raidz_math_kstat);
	519	}
	520
	521	/* Finish initialization */
c9187d86	522	atomic_swap_32(&zfs_vdev_raidz_impl, user_sel_impl);
ab9f4b0b	523	raidz_math_initialized = B_TRUE;
ab9f4b0b GN	524	}
	525
	526	void
	527	vdev_raidz_math_fini(void)
	528	{
	529	raidz_impl_ops_t const *curr_impl;
	530	int i;
	531
	532	if (raidz_math_kstat != NULL) {
	533	kstat_delete(raidz_math_kstat);
	534	raidz_math_kstat = NULL;
	535	}
	536
ab9f4b0b GN	537	/* fini impl */
	538	for (i = 0; i < ARRAY_SIZE(raidz_all_maths); i++) {
	539	curr_impl = raidz_all_maths[i];
ab9f4b0b GN	540	if (curr_impl->fini)
	541	curr_impl->fini();
	542	}
	543	}
	544
c9187d86	545	static const struct {
02730c33	546	char *name;
c9187d86	547	uint32_t sel;
ab9f4b0b	548	} math_impl_opts[] = {
ab9f4b0b	549	#if !defined(_KERNEL)
c9187d86	550	{ "cycle", IMPL_CYCLE },
ab9f4b0b	551	#endif
c9187d86 GN	552	{ "fastest", IMPL_FASTEST },
	553	{ "original", IMPL_ORIGINAL },
	554	{ "scalar", IMPL_SCALAR }
ab9f4b0b GN	555	};
	556
	557	/*
	558	* Function sets desired raidz implementation.
c9187d86 GN	559	*
	560	* If we are called before init(), user preference will be saved in
	561	* user_sel_impl, and applied in later init() call. This occurs when module
	562	* parameter is specified on module load. Otherwise, directly update
	563	* zfs_vdev_raidz_impl.
ab9f4b0b GN	564	*
	565	* @val Name of raidz implementation to use
	566	* @param Unused.
	567	*/
9cc1844a GN	568	int
9cc1844a GN	569	vdev_raidz_impl_set(const char *val)
ab9f4b0b	570	{
c9187d86 GN	571	int err = -EINVAL;
	572	char req_name[RAIDZ_IMPL_NAME_MAX];
	573	uint32_t impl = RAIDZ_IMPL_READ(user_sel_impl);
ab9f4b0b GN	574	size_t i;
ab9f4b0b GN	575
c9187d86 GN	576	/* sanitize input */
	577	i = strnlen(val, RAIDZ_IMPL_NAME_MAX);
	578	if (i == 0 \|\| i == RAIDZ_IMPL_NAME_MAX)
	579	return (err);
	580
	581	strlcpy(req_name, val, RAIDZ_IMPL_NAME_MAX);
	582	while (i > 0 && !!isspace(req_name[i-1]))
	583	i--;
	584	req_name[i] = '\0';
	585
ab9f4b0b GN	586	/* Check mandatory options */
ab9f4b0b GN	587	for (i = 0; i < ARRAY_SIZE(math_impl_opts); i++) {
c9187d86 GN	588	if (strcmp(req_name, math_impl_opts[i].name) == 0) {
	589	impl = math_impl_opts[i].sel;
	590	err = 0;
	591	break;
ab9f4b0b GN	592	}
	593	}
	594
c9187d86 GN	595	/* check all supported impl if init() was already called */
	596	if (err != 0 && raidz_math_initialized) {
	597	/* check all supported implementations */
	598	for (i = 0; i < raidz_supp_impl_cnt; i++) {
	599	if (strcmp(req_name, raidz_supp_impl[i]->name) == 0) {
	600	impl = i;
	601	err = 0;
	602	break;
	603	}
ab9f4b0b GN	604	}
	605	}
	606
c9187d86 GN	607	if (err == 0) {
	608	if (raidz_math_initialized)
	609	atomic_swap_32(&zfs_vdev_raidz_impl, impl);
	610	else
	611	atomic_swap_32(&user_sel_impl, impl);
	612	}
	613
	614	return (err);
ab9f4b0b GN	615	}
ab9f4b0b GN	616
93ce2b4c	617	#if defined(_KERNEL)
9cc1844a	618	#include <linux/mod_compat.h>
ab9f4b0b	619
9cc1844a GN	620	static int
	621	zfs_vdev_raidz_impl_set(const char val, zfs_kernel_param_t kp)
	622	{
	623	return (vdev_raidz_impl_set(val));
ab9f4b0b GN	624	}
ab9f4b0b GN	625
ab9f4b0b	626	static int
9cc1844a	627	zfs_vdev_raidz_impl_get(char buffer, zfs_kernel_param_t kp)
ab9f4b0b GN	628	{
	629	int i, cnt = 0;
	630	char *fmt;
c9187d86	631	const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl);
ab9f4b0b GN	632
	633	ASSERT(raidz_math_initialized);
	634
ab9f4b0b	635	/* list mandatory options */
c9187d86 GN	636	for (i = 0; i < ARRAY_SIZE(math_impl_opts) - 2; i++) {
c9187d86 GN	637	fmt = (impl == math_impl_opts[i].sel) ? "[%s] " : "%s ";
ab9f4b0b GN	638	cnt += sprintf(buffer + cnt, fmt, math_impl_opts[i].name);
	639	}
	640
	641	/* list all supported implementations */
	642	for (i = 0; i < raidz_supp_impl_cnt; i++) {
c9187d86	643	fmt = (i == impl) ? "[%s] " : "%s ";
ab9f4b0b GN	644	cnt += sprintf(buffer + cnt, fmt, raidz_supp_impl[i]->name);
	645	}
	646
ab9f4b0b GN	647	return (cnt);
	648	}
	649
	650	module_param_call(zfs_vdev_raidz_impl, zfs_vdev_raidz_impl_set,
4ea3f864	651	zfs_vdev_raidz_impl_get, NULL, 0644);
ab9f4b0b GN	652	MODULE_PARM_DESC(zfs_vdev_raidz_impl, "Select raidz implementation.");
ab9f4b0b GN	653	#endif