[mirror_zfs.git] / module / zcommon / zfs_fletcher.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Fletcher Checksums
 * ------------------
 *
 * ZFS's 2nd and 4th order Fletcher checksums are defined by the following
 * recurrence relations:
 *
 *	a  = a    + f
 *	 i    i-1    i-1
 *
 *	b  = b    + a
 *	 i    i-1    i
 *
 *	c  = c    + b		(fletcher-4 only)
 *	 i    i-1    i
 *
 *	d  = d    + c		(fletcher-4 only)
 *	 i    i-1    i
 *
 * Where
 *	a_0 = b_0 = c_0 = d_0 = 0
 * and
 *	f_0 .. f_(n-1) are the input data.
 *
 * Using standard techniques, these translate into the following series:
 *
 *	     __n_			     __n_
 *	     \   |			     \   |
 *	a  =  >     f			b  =  >     i * f
 *	 n   /___|   n - i		 n   /___|	 n - i
 *	     i = 1			     i = 1
 *
 *
 *	     __n_			     __n_
 *	     \   |  i*(i+1)		     \   |  i*(i+1)*(i+2)
 *	c  =  >     ------- f		d  =  >     ------------- f
 *	 n   /___|     2     n - i	 n   /___|	  6	   n - i
 *	     i = 1			     i = 1
 *
 * For fletcher-2, the f_is are 64-bit, and [ab]_i are 64-bit accumulators.
 * Since the additions are done mod (2^64), errors in the high bits may not
 * be noticed.  For this reason, fletcher-2 is deprecated.
 *
 * For fletcher-4, the f_is are 32-bit, and [abcd]_i are 64-bit accumulators.
 * A conservative estimate of how big the buffer can get before we overflow
 * can be estimated using f_i = 0xffffffff for all i:
 *
 * % bc
 *  f=2^32-1;d=0; for (i = 1; d<2^64; i++) { d += f*i*(i+1)*(i+2)/6 }; (i-1)*4
 * 2264
 *  quit
 * %
 *
 * So blocks of up to 2k will not overflow.  Our largest block size is
 * 128k, which has 32k 4-byte words, so we can compute the largest possible
 * accumulators, then divide by 2^64 to figure the max amount of overflow:
 *
 * % bc
 *  a=b=c=d=0; f=2^32-1; for (i=1; i<=32*1024; i++) { a+=f; b+=a; c+=b; d+=c }
 *  a/2^64;b/2^64;c/2^64;d/2^64
 * 0
 * 0
 * 1365
 * 11186858
 *  quit
 * %
 *
 * So a and b cannot overflow.  To make sure each bit of input has some
 * effect on the contents of c and d, we can look at what the factors of
 * the coefficients in the equations for c_n and d_n are.  The number of 2s
 * in the factors determines the lowest set bit in the multiplier.  Running
 * through the cases for n*(n+1)/2 reveals that the highest power of 2 is
 * 2^14, and for n*(n+1)*(n+2)/6 it is 2^15.  So while some data may overflow
 * the 64-bit accumulators, every bit of every f_i effects every accumulator,
 * even for 128k blocks.
 *
 * If we wanted to make a stronger version of fletcher4 (fletcher4c?),
 * we could do our calculations mod (2^32 - 1) by adding in the carries
 * periodically, and store the number of carries in the top 32-bits.
 *
 * --------------------
 * Checksum Performance
 * --------------------
 *
 * There are two interesting components to checksum performance: cached and
 * uncached performance.  With cached data, fletcher-2 is about four times
 * faster than fletcher-4.  With uncached data, the performance difference is
 * negligible, since the cost of a cache fill dominates the processing time.
 * Even though fletcher-4 is slower than fletcher-2, it is still a pretty
 * efficient pass over the data.
 *
 * In normal operation, the data which is being checksummed is in a buffer
 * which has been filled either by:
 *
 *	1. a compression step, which will be mostly cached, or
 *	2. a bcopy() or copyin(), which will be uncached (because the
 *	   copy is cache-bypassing).
 *
 * For both cached and uncached data, both fletcher checksums are much faster
 * than sha-256, and slower than 'off', which doesn't touch the data at all.
 */

#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/byteorder.h>
#include <sys/spa.h>
#include <sys/zfs_context.h>
#include <zfs_fletcher.h>

static void fletcher_4_scalar_init(zio_cksum_t *zcp);
static void fletcher_4_scalar(const void *buf, uint64_t size,
    zio_cksum_t *zcp);
static void fletcher_4_scalar_byteswap(const void *buf, uint64_t size,
    zio_cksum_t *zcp);
static boolean_t fletcher_4_scalar_valid(void);

static const fletcher_4_ops_t fletcher_4_scalar_ops = {
	.init = fletcher_4_scalar_init,
	.compute = fletcher_4_scalar,
	.compute_byteswap = fletcher_4_scalar_byteswap,
	.valid = fletcher_4_scalar_valid,
	.name = "scalar"
};

static const fletcher_4_ops_t *fletcher_4_algos[] = {
	&fletcher_4_scalar_ops,
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
	&fletcher_4_avx2_ops,
#endif
};

static enum fletcher_selector {
	FLETCHER_FASTEST = 0,
	FLETCHER_SCALAR,
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
	FLETCHER_AVX2,
#endif
	FLETCHER_CYCLE
} fletcher_4_impl_chosen = FLETCHER_SCALAR;

static struct fletcher_4_impl_selector {
	const char		*fis_name;
	const fletcher_4_ops_t	*fis_ops;
} fletcher_4_impl_selectors[] = {
	[ FLETCHER_FASTEST ]	= { "fastest", NULL },
	[ FLETCHER_SCALAR ]	= { "scalar", &fletcher_4_scalar_ops },
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
	[ FLETCHER_AVX2 ]	= { "avx2", &fletcher_4_avx2_ops },
#endif
#if !defined(_KERNEL)
	[ FLETCHER_CYCLE ]	= { "cycle", &fletcher_4_scalar_ops }
#endif
};

static kmutex_t fletcher_4_impl_lock;

static kstat_t *fletcher_4_kstat;

static kstat_named_t fletcher_4_kstat_data[ARRAY_SIZE(fletcher_4_algos)];

void
fletcher_2_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
	const uint64_t *ip = buf;
	const uint64_t *ipend = ip + (size / sizeof (uint64_t));
	uint64_t a0, b0, a1, b1;

	for (a0 = b0 = a1 = b1 = 0; ip < ipend; ip += 2) {
		a0 += ip[0];
		a1 += ip[1];
		b0 += a0;
		b1 += a1;
	}

	ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
}

void
fletcher_2_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
	const uint64_t *ip = buf;
	const uint64_t *ipend = ip + (size / sizeof (uint64_t));
	uint64_t a0, b0, a1, b1;

	for (a0 = b0 = a1 = b1 = 0; ip < ipend; ip += 2) {
		a0 += BSWAP_64(ip[0]);
		a1 += BSWAP_64(ip[1]);
		b0 += a0;
		b1 += a1;
	}

	ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
}

static void fletcher_4_scalar_init(zio_cksum_t *zcp)
{
	ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
}

static void
fletcher_4_scalar(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
	const uint32_t *ip = buf;
	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
	uint64_t a, b, c, d;

	a = zcp->zc_word[0];
	b = zcp->zc_word[1];
	c = zcp->zc_word[2];
	d = zcp->zc_word[3];

	for (; ip < ipend; ip++) {
		a += ip[0];
		b += a;
		c += b;
		d += c;
	}

	ZIO_SET_CHECKSUM(zcp, a, b, c, d);
}

static void
fletcher_4_scalar_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
	const uint32_t *ip = buf;
	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
	uint64_t a, b, c, d;

	a = zcp->zc_word[0];
	b = zcp->zc_word[1];
	c = zcp->zc_word[2];
	d = zcp->zc_word[3];

	for (; ip < ipend; ip++) {
		a += BSWAP_32(ip[0]);
		b += a;
		c += b;
		d += c;
	}

	ZIO_SET_CHECKSUM(zcp, a, b, c, d);
}

static boolean_t
fletcher_4_scalar_valid(void)
{
	return (B_TRUE);
}

int
fletcher_4_impl_set(const char *val)
{
	const fletcher_4_ops_t *ops;
	enum fletcher_selector idx;
	size_t val_len;
	unsigned i;

	val_len = strlen(val);
	while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */
		val_len--;

	for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) {
		const char *name = fletcher_4_impl_selectors[i].fis_name;

		if (val_len == strlen(name) &&
		    strncmp(val, name, val_len) == 0) {
			idx = i;
			break;
		}
	}
	if (i >= ARRAY_SIZE(fletcher_4_impl_selectors))
		return (-EINVAL);

	ops = fletcher_4_impl_selectors[idx].fis_ops;
	if (ops == NULL || !ops->valid())
		return (-ENOTSUP);

	mutex_enter(&fletcher_4_impl_lock);
	if (fletcher_4_impl_chosen != idx)
		fletcher_4_impl_chosen = idx;
	mutex_exit(&fletcher_4_impl_lock);

	return (0);
}

static inline const fletcher_4_ops_t *
fletcher_4_impl_get(void)
{
#if !defined(_KERNEL)
	if (fletcher_4_impl_chosen == FLETCHER_CYCLE) {
		static volatile unsigned int cycle_count = 0;
		const fletcher_4_ops_t *ops = NULL;
		unsigned int index;

		while (1) {
			index = atomic_inc_uint_nv(&cycle_count);
			ops = fletcher_4_algos[
			    index % ARRAY_SIZE(fletcher_4_algos)];
			if (ops->valid())
				break;
		}
		return (ops);
	}
#endif
	membar_producer();
	return (fletcher_4_impl_selectors[fletcher_4_impl_chosen].fis_ops);
}

void
fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
	const fletcher_4_ops_t *ops = fletcher_4_impl_get();

	ops->init(zcp);
	ops->compute(buf, size, zcp);
	if (ops->fini != NULL)
		ops->fini(zcp);
}

void
fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
	const fletcher_4_ops_t *ops = fletcher_4_impl_get();

	ops->init(zcp);
	ops->compute_byteswap(buf, size, zcp);
	if (ops->fini != NULL)
		ops->fini(zcp);
}

void
fletcher_4_incremental_native(const void *buf, uint64_t size,
    zio_cksum_t *zcp)
{
	fletcher_4_scalar(buf, size, zcp);
}

void
fletcher_4_incremental_byteswap(const void *buf, uint64_t size,
    zio_cksum_t *zcp)
{
	fletcher_4_scalar_byteswap(buf, size, zcp);
}

void
fletcher_4_init(void)
{
	const uint64_t const bench_ns = (50 * MICROSEC); /* 50ms */
	unsigned long best_run_count = 0;
	unsigned long best_run_index = 0;
	const unsigned data_size = 4096;
	char *databuf;
	int i;

	databuf = kmem_alloc(data_size, KM_SLEEP);
	for (i = 0; i < ARRAY_SIZE(fletcher_4_algos); i++) {
		const fletcher_4_ops_t *ops = fletcher_4_algos[i];
		kstat_named_t *stat = &fletcher_4_kstat_data[i];
		unsigned long run_count = 0;
		hrtime_t start;
		zio_cksum_t zc;

		strncpy(stat->name, ops->name, sizeof (stat->name) - 1);
		stat->data_type = KSTAT_DATA_UINT64;
		stat->value.ui64 = 0;

		if (!ops->valid())
			continue;

		kpreempt_disable();
		start = gethrtime();
		ops->init(&zc);
		do {
			ops->compute(databuf, data_size, &zc);
			run_count++;
		} while (gethrtime() < start + bench_ns);
		if (ops->fini != NULL)
			ops->fini(&zc);
		kpreempt_enable();

		if (run_count > best_run_count) {
			best_run_count = run_count;
			best_run_index = i;
		}

		/*
		 * Due to high overhead of gethrtime(), the performance data
		 * here is inaccurate and much slower than it could be.
		 * It's fine for our use though because only relative speed
		 * is important.
		 */
		stat->value.ui64 = data_size * run_count *
		    (NANOSEC / bench_ns) >> 20; /* by MB/s */
	}
	kmem_free(databuf, data_size);

	fletcher_4_impl_selectors[FLETCHER_FASTEST].fis_ops =
	    fletcher_4_algos[best_run_index];

	mutex_init(&fletcher_4_impl_lock, NULL, MUTEX_DEFAULT, NULL);
	fletcher_4_impl_set("fastest");

	fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench",
	    "misc", KSTAT_TYPE_NAMED, ARRAY_SIZE(fletcher_4_algos),
	    KSTAT_FLAG_VIRTUAL);
	if (fletcher_4_kstat != NULL) {
		fletcher_4_kstat->ks_data = fletcher_4_kstat_data;
		kstat_install(fletcher_4_kstat);
	}
}

void
fletcher_4_fini(void)
{
	mutex_destroy(&fletcher_4_impl_lock);
	if (fletcher_4_kstat != NULL) {
		kstat_delete(fletcher_4_kstat);
		fletcher_4_kstat = NULL;
	}
}

#if defined(_KERNEL) && defined(HAVE_SPL)

static int
fletcher_4_param_get(char *buffer, struct kernel_param *unused)
{
	int i, cnt = 0;

	for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) {
		const fletcher_4_ops_t *ops;

		ops = fletcher_4_impl_selectors[i].fis_ops;
		if (!ops->valid())
			continue;

		cnt += sprintf(buffer + cnt,
		    fletcher_4_impl_chosen == i ? "[%s] " : "%s ",
		    fletcher_4_impl_selectors[i].fis_name);
	}

	return (cnt);
}

static int
fletcher_4_param_set(const char *val, struct kernel_param *unused)
{
	return (fletcher_4_impl_set(val));
}

/*
 * Choose a fletcher 4 implementation in ZFS.
 * Users can choose the "fastest" algorithm, or "scalar" and "avx2" which means
 * to compute fletcher 4 by CPU or vector instructions respectively.
 * Users can also choose "cycle" to exercise all implementions, but this is
 * for testing purpose therefore it can only be set in user space.
 */
module_param_call(zfs_fletcher_4_impl,
    fletcher_4_param_set, fletcher_4_param_get, NULL, 0644);
MODULE_PARM_DESC(zfs_fletcher_4_impl, "Select fletcher 4 algorithm");

EXPORT_SYMBOL(fletcher_4_init);
EXPORT_SYMBOL(fletcher_4_fini);
EXPORT_SYMBOL(fletcher_2_native);
EXPORT_SYMBOL(fletcher_2_byteswap);
EXPORT_SYMBOL(fletcher_4_native);
EXPORT_SYMBOL(fletcher_4_byteswap);
EXPORT_SYMBOL(fletcher_4_incremental_native);
EXPORT_SYMBOL(fletcher_4_incremental_byteswap);
#endif
Commit	Line	Data
34dc7c2f BB	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
9babb374	22	* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
34dc7c2f BB	23	* Use is subject to license terms.
	24	*/
	25
9babb374 BB	26	/*
	27	* Fletcher Checksums
	28	* ------------------
	29	*
	30	* ZFS's 2nd and 4th order Fletcher checksums are defined by the following
	31	* recurrence relations:
	32	*
	33	* a = a + f
	34	* i i-1 i-1
	35	*
	36	* b = b + a
	37	* i i-1 i
	38	*
	39	* c = c + b (fletcher-4 only)
	40	* i i-1 i
	41	*
	42	* d = d + c (fletcher-4 only)
	43	* i i-1 i
	44	*
	45	* Where
	46	* a_0 = b_0 = c_0 = d_0 = 0
	47	* and
	48	* f_0 .. f_(n-1) are the input data.
	49	*
	50	* Using standard techniques, these translate into the following series:
	51	*
	52	* __n_ __n_
	53	* \ \| \ \|
	54	* a = > f b = > i * f
	55	* n /___\| n - i n /___\| n - i
	56	* i = 1 i = 1
	57	*
	58	*
	59	* __n_ __n_
	60	* \ \| i(i+1) \ \| i(i+1)*(i+2)
	61	* c = > ------- f d = > ------------- f
	62	* n /___\| 2 n - i n /___\| 6 n - i
	63	* i = 1 i = 1
	64	*
	65	* For fletcher-2, the f_is are 64-bit, and [ab]_i are 64-bit accumulators.
	66	* Since the additions are done mod (2^64), errors in the high bits may not
	67	* be noticed. For this reason, fletcher-2 is deprecated.
	68	*
	69	* For fletcher-4, the f_is are 32-bit, and [abcd]_i are 64-bit accumulators.
	70	* A conservative estimate of how big the buffer can get before we overflow
	71	* can be estimated using f_i = 0xffffffff for all i:
	72	*
	73	* % bc
	74	* f=2^32-1;d=0; for (i = 1; d<2^64; i++) { d += fi(i+1)(i+2)/6 }; (i-1)4
	75	* 2264
	76	* quit
	77	* %
	78	*
	79	* So blocks of up to 2k will not overflow. Our largest block size is
	80	* 128k, which has 32k 4-byte words, so we can compute the largest possible
	81	* accumulators, then divide by 2^64 to figure the max amount of overflow:
	82	*
	83	* % bc
	84	* a=b=c=d=0; f=2^32-1; for (i=1; i<=32*1024; i++) { a+=f; b+=a; c+=b; d+=c }
	85	* a/2^64;b/2^64;c/2^64;d/2^64
	86	* 0
	87	* 0
	88	* 1365
	89	* 11186858
90	* quit
91	* %
92	*
93	* So a and b cannot overflow. To make sure each bit of input has some
94	* effect on the contents of c and d, we can look at what the factors of
95	* the coefficients in the equations for c_n and d_n are. The number of 2s
96	* in the factors determines the lowest set bit in the multiplier. Running
97	* through the cases for n*(n+1)/2 reveals that the highest power of 2 is
98	* 2^14, and for n(n+1)(n+2)/6 it is 2^15. So while some data may overflow
99	* the 64-bit accumulators, every bit of every f_i effects every accumulator,
100	* even for 128k blocks.
101	*
102	* If we wanted to make a stronger version of fletcher4 (fletcher4c?),
103	* we could do our calculations mod (2^32 - 1) by adding in the carries
104	* periodically, and store the number of carries in the top 32-bits.
105	*
106	* --------------------
107	* Checksum Performance
108	* --------------------
109	*
110	* There are two interesting components to checksum performance: cached and
111	* uncached performance. With cached data, fletcher-2 is about four times
112	* faster than fletcher-4. With uncached data, the performance difference is
113	* negligible, since the cost of a cache fill dominates the processing time.
114	* Even though fletcher-4 is slower than fletcher-2, it is still a pretty
115	* efficient pass over the data.
116	*
117	* In normal operation, the data which is being checksummed is in a buffer
118	* which has been filled either by:
119	*
120	* 1. a compression step, which will be mostly cached, or
121	* 2. a bcopy() or copyin(), which will be uncached (because the
122	* copy is cache-bypassing).
123	*
124	* For both cached and uncached data, both fletcher checksums are much faster
125	* than sha-256, and slower than 'off', which doesn't touch the data at all.
126	*/
34dc7c2f BB	127
	128	#include <sys/types.h>
	129	#include <sys/sysmacros.h>
	130	#include <sys/byteorder.h>
	131	#include <sys/spa.h>
1eeb4562 JX	132	#include <sys/zfs_context.h>
	133	#include <zfs_fletcher.h>
	134
	135	static void fletcher_4_scalar_init(zio_cksum_t *zcp);
	136	static void fletcher_4_scalar(const void *buf, uint64_t size,
	137	zio_cksum_t *zcp);
	138	static void fletcher_4_scalar_byteswap(const void *buf, uint64_t size,
	139	zio_cksum_t *zcp);
	140	static boolean_t fletcher_4_scalar_valid(void);
	141
	142	static const fletcher_4_ops_t fletcher_4_scalar_ops = {
	143	.init = fletcher_4_scalar_init,
	144	.compute = fletcher_4_scalar,
	145	.compute_byteswap = fletcher_4_scalar_byteswap,
	146	.valid = fletcher_4_scalar_valid,
	147	.name = "scalar"
	148	};
	149
	150	static const fletcher_4_ops_t *fletcher_4_algos[] = {
	151	&fletcher_4_scalar_ops,
	152	#if defined(HAVE_AVX) && defined(HAVE_AVX2)
	153	&fletcher_4_avx2_ops,
	154	#endif
	155	};
	156
	157	static enum fletcher_selector {
	158	FLETCHER_FASTEST = 0,
	159	FLETCHER_SCALAR,
	160	#if defined(HAVE_AVX) && defined(HAVE_AVX2)
	161	FLETCHER_AVX2,
	162	#endif
	163	FLETCHER_CYCLE
	164	} fletcher_4_impl_chosen = FLETCHER_SCALAR;
	165
	166	static struct fletcher_4_impl_selector {
	167	const char *fis_name;
	168	const fletcher_4_ops_t *fis_ops;
	169	} fletcher_4_impl_selectors[] = {
	170	[ FLETCHER_FASTEST ] = { "fastest", NULL },
	171	[ FLETCHER_SCALAR ] = { "scalar", &fletcher_4_scalar_ops },
	172	#if defined(HAVE_AVX) && defined(HAVE_AVX2)
	173	[ FLETCHER_AVX2 ] = { "avx2", &fletcher_4_avx2_ops },
	174	#endif
	175	#if !defined(_KERNEL)
	176	[ FLETCHER_CYCLE ] = { "cycle", &fletcher_4_scalar_ops }
	177	#endif
	178	};
	179
	180	static kmutex_t fletcher_4_impl_lock;
	181
	182	static kstat_t *fletcher_4_kstat;
	183
	184	static kstat_named_t fletcher_4_kstat_data[ARRAY_SIZE(fletcher_4_algos)];
34dc7c2f BB	185
	186	void
	187	fletcher_2_native(const void buf, uint64_t size, zio_cksum_t zcp)
	188	{
	189	const uint64_t *ip = buf;
	190	const uint64_t *ipend = ip + (size / sizeof (uint64_t));
	191	uint64_t a0, b0, a1, b1;
	192
	193	for (a0 = b0 = a1 = b1 = 0; ip < ipend; ip += 2) {
	194	a0 += ip[0];
	195	a1 += ip[1];
	196	b0 += a0;
	197	b1 += a1;
	198	}
	199
	200	ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
	201	}
	202
	203	void
	204	fletcher_2_byteswap(const void buf, uint64_t size, zio_cksum_t zcp)
	205	{
	206	const uint64_t *ip = buf;
	207	const uint64_t *ipend = ip + (size / sizeof (uint64_t));
	208	uint64_t a0, b0, a1, b1;
	209
	210	for (a0 = b0 = a1 = b1 = 0; ip < ipend; ip += 2) {
	211	a0 += BSWAP_64(ip[0]);
	212	a1 += BSWAP_64(ip[1]);
	213	b0 += a0;
	214	b1 += a1;
	215	}
	216
	217	ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
	218	}
	219
1eeb4562	220	static void fletcher_4_scalar_init(zio_cksum_t *zcp)
34dc7c2f	221	{
1eeb4562	222	ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
34dc7c2f BB	223	}
34dc7c2f BB	224
1eeb4562 JX	225	static void
1eeb4562 JX	226	fletcher_4_scalar(const void buf, uint64_t size, zio_cksum_t zcp)
34dc7c2f BB	227	{
	228	const uint32_t *ip = buf;
	229	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
	230	uint64_t a, b, c, d;
	231
1eeb4562 JX	232	a = zcp->zc_word[0];
	233	b = zcp->zc_word[1];
	234	c = zcp->zc_word[2];
	235	d = zcp->zc_word[3];
	236
	237	for (; ip < ipend; ip++) {
	238	a += ip[0];
34dc7c2f BB	239	b += a;
	240	c += b;
	241	d += c;
	242	}
	243
	244	ZIO_SET_CHECKSUM(zcp, a, b, c, d);
	245	}
	246
1eeb4562 JX	247	static void
1eeb4562 JX	248	fletcher_4_scalar_byteswap(const void buf, uint64_t size, zio_cksum_t zcp)
34dc7c2f BB	249	{
	250	const uint32_t *ip = buf;
	251	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
	252	uint64_t a, b, c, d;
	253
	254	a = zcp->zc_word[0];
	255	b = zcp->zc_word[1];
	256	c = zcp->zc_word[2];
	257	d = zcp->zc_word[3];
	258
	259	for (; ip < ipend; ip++) {
1eeb4562	260	a += BSWAP_32(ip[0]);
34dc7c2f BB	261	b += a;
	262	c += b;
	263	d += c;
	264	}
	265
	266	ZIO_SET_CHECKSUM(zcp, a, b, c, d);
	267	}
	268
1eeb4562 JX	269	static boolean_t
	270	fletcher_4_scalar_valid(void)
	271	{
	272	return (B_TRUE);
	273	}
	274
	275	int
	276	fletcher_4_impl_set(const char *val)
	277	{
	278	const fletcher_4_ops_t *ops;
	279	enum fletcher_selector idx;
	280	size_t val_len;
	281	unsigned i;
	282
	283	val_len = strlen(val);
	284	while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */
	285	val_len--;
	286
	287	for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) {
	288	const char *name = fletcher_4_impl_selectors[i].fis_name;
	289
	290	if (val_len == strlen(name) &&
	291	strncmp(val, name, val_len) == 0) {
	292	idx = i;
	293	break;
	294	}
	295	}
	296	if (i >= ARRAY_SIZE(fletcher_4_impl_selectors))
	297	return (-EINVAL);
	298
	299	ops = fletcher_4_impl_selectors[idx].fis_ops;
	300	if (ops == NULL \|\| !ops->valid())
	301	return (-ENOTSUP);
	302
	303	mutex_enter(&fletcher_4_impl_lock);
	304	if (fletcher_4_impl_chosen != idx)
	305	fletcher_4_impl_chosen = idx;
	306	mutex_exit(&fletcher_4_impl_lock);
	307
	308	return (0);
	309	}
	310
	311	static inline const fletcher_4_ops_t *
	312	fletcher_4_impl_get(void)
	313	{
	314	#if !defined(_KERNEL)
	315	if (fletcher_4_impl_chosen == FLETCHER_CYCLE) {
	316	static volatile unsigned int cycle_count = 0;
	317	const fletcher_4_ops_t *ops = NULL;
	318	unsigned int index;
	319
	320	while (1) {
	321	index = atomic_inc_uint_nv(&cycle_count);
	322	ops = fletcher_4_algos[
	323	index % ARRAY_SIZE(fletcher_4_algos)];
	324	if (ops->valid())
	325	break;
	326	}
	327	return (ops);
	328	}
	329	#endif
	330	membar_producer();
	331	return (fletcher_4_impl_selectors[fletcher_4_impl_chosen].fis_ops);
	332	}
333
334	void
335	fletcher_4_native(const void buf, uint64_t size, zio_cksum_t zcp)
336	{
337	const fletcher_4_ops_t *ops = fletcher_4_impl_get();
338
339	ops->init(zcp);
340	ops->compute(buf, size, zcp);
341	if (ops->fini != NULL)
342	ops->fini(zcp);
343	}
344
345	void
346	fletcher_4_byteswap(const void buf, uint64_t size, zio_cksum_t zcp)
347	{
348	const fletcher_4_ops_t *ops = fletcher_4_impl_get();
349
350	ops->init(zcp);
351	ops->compute_byteswap(buf, size, zcp);
352	if (ops->fini != NULL)
353	ops->fini(zcp);
354	}
355
356	void
357	fletcher_4_incremental_native(const void *buf, uint64_t size,
358	zio_cksum_t *zcp)
359	{
360	fletcher_4_scalar(buf, size, zcp);
361	}
362
34dc7c2f BB	363	void
	364	fletcher_4_incremental_byteswap(const void *buf, uint64_t size,
	365	zio_cksum_t *zcp)
	366	{
1eeb4562 JX	367	fletcher_4_scalar_byteswap(buf, size, zcp);
1eeb4562 JX	368	}
34dc7c2f	369
1eeb4562 JX	370	void
	371	fletcher_4_init(void)
	372	{
	373	const uint64_t const bench_ns = (50 * MICROSEC); /* 50ms */
	374	unsigned long best_run_count = 0;
	375	unsigned long best_run_index = 0;
	376	const unsigned data_size = 4096;
	377	char *databuf;
	378	int i;
34dc7c2f	379
1eeb4562 JX	380	databuf = kmem_alloc(data_size, KM_SLEEP);
	381	for (i = 0; i < ARRAY_SIZE(fletcher_4_algos); i++) {
	382	const fletcher_4_ops_t *ops = fletcher_4_algos[i];
	383	kstat_named_t *stat = &fletcher_4_kstat_data[i];
	384	unsigned long run_count = 0;
	385	hrtime_t start;
	386	zio_cksum_t zc;
	387
	388	strncpy(stat->name, ops->name, sizeof (stat->name) - 1);
	389	stat->data_type = KSTAT_DATA_UINT64;
	390	stat->value.ui64 = 0;
	391
	392	if (!ops->valid())
	393	continue;
	394
	395	kpreempt_disable();
	396	start = gethrtime();
	397	ops->init(&zc);
	398	do {
	399	ops->compute(databuf, data_size, &zc);
	400	run_count++;
	401	} while (gethrtime() < start + bench_ns);
	402	if (ops->fini != NULL)
	403	ops->fini(&zc);
	404	kpreempt_enable();
	405
	406	if (run_count > best_run_count) {
	407	best_run_count = run_count;
	408	best_run_index = i;
	409	}
	410
	411	/*
	412	* Due to high overhead of gethrtime(), the performance data
	413	* here is inaccurate and much slower than it could be.
	414	* It's fine for our use though because only relative speed
	415	* is important.
	416	*/
	417	stat->value.ui64 = data_size * run_count *
	418	(NANOSEC / bench_ns) >> 20; /* by MB/s */
34dc7c2f	419	}
1eeb4562	420	kmem_free(databuf, data_size);
34dc7c2f	421
1eeb4562 JX	422	fletcher_4_impl_selectors[FLETCHER_FASTEST].fis_ops =
	423	fletcher_4_algos[best_run_index];
	424
	425	mutex_init(&fletcher_4_impl_lock, NULL, MUTEX_DEFAULT, NULL);
	426	fletcher_4_impl_set("fastest");
	427
	428	fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench",
	429	"misc", KSTAT_TYPE_NAMED, ARRAY_SIZE(fletcher_4_algos),
	430	KSTAT_FLAG_VIRTUAL);
	431	if (fletcher_4_kstat != NULL) {
	432	fletcher_4_kstat->ks_data = fletcher_4_kstat_data;
	433	kstat_install(fletcher_4_kstat);
	434	}
	435	}
	436
	437	void
	438	fletcher_4_fini(void)
	439	{
	440	mutex_destroy(&fletcher_4_impl_lock);
	441	if (fletcher_4_kstat != NULL) {
	442	kstat_delete(fletcher_4_kstat);
	443	fletcher_4_kstat = NULL;
	444	}
34dc7c2f	445	}
c28b2279 BB	446
c28b2279 BB	447	#if defined(_KERNEL) && defined(HAVE_SPL)
1eeb4562 JX	448
	449	static int
	450	fletcher_4_param_get(char buffer, struct kernel_param unused)
	451	{
	452	int i, cnt = 0;
	453
	454	for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) {
	455	const fletcher_4_ops_t *ops;
	456
	457	ops = fletcher_4_impl_selectors[i].fis_ops;
	458	if (!ops->valid())
	459	continue;
	460
	461	cnt += sprintf(buffer + cnt,
	462	fletcher_4_impl_chosen == i ? "[%s] " : "%s ",
	463	fletcher_4_impl_selectors[i].fis_name);
	464	}
	465
	466	return (cnt);
	467	}
	468
	469	static int
	470	fletcher_4_param_set(const char val, struct kernel_param unused)
	471	{
	472	return (fletcher_4_impl_set(val));
	473	}
	474
	475	/*
	476	* Choose a fletcher 4 implementation in ZFS.
	477	* Users can choose the "fastest" algorithm, or "scalar" and "avx2" which means
	478	* to compute fletcher 4 by CPU or vector instructions respectively.
	479	* Users can also choose "cycle" to exercise all implementions, but this is
	480	* for testing purpose therefore it can only be set in user space.
	481	*/
	482	module_param_call(zfs_fletcher_4_impl,
	483	fletcher_4_param_set, fletcher_4_param_get, NULL, 0644);
	484	MODULE_PARM_DESC(zfs_fletcher_4_impl, "Select fletcher 4 algorithm");
	485
	486	EXPORT_SYMBOL(fletcher_4_init);
	487	EXPORT_SYMBOL(fletcher_4_fini);
c28b2279 BB	488	EXPORT_SYMBOL(fletcher_2_native);
	489	EXPORT_SYMBOL(fletcher_2_byteswap);
	490	EXPORT_SYMBOL(fletcher_4_native);
	491	EXPORT_SYMBOL(fletcher_4_byteswap);
	492	EXPORT_SYMBOL(fletcher_4_incremental_native);
	493	EXPORT_SYMBOL(fletcher_4_incremental_byteswap);
	494	#endif