[mirror_zfs.git] / include / zfs_fletcher.h

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or https://opensource.org/licenses/CDDL-1.0.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright 2013 Saso Kiselkov. All rights reserved.
 */

#ifndef	_ZFS_FLETCHER_H
#define	_ZFS_FLETCHER_H extern __attribute__((visibility("default")))

#include <sys/types.h>
#include <sys/spa_checksum.h>

#ifdef	__cplusplus
extern "C" {
#endif

/*
 * fletcher checksum functions
 *
 * Note: Fletcher checksum methods expect buffer size to be 4B aligned. This
 * limitation stems from the algorithm design. Performing incremental checksum
 * without said alignment would yield different results. Therefore, the code
 * includes assertions for the size alignment.
 * For compatibility, it is required that some code paths calculate checksum of
 * non-aligned buffer sizes. For this purpose, `fletcher_4_native_varsize()`
 * checksum method is added. This method will ignore last (size % 4) bytes of
 * the data buffer.
 */
_ZFS_FLETCHER_H void fletcher_init(zio_cksum_t *);
_ZFS_FLETCHER_H void fletcher_2_native(const void *, uint64_t, const void *,
    zio_cksum_t *);
_ZFS_FLETCHER_H void fletcher_2_byteswap(const void *, uint64_t, const void *,
    zio_cksum_t *);
_ZFS_FLETCHER_H void fletcher_4_native(const void *, uint64_t, const void *,
    zio_cksum_t *);
_ZFS_FLETCHER_H int fletcher_2_incremental_native(void *, size_t, void *);
_ZFS_FLETCHER_H int fletcher_2_incremental_byteswap(void *, size_t, void *);
_ZFS_FLETCHER_H void fletcher_4_native_varsize(const void *, uint64_t,
    zio_cksum_t *);
_ZFS_FLETCHER_H void fletcher_4_byteswap(const void *, uint64_t, const void *,
    zio_cksum_t *);
_ZFS_FLETCHER_H int fletcher_4_incremental_native(void *, size_t, void *);
_ZFS_FLETCHER_H int fletcher_4_incremental_byteswap(void *, size_t, void *);
_ZFS_FLETCHER_H int fletcher_4_impl_set(const char *selector);
_ZFS_FLETCHER_H void fletcher_4_init(void);
_ZFS_FLETCHER_H void fletcher_4_fini(void);


/* Internal fletcher ctx */

typedef struct zfs_fletcher_superscalar {
	uint64_t v[4];
} zfs_fletcher_superscalar_t;

typedef struct zfs_fletcher_sse {
	uint64_t v[2];
} zfs_fletcher_sse_t;

typedef struct zfs_fletcher_avx {
	uint64_t v[4];
} zfs_fletcher_avx_t;

typedef struct zfs_fletcher_avx512 {
	uint64_t v[8];
} zfs_fletcher_avx512_t;

typedef struct zfs_fletcher_aarch64_neon {
	uint64_t v[2];
} zfs_fletcher_aarch64_neon_t;


typedef union fletcher_4_ctx {
	zio_cksum_t scalar;
	zfs_fletcher_superscalar_t superscalar[4];

#if defined(HAVE_SSE2) || (defined(HAVE_SSE2) && defined(HAVE_SSSE3))
	zfs_fletcher_sse_t sse[4];
#endif
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
	zfs_fletcher_avx_t avx[4];
#endif
#if defined(__x86_64) && defined(HAVE_AVX512F)
	zfs_fletcher_avx512_t avx512[4];
#endif
#if defined(__aarch64__)
	zfs_fletcher_aarch64_neon_t aarch64_neon[4];
#endif
} fletcher_4_ctx_t;

/*
 * fletcher checksum struct
 */
typedef void (*fletcher_4_init_f)(fletcher_4_ctx_t *);
typedef void (*fletcher_4_fini_f)(fletcher_4_ctx_t *, zio_cksum_t *);
typedef void (*fletcher_4_compute_f)(fletcher_4_ctx_t *,
    const void *, uint64_t);

typedef struct fletcher_4_func {
	fletcher_4_init_f init_native;
	fletcher_4_fini_f fini_native;
	fletcher_4_compute_f compute_native;
	fletcher_4_init_f init_byteswap;
	fletcher_4_fini_f fini_byteswap;
	fletcher_4_compute_f compute_byteswap;
	boolean_t (*valid)(void);
	boolean_t uses_fpu;
	const char *name;
} __attribute__((aligned(64))) fletcher_4_ops_t;

_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar_ops;
_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar4_ops;

#if defined(HAVE_SSE2)
_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_sse2_ops;
#endif

#if defined(HAVE_SSE2) && defined(HAVE_SSSE3)
_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_ssse3_ops;
#endif

#if defined(HAVE_AVX) && defined(HAVE_AVX2)
_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx2_ops;
#endif

#if defined(__x86_64) && defined(HAVE_AVX512F)
_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512f_ops;
#endif

#if defined(__x86_64) && defined(HAVE_AVX512BW)
_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512bw_ops;
#endif

#if defined(__aarch64__)
_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_aarch64_neon_ops;
#endif

#ifdef	__cplusplus
}
#endif

#endif	/* _ZFS_FLETCHER_H */
Commit	Line	Data
	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or https://opensource.org/licenses/CDDL-1.0.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
	22	* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
	23	* Use is subject to license terms.
	24	*/
	25	/*
	26	* Copyright 2013 Saso Kiselkov. All rights reserved.
	27	*/
	28
	29	#ifndef _ZFS_FLETCHER_H
	30	#define _ZFS_FLETCHER_H extern __attribute__((visibility("default")))
	31
	32	#include <sys/types.h>
	33	#include <sys/spa_checksum.h>
	34
	35	#ifdef __cplusplus
	36	extern "C" {
	37	#endif
	38
	39	/*
	40	* fletcher checksum functions
	41	*
	42	* Note: Fletcher checksum methods expect buffer size to be 4B aligned. This
	43	* limitation stems from the algorithm design. Performing incremental checksum
	44	* without said alignment would yield different results. Therefore, the code
	45	* includes assertions for the size alignment.
	46	* For compatibility, it is required that some code paths calculate checksum of
	47	* non-aligned buffer sizes. For this purpose, `fletcher_4_native_varsize()`
	48	* checksum method is added. This method will ignore last (size % 4) bytes of
	49	* the data buffer.
	50	*/
	51	_ZFS_FLETCHER_H void fletcher_init(zio_cksum_t *);
	52	_ZFS_FLETCHER_H void fletcher_2_native(const void , uint64_t, const void ,
	53	zio_cksum_t *);
	54	_ZFS_FLETCHER_H void fletcher_2_byteswap(const void , uint64_t, const void ,
	55	zio_cksum_t *);
	56	_ZFS_FLETCHER_H void fletcher_4_native(const void , uint64_t, const void ,
	57	zio_cksum_t *);
	58	_ZFS_FLETCHER_H int fletcher_2_incremental_native(void , size_t, void );
	59	_ZFS_FLETCHER_H int fletcher_2_incremental_byteswap(void , size_t, void );
	60	_ZFS_FLETCHER_H void fletcher_4_native_varsize(const void *, uint64_t,
	61	zio_cksum_t *);
	62	_ZFS_FLETCHER_H void fletcher_4_byteswap(const void , uint64_t, const void ,
	63	zio_cksum_t *);
	64	_ZFS_FLETCHER_H int fletcher_4_incremental_native(void , size_t, void );
	65	_ZFS_FLETCHER_H int fletcher_4_incremental_byteswap(void , size_t, void );
	66	_ZFS_FLETCHER_H int fletcher_4_impl_set(const char *selector);
	67	_ZFS_FLETCHER_H void fletcher_4_init(void);
	68	_ZFS_FLETCHER_H void fletcher_4_fini(void);
	69
	70
	71
	72	/* Internal fletcher ctx */
	73
	74	typedef struct zfs_fletcher_superscalar {
	75	uint64_t v[4];
	76	} zfs_fletcher_superscalar_t;
	77
	78	typedef struct zfs_fletcher_sse {
	79	uint64_t v[2];
	80	} zfs_fletcher_sse_t;
	81
	82	typedef struct zfs_fletcher_avx {
	83	uint64_t v[4];
	84	} zfs_fletcher_avx_t;
	85
	86	typedef struct zfs_fletcher_avx512 {
	87	uint64_t v[8];
	88	} zfs_fletcher_avx512_t;
	89
	90	typedef struct zfs_fletcher_aarch64_neon {
	91	uint64_t v[2];
	92	} zfs_fletcher_aarch64_neon_t;
	93
	94
	95	typedef union fletcher_4_ctx {
	96	zio_cksum_t scalar;
	97	zfs_fletcher_superscalar_t superscalar[4];
	98
	99	#if defined(HAVE_SSE2) \|\| (defined(HAVE_SSE2) && defined(HAVE_SSSE3))
	100	zfs_fletcher_sse_t sse[4];
	101	#endif
	102	#if defined(HAVE_AVX) && defined(HAVE_AVX2)
	103	zfs_fletcher_avx_t avx[4];
	104	#endif
	105	#if defined(__x86_64) && defined(HAVE_AVX512F)
	106	zfs_fletcher_avx512_t avx512[4];
	107	#endif
	108	#if defined(__aarch64__)
	109	zfs_fletcher_aarch64_neon_t aarch64_neon[4];
	110	#endif
	111	} fletcher_4_ctx_t;
	112
	113	/*
	114	* fletcher checksum struct
	115	*/
	116	typedef void (fletcher_4_init_f)(fletcher_4_ctx_t );
	117	typedef void (fletcher_4_fini_f)(fletcher_4_ctx_t , zio_cksum_t *);
	118	typedef void (fletcher_4_compute_f)(fletcher_4_ctx_t ,
	119	const void *, uint64_t);
	120
	121	typedef struct fletcher_4_func {
	122	fletcher_4_init_f init_native;
	123	fletcher_4_fini_f fini_native;
	124	fletcher_4_compute_f compute_native;
	125	fletcher_4_init_f init_byteswap;
	126	fletcher_4_fini_f fini_byteswap;
	127	fletcher_4_compute_f compute_byteswap;
	128	boolean_t (*valid)(void);
	129	boolean_t uses_fpu;
	130	const char *name;
	131	} __attribute__((aligned(64))) fletcher_4_ops_t;
	132
	133	_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar_ops;
	134	_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar4_ops;
	135
	136	#if defined(HAVE_SSE2)
	137	_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_sse2_ops;
	138	#endif
	139
	140	#if defined(HAVE_SSE2) && defined(HAVE_SSSE3)
	141	_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_ssse3_ops;
	142	#endif
	143
	144	#if defined(HAVE_AVX) && defined(HAVE_AVX2)
	145	_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx2_ops;
	146	#endif
	147
	148	#if defined(__x86_64) && defined(HAVE_AVX512F)
	149	_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512f_ops;
	150	#endif
	151
	152	#if defined(__x86_64) && defined(HAVE_AVX512BW)
	153	_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512bw_ops;
	154	#endif
	155
	156	#if defined(__aarch64__)
	157	_ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_aarch64_neon_ops;
	158	#endif
	159
	160	#ifdef __cplusplus
	161	}
	162	#endif
	163
	164	#endif /* _ZFS_FLETCHER_H */