4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
27 * Copyright 2013 Saso Kiselkov. All rights reserved.
34 * ZFS's 2nd and 4th order Fletcher checksums are defined by the following
35 * recurrence relations:
43 * c = c + b (fletcher-4 only)
46 * d = d + c (fletcher-4 only)
50 * a_0 = b_0 = c_0 = d_0 = 0
52 * f_0 .. f_(n-1) are the input data.
54 * Using standard techniques, these translate into the following series:
59 * n /___| n - i n /___| n - i
64 * \ | i*(i+1) \ | i*(i+1)*(i+2)
65 * c = > ------- f d = > ------------- f
66 * n /___| 2 n - i n /___| 6 n - i
69 * For fletcher-2, the f_is are 64-bit, and [ab]_i are 64-bit accumulators.
70 * Since the additions are done mod (2^64), errors in the high bits may not
71 * be noticed. For this reason, fletcher-2 is deprecated.
73 * For fletcher-4, the f_is are 32-bit, and [abcd]_i are 64-bit accumulators.
74 * A conservative estimate of how big the buffer can get before we overflow
75 * can be estimated using f_i = 0xffffffff for all i:
78 * f=2^32-1;d=0; for (i = 1; d<2^64; i++) { d += f*i*(i+1)*(i+2)/6 }; (i-1)*4
83 * So blocks of up to 2k will not overflow. Our largest block size is
84 * 128k, which has 32k 4-byte words, so we can compute the largest possible
85 * accumulators, then divide by 2^64 to figure the max amount of overflow:
88 * a=b=c=d=0; f=2^32-1; for (i=1; i<=32*1024; i++) { a+=f; b+=a; c+=b; d+=c }
89 * a/2^64;b/2^64;c/2^64;d/2^64
97 * So a and b cannot overflow. To make sure each bit of input has some
98 * effect on the contents of c and d, we can look at what the factors of
99 * the coefficients in the equations for c_n and d_n are. The number of 2s
100 * in the factors determines the lowest set bit in the multiplier. Running
101 * through the cases for n*(n+1)/2 reveals that the highest power of 2 is
102 * 2^14, and for n*(n+1)*(n+2)/6 it is 2^15. So while some data may overflow
103 * the 64-bit accumulators, every bit of every f_i effects every accumulator,
104 * even for 128k blocks.
106 * If we wanted to make a stronger version of fletcher4 (fletcher4c?),
107 * we could do our calculations mod (2^32 - 1) by adding in the carries
108 * periodically, and store the number of carries in the top 32-bits.
110 * --------------------
111 * Checksum Performance
112 * --------------------
114 * There are two interesting components to checksum performance: cached and
115 * uncached performance. With cached data, fletcher-2 is about four times
116 * faster than fletcher-4. With uncached data, the performance difference is
117 * negligible, since the cost of a cache fill dominates the processing time.
118 * Even though fletcher-4 is slower than fletcher-2, it is still a pretty
119 * efficient pass over the data.
121 * In normal operation, the data which is being checksummed is in a buffer
122 * which has been filled either by:
124 * 1. a compression step, which will be mostly cached, or
125 * 2. a bcopy() or copyin(), which will be uncached (because the
126 * copy is cache-bypassing).
128 * For both cached and uncached data, both fletcher checksums are much faster
129 * than sha-256, and slower than 'off', which doesn't touch the data at all.
132 #include <sys/types.h>
133 #include <sys/sysmacros.h>
134 #include <sys/byteorder.h>
136 #include <sys/zio_checksum.h>
137 #include <sys/zfs_context.h>
138 #include <zfs_fletcher.h>
141 static void fletcher_4_scalar_init(zio_cksum_t
*zcp
);
142 static void fletcher_4_scalar_native(const void *buf
, uint64_t size
,
144 static void fletcher_4_scalar_byteswap(const void *buf
, uint64_t size
,
146 static boolean_t
fletcher_4_scalar_valid(void);
148 static const fletcher_4_ops_t fletcher_4_scalar_ops
= {
149 .init_native
= fletcher_4_scalar_init
,
150 .compute_native
= fletcher_4_scalar_native
,
151 .init_byteswap
= fletcher_4_scalar_init
,
152 .compute_byteswap
= fletcher_4_scalar_byteswap
,
153 .valid
= fletcher_4_scalar_valid
,
157 static fletcher_4_ops_t fletcher_4_fastest_impl
= {
159 .valid
= fletcher_4_scalar_valid
162 static const fletcher_4_ops_t
*fletcher_4_impls
[] = {
163 &fletcher_4_scalar_ops
,
164 #if defined(HAVE_SSE2)
165 &fletcher_4_sse2_ops
,
167 #if defined(HAVE_SSE2) && defined(HAVE_SSSE3)
168 &fletcher_4_ssse3_ops
,
170 #if defined(HAVE_AVX) && defined(HAVE_AVX2)
171 &fletcher_4_avx2_ops
,
173 #if defined(__x86_64) && defined(HAVE_AVX512F)
174 &fletcher_4_avx512f_ops
,
178 /* Hold all supported implementations */
179 static uint32_t fletcher_4_supp_impls_cnt
= 0;
180 static fletcher_4_ops_t
*fletcher_4_supp_impls
[ARRAY_SIZE(fletcher_4_impls
)];
182 /* Select fletcher4 implementation */
183 #define IMPL_FASTEST (UINT32_MAX)
184 #define IMPL_CYCLE (UINT32_MAX - 1)
185 #define IMPL_SCALAR (0)
187 static uint32_t fletcher_4_impl_chosen
= IMPL_FASTEST
;
189 #define IMPL_READ(i) (*(volatile uint32_t *) &(i))
191 static struct fletcher_4_impl_selector
{
192 const char *fis_name
;
194 } fletcher_4_impl_selectors
[] = {
195 #if !defined(_KERNEL)
196 { "cycle", IMPL_CYCLE
},
198 { "fastest", IMPL_FASTEST
},
199 { "scalar", IMPL_SCALAR
}
202 static kstat_t
*fletcher_4_kstat
;
204 static struct fletcher_4_kstat
{
207 } fletcher_4_stat_data
[ARRAY_SIZE(fletcher_4_impls
) + 1];
209 /* Indicate that benchmark has been completed */
210 static boolean_t fletcher_4_initialized
= B_FALSE
;
214 fletcher_2_native(const void *buf
, uint64_t size
,
215 const void *ctx_template
, zio_cksum_t
*zcp
)
217 const uint64_t *ip
= buf
;
218 const uint64_t *ipend
= ip
+ (size
/ sizeof (uint64_t));
219 uint64_t a0
, b0
, a1
, b1
;
221 for (a0
= b0
= a1
= b1
= 0; ip
< ipend
; ip
+= 2) {
228 ZIO_SET_CHECKSUM(zcp
, a0
, a1
, b0
, b1
);
233 fletcher_2_byteswap(const void *buf
, uint64_t size
,
234 const void *ctx_template
, zio_cksum_t
*zcp
)
236 const uint64_t *ip
= buf
;
237 const uint64_t *ipend
= ip
+ (size
/ sizeof (uint64_t));
238 uint64_t a0
, b0
, a1
, b1
;
240 for (a0
= b0
= a1
= b1
= 0; ip
< ipend
; ip
+= 2) {
241 a0
+= BSWAP_64(ip
[0]);
242 a1
+= BSWAP_64(ip
[1]);
247 ZIO_SET_CHECKSUM(zcp
, a0
, a1
, b0
, b1
);
251 fletcher_4_scalar_init(zio_cksum_t
*zcp
)
253 ZIO_SET_CHECKSUM(zcp
, 0, 0, 0, 0);
257 fletcher_4_scalar_native(const void *buf
, uint64_t size
, zio_cksum_t
*zcp
)
259 const uint32_t *ip
= buf
;
260 const uint32_t *ipend
= ip
+ (size
/ sizeof (uint32_t));
268 for (; ip
< ipend
; ip
++) {
275 ZIO_SET_CHECKSUM(zcp
, a
, b
, c
, d
);
279 fletcher_4_scalar_byteswap(const void *buf
, uint64_t size
, zio_cksum_t
*zcp
)
281 const uint32_t *ip
= buf
;
282 const uint32_t *ipend
= ip
+ (size
/ sizeof (uint32_t));
290 for (; ip
< ipend
; ip
++) {
291 a
+= BSWAP_32(ip
[0]);
297 ZIO_SET_CHECKSUM(zcp
, a
, b
, c
, d
);
301 fletcher_4_scalar_valid(void)
307 fletcher_4_impl_set(const char *val
)
310 uint32_t impl
= IMPL_READ(fletcher_4_impl_chosen
);
313 val_len
= strlen(val
);
314 while ((val_len
> 0) && !!isspace(val
[val_len
-1])) /* trim '\n' */
317 /* check mandatory implementations */
318 for (i
= 0; i
< ARRAY_SIZE(fletcher_4_impl_selectors
); i
++) {
319 const char *name
= fletcher_4_impl_selectors
[i
].fis_name
;
321 if (val_len
== strlen(name
) &&
322 strncmp(val
, name
, val_len
) == 0) {
323 impl
= fletcher_4_impl_selectors
[i
].fis_sel
;
329 if (err
!= 0 && fletcher_4_initialized
) {
330 /* check all supported implementations */
331 for (i
= 0; i
< fletcher_4_supp_impls_cnt
; i
++) {
332 const char *name
= fletcher_4_supp_impls
[i
]->name
;
334 if (val_len
== strlen(name
) &&
335 strncmp(val
, name
, val_len
) == 0) {
344 atomic_swap_32(&fletcher_4_impl_chosen
, impl
);
351 static inline const fletcher_4_ops_t
*
352 fletcher_4_impl_get(void)
354 fletcher_4_ops_t
*ops
= NULL
;
355 const uint32_t impl
= IMPL_READ(fletcher_4_impl_chosen
);
359 ASSERT(fletcher_4_initialized
);
360 ops
= &fletcher_4_fastest_impl
;
362 #if !defined(_KERNEL)
364 ASSERT(fletcher_4_initialized
);
365 ASSERT3U(fletcher_4_supp_impls_cnt
, >, 0);
367 static uint32_t cycle_count
= 0;
368 uint32_t idx
= (++cycle_count
) % fletcher_4_supp_impls_cnt
;
369 ops
= fletcher_4_supp_impls
[idx
];
374 ASSERT3U(fletcher_4_supp_impls_cnt
, >, 0);
375 ASSERT3U(impl
, <, fletcher_4_supp_impls_cnt
);
377 ops
= fletcher_4_supp_impls
[impl
];
381 ASSERT3P(ops
, !=, NULL
);
387 fletcher_4_incremental_native(const void *buf
, uint64_t size
,
390 ASSERT(IS_P2ALIGNED(size
, sizeof (uint32_t)));
392 fletcher_4_scalar_native(buf
, size
, zcp
);
396 fletcher_4_incremental_byteswap(const void *buf
, uint64_t size
,
399 ASSERT(IS_P2ALIGNED(size
, sizeof (uint32_t)));
401 fletcher_4_scalar_byteswap(buf
, size
, zcp
);
405 fletcher_4_native_impl(const fletcher_4_ops_t
*ops
, const void *buf
,
406 uint64_t size
, zio_cksum_t
*zcp
)
408 ops
->init_native(zcp
);
409 ops
->compute_native(buf
, size
, zcp
);
410 if (ops
->fini_native
!= NULL
)
411 ops
->fini_native(zcp
);
416 fletcher_4_native(const void *buf
, uint64_t size
,
417 const void *ctx_template
, zio_cksum_t
*zcp
)
419 const fletcher_4_ops_t
*ops
;
420 uint64_t p2size
= P2ALIGN(size
, 64);
422 ASSERT(IS_P2ALIGNED(size
, sizeof (uint32_t)));
425 ZIO_SET_CHECKSUM(zcp
, 0, 0, 0, 0);
426 } else if (p2size
== 0) {
427 ops
= &fletcher_4_scalar_ops
;
428 fletcher_4_native_impl(ops
, buf
, size
, zcp
);
430 ops
= fletcher_4_impl_get();
431 fletcher_4_native_impl(ops
, buf
, p2size
, zcp
);
434 fletcher_4_incremental_native((char *)buf
+ p2size
,
440 fletcher_4_native_varsize(const void *buf
, uint64_t size
, zio_cksum_t
*zcp
)
442 fletcher_4_native_impl(&fletcher_4_scalar_ops
, buf
, size
, zcp
);
446 fletcher_4_byteswap_impl(const fletcher_4_ops_t
*ops
, const void *buf
,
447 uint64_t size
, zio_cksum_t
*zcp
)
449 ops
->init_byteswap(zcp
);
450 ops
->compute_byteswap(buf
, size
, zcp
);
451 if (ops
->fini_byteswap
!= NULL
)
452 ops
->fini_byteswap(zcp
);
457 fletcher_4_byteswap(const void *buf
, uint64_t size
,
458 const void *ctx_template
, zio_cksum_t
*zcp
)
460 const fletcher_4_ops_t
*ops
;
461 uint64_t p2size
= P2ALIGN(size
, 64);
463 ASSERT(IS_P2ALIGNED(size
, sizeof (uint32_t)));
466 ZIO_SET_CHECKSUM(zcp
, 0, 0, 0, 0);
467 } else if (p2size
== 0) {
468 ops
= &fletcher_4_scalar_ops
;
469 fletcher_4_byteswap_impl(ops
, buf
, size
, zcp
);
471 ops
= fletcher_4_impl_get();
472 fletcher_4_byteswap_impl(ops
, buf
, p2size
, zcp
);
475 fletcher_4_incremental_byteswap((char *)buf
+ p2size
,
481 fletcher_4_kstat_headers(char *buf
, size_t size
)
485 off
+= snprintf(buf
+ off
, size
, "%-17s", "implementation");
486 off
+= snprintf(buf
+ off
, size
- off
, "%-15s", "native");
487 (void) snprintf(buf
+ off
, size
- off
, "%-15s\n", "byteswap");
493 fletcher_4_kstat_data(char *buf
, size_t size
, void *data
)
495 struct fletcher_4_kstat
*fastest_stat
=
496 &fletcher_4_stat_data
[fletcher_4_supp_impls_cnt
];
497 struct fletcher_4_kstat
*curr_stat
= (struct fletcher_4_kstat
*) data
;
500 if (curr_stat
== fastest_stat
) {
501 off
+= snprintf(buf
+ off
, size
- off
, "%-17s", "fastest");
502 off
+= snprintf(buf
+ off
, size
- off
, "%-15s",
503 fletcher_4_supp_impls
[fastest_stat
->native
]->name
);
504 off
+= snprintf(buf
+ off
, size
- off
, "%-15s\n",
505 fletcher_4_supp_impls
[fastest_stat
->byteswap
]->name
);
507 ptrdiff_t id
= curr_stat
- fletcher_4_stat_data
;
509 off
+= snprintf(buf
+ off
, size
- off
, "%-17s",
510 fletcher_4_supp_impls
[id
]->name
);
511 off
+= snprintf(buf
+ off
, size
- off
, "%-15llu",
512 (u_longlong_t
) curr_stat
->native
);
513 off
+= snprintf(buf
+ off
, size
- off
, "%-15llu\n",
514 (u_longlong_t
) curr_stat
->byteswap
);
521 fletcher_4_kstat_addr(kstat_t
*ksp
, loff_t n
)
523 if (n
<= fletcher_4_supp_impls_cnt
)
524 ksp
->ks_private
= (void *) (fletcher_4_stat_data
+ n
);
526 ksp
->ks_private
= NULL
;
528 return (ksp
->ks_private
);
531 #define FLETCHER_4_FASTEST_FN_COPY(type, src) \
533 fletcher_4_fastest_impl.init_ ## type = src->init_ ## type; \
534 fletcher_4_fastest_impl.fini_ ## type = src->fini_ ## type; \
535 fletcher_4_fastest_impl.compute_ ## type = src->compute_ ## type; \
538 #define FLETCHER_4_BENCH_NS (MSEC2NSEC(50)) /* 50ms */
541 fletcher_4_benchmark_impl(boolean_t native
, char *data
, uint64_t data_size
)
544 struct fletcher_4_kstat
*fastest_stat
=
545 &fletcher_4_stat_data
[fletcher_4_supp_impls_cnt
];
547 uint64_t run_bw
, run_time_ns
, best_run
= 0;
549 uint32_t i
, l
, sel_save
= IMPL_READ(fletcher_4_impl_chosen
);
551 zio_checksum_func_t
*fletcher_4_test
= native
? fletcher_4_native
:
554 for (i
= 0; i
< fletcher_4_supp_impls_cnt
; i
++) {
555 struct fletcher_4_kstat
*stat
= &fletcher_4_stat_data
[i
];
556 uint64_t run_count
= 0;
558 /* temporary set an implementation */
559 fletcher_4_impl_chosen
= i
;
564 for (l
= 0; l
< 32; l
++, run_count
++)
565 fletcher_4_test(data
, data_size
, NULL
, &zc
);
567 run_time_ns
= gethrtime() - start
;
568 } while (run_time_ns
< FLETCHER_4_BENCH_NS
);
571 run_bw
= data_size
* run_count
* NANOSEC
;
572 run_bw
/= run_time_ns
; /* B/s */
575 stat
->native
= run_bw
;
577 stat
->byteswap
= run_bw
;
579 if (run_bw
> best_run
) {
583 fastest_stat
->native
= i
;
584 FLETCHER_4_FASTEST_FN_COPY(native
,
585 fletcher_4_supp_impls
[i
]);
587 fastest_stat
->byteswap
= i
;
588 FLETCHER_4_FASTEST_FN_COPY(byteswap
,
589 fletcher_4_supp_impls
[i
]);
594 /* restore original selection */
595 atomic_swap_32(&fletcher_4_impl_chosen
, sel_save
);
599 fletcher_4_init(void)
601 static const size_t data_size
= 1 << SPA_OLD_MAXBLOCKSHIFT
; /* 128kiB */
602 fletcher_4_ops_t
*curr_impl
;
606 /* move supported impl into fletcher_4_supp_impls */
607 for (i
= 0, c
= 0; i
< ARRAY_SIZE(fletcher_4_impls
); i
++) {
608 curr_impl
= (fletcher_4_ops_t
*) fletcher_4_impls
[i
];
610 if (curr_impl
->valid
&& curr_impl
->valid())
611 fletcher_4_supp_impls
[c
++] = curr_impl
;
613 membar_producer(); /* complete fletcher_4_supp_impls[] init */
614 fletcher_4_supp_impls_cnt
= c
; /* number of supported impl */
616 #if !defined(_KERNEL)
617 /* Skip benchmarking and use last implementation as fastest */
618 memcpy(&fletcher_4_fastest_impl
,
619 fletcher_4_supp_impls
[fletcher_4_supp_impls_cnt
-1],
620 sizeof (fletcher_4_fastest_impl
));
621 fletcher_4_fastest_impl
.name
= "fastest";
624 fletcher_4_initialized
= B_TRUE
;
626 /* Use 'cycle' math selection method for userspace */
627 VERIFY0(fletcher_4_impl_set("cycle"));
630 /* Benchmark all supported implementations */
631 databuf
= vmem_alloc(data_size
, KM_SLEEP
);
632 for (i
= 0; i
< data_size
/ sizeof (uint64_t); i
++)
633 ((uint64_t *)databuf
)[i
] = (uintptr_t)(databuf
+i
); /* warm-up */
635 fletcher_4_benchmark_impl(B_FALSE
, databuf
, data_size
);
636 fletcher_4_benchmark_impl(B_TRUE
, databuf
, data_size
);
638 vmem_free(databuf
, data_size
);
640 /* install kstats for all implementations */
641 fletcher_4_kstat
= kstat_create("zfs", 0, "fletcher_4_bench", "misc",
642 KSTAT_TYPE_RAW
, 0, KSTAT_FLAG_VIRTUAL
);
643 if (fletcher_4_kstat
!= NULL
) {
644 fletcher_4_kstat
->ks_data
= NULL
;
645 fletcher_4_kstat
->ks_ndata
= UINT32_MAX
;
646 kstat_set_raw_ops(fletcher_4_kstat
,
647 fletcher_4_kstat_headers
,
648 fletcher_4_kstat_data
,
649 fletcher_4_kstat_addr
);
650 kstat_install(fletcher_4_kstat
);
653 /* Finish initialization */
654 fletcher_4_initialized
= B_TRUE
;
658 fletcher_4_fini(void)
660 if (fletcher_4_kstat
!= NULL
) {
661 kstat_delete(fletcher_4_kstat
);
662 fletcher_4_kstat
= NULL
;
666 #if defined(_KERNEL) && defined(HAVE_SPL)
667 #include <linux/mod_compat.h>
670 fletcher_4_param_get(char *buffer
, zfs_kernel_param_t
*unused
)
672 const uint32_t impl
= IMPL_READ(fletcher_4_impl_chosen
);
677 fmt
= (impl
== IMPL_FASTEST
) ? "[%s] " : "%s ";
678 cnt
+= sprintf(buffer
+ cnt
, fmt
, "fastest");
680 /* list all supported implementations */
681 for (i
= 0; i
< fletcher_4_supp_impls_cnt
; i
++) {
682 fmt
= (i
== impl
) ? "[%s] " : "%s ";
683 cnt
+= sprintf(buffer
+ cnt
, fmt
,
684 fletcher_4_supp_impls
[i
]->name
);
691 fletcher_4_param_set(const char *val
, zfs_kernel_param_t
*unused
)
693 return (fletcher_4_impl_set(val
));
697 * Choose a fletcher 4 implementation in ZFS.
698 * Users can choose "cycle" to exercise all implementations, but this is
699 * for testing purpose therefore it can only be set in user space.
701 module_param_call(zfs_fletcher_4_impl
,
702 fletcher_4_param_set
, fletcher_4_param_get
, NULL
, 0644);
703 MODULE_PARM_DESC(zfs_fletcher_4_impl
, "Select fletcher 4 implementation.");
705 EXPORT_SYMBOL(fletcher_4_init
);
706 EXPORT_SYMBOL(fletcher_4_fini
);
707 EXPORT_SYMBOL(fletcher_2_native
);
708 EXPORT_SYMBOL(fletcher_2_byteswap
);
709 EXPORT_SYMBOL(fletcher_4_native
);
710 EXPORT_SYMBOL(fletcher_4_native_varsize
);
711 EXPORT_SYMBOL(fletcher_4_byteswap
);
712 EXPORT_SYMBOL(fletcher_4_incremental_native
);
713 EXPORT_SYMBOL(fletcher_4_incremental_byteswap
);