* bytes. For that reason, chaining values in the CV stack are represented as
* bytes.
*/
-static void output_chaining_value(const blake3_impl_ops_t *ops,
+static void output_chaining_value(const blake3_ops_t *ops,
const output_t *ctx, uint8_t cv[32])
{
uint32_t cv_words[8];
store_cv_words(cv, cv_words);
}
-static void output_root_bytes(const blake3_impl_ops_t *ops, const output_t *ctx,
+static void output_root_bytes(const blake3_ops_t *ops, const output_t *ctx,
uint64_t seek, uint8_t *out, size_t out_len)
{
uint64_t output_block_counter = seek / 64;
}
}
-static void chunk_state_update(const blake3_impl_ops_t *ops,
+static void chunk_state_update(const blake3_ops_t *ops,
blake3_chunk_state_t *ctx, const uint8_t *input, size_t input_len)
{
if (ctx->buf_len > 0) {
* number of chunks hashed. These chunks are never the root and never empty;
* those cases use a different codepath.
*/
-static size_t compress_chunks_parallel(const blake3_impl_ops_t *ops,
+static size_t compress_chunks_parallel(const blake3_ops_t *ops,
const uint8_t *input, size_t input_len, const uint32_t key[8],
uint64_t chunk_counter, uint8_t flags, uint8_t *out)
{
* return it as an additional output.) These parents are never the root and
* never empty; those cases use a different codepath.
*/
-static size_t compress_parents_parallel(const blake3_impl_ops_t *ops,
+static size_t compress_parents_parallel(const blake3_ops_t *ops,
const uint8_t *child_chaining_values, size_t num_chaining_values,
const uint32_t key[8], uint8_t flags, uint8_t *out)
{
* of implementing this special rule? Because we don't want to limit SIMD or
* multi-threading parallelism for that update().
*/
-static size_t blake3_compress_subtree_wide(const blake3_impl_ops_t *ops,
+static size_t blake3_compress_subtree_wide(const blake3_ops_t *ops,
const uint8_t *input, size_t input_len, const uint32_t key[8],
uint64_t chunk_counter, uint8_t flags, uint8_t *out)
{
* As with compress_subtree_wide(), this function is not used on inputs of 1
* chunk or less. That's a different codepath.
*/
-static void compress_subtree_to_parent_node(const blake3_impl_ops_t *ops,
+static void compress_subtree_to_parent_node(const blake3_ops_t *ops,
const uint8_t *input, size_t input_len, const uint32_t key[8],
uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN])
{
#include "blake3_impl.h"
-static const blake3_impl_ops_t *const blake3_impls[] = {
+static const blake3_ops_t *const blake3_impls[] = {
&blake3_generic_impl,
#if defined(__aarch64__) || \
(defined(__x86_64) && defined(HAVE_SSE2)) || \
#endif
};
-/* this pointer holds current ops for implementation */
-static const blake3_impl_ops_t *blake3_selected_impl = &blake3_generic_impl;
-
-/* special implementation selections */
+/* Select BLAKE3 implementation */
#define IMPL_FASTEST (UINT32_MAX)
-#define IMPL_CYCLE (UINT32_MAX-1)
-#define IMPL_USER (UINT32_MAX-2)
-#define IMPL_PARAM (UINT32_MAX-3)
+#define IMPL_CYCLE (UINT32_MAX - 1)
-#define IMPL_READ(i) (*(volatile uint32_t *) &(i))
-static uint32_t icp_blake3_impl = IMPL_FASTEST;
+#define IMPL_READ(i) (*(volatile uint32_t *) &(i))
-#define BLAKE3_IMPL_NAME_MAX 16
+/* Indicate that benchmark has been done */
+static boolean_t blake3_initialized = B_FALSE;
-/* id of fastest implementation */
-static uint32_t blake3_fastest_id = 0;
+/* Implementation that contains the fastest methods */
+static blake3_ops_t blake3_fastest_impl = {
+ .name = "fastest"
+};
-/* currently used id */
-static uint32_t blake3_current_id = 0;
+/* Hold all supported implementations */
+static const blake3_ops_t *blake3_supp_impls[ARRAY_SIZE(blake3_impls)];
+static uint32_t blake3_supp_impls_cnt = 0;
-/* id of module parameter (-1 == unused) */
-static int blake3_param_id = -1;
+/* Currently selected implementation */
+static uint32_t blake3_impl_chosen = IMPL_FASTEST;
-/* return number of supported implementations */
-int
-blake3_get_impl_count(void)
+static struct blake3_impl_selector {
+ const char *name;
+ uint32_t sel;
+} blake3_impl_selectors[] = {
+ { "cycle", IMPL_CYCLE },
+ { "fastest", IMPL_FASTEST }
+};
+
+/* check the supported implementations */
+static void blake3_impl_init(void)
{
- static int impls = 0;
- int i;
+ int i, c;
+
+ /* init only once */
+ if (likely(blake3_initialized))
+ return;
- if (impls)
- return (impls);
+ /* move supported implementations into blake3_supp_impls */
+ for (i = 0, c = 0; i < ARRAY_SIZE(blake3_impls); i++) {
+ const blake3_ops_t *impl = blake3_impls[i];
- for (i = 0; i < ARRAY_SIZE(blake3_impls); i++) {
- if (!blake3_impls[i]->is_supported()) continue;
- impls++;
+ if (impl->is_supported && impl->is_supported())
+ blake3_supp_impls[c++] = impl;
}
+ blake3_supp_impls_cnt = c;
- return (impls);
+ /* first init generic impl, may be changed via set_fastest() */
+ memcpy(&blake3_fastest_impl, blake3_impls[0],
+ sizeof (blake3_fastest_impl));
+ blake3_initialized = B_TRUE;
}
-/* return id of selected implementation */
-int
-blake3_get_impl_id(void)
+/* get number of supported implementations */
+uint32_t
+blake3_impl_getcnt(void)
+{
+ blake3_impl_init();
+ return (blake3_supp_impls_cnt);
+}
+
+/* get id of selected implementation */
+uint32_t
+blake3_impl_getid(void)
{
- return (blake3_current_id);
+ return (IMPL_READ(blake3_impl_chosen));
}
-/* return name of selected implementation */
+/* get name of selected implementation */
const char *
-blake3_get_impl_name(void)
+blake3_impl_getname(void)
{
- return (blake3_selected_impl->name);
+ uint32_t impl = IMPL_READ(blake3_impl_chosen);
+
+ blake3_impl_init();
+ switch (impl) {
+ case IMPL_FASTEST:
+ return ("fastest");
+ case IMPL_CYCLE:
+ return ("cycle");
+ default:
+ return (blake3_supp_impls[impl]->name);
+ }
}
/* setup id as fastest implementation */
void
-blake3_set_impl_fastest(uint32_t id)
+blake3_impl_set_fastest(uint32_t id)
{
- blake3_fastest_id = id;
+ /* setup fastest impl */
+ memcpy(&blake3_fastest_impl, blake3_supp_impls[id],
+ sizeof (blake3_fastest_impl));
}
/* set implementation by id */
void
-blake3_set_impl_id(uint32_t id)
+blake3_impl_setid(uint32_t id)
{
- int i, cid;
-
- /* select fastest */
- if (id == IMPL_FASTEST)
- id = blake3_fastest_id;
-
- /* select next or first */
- if (id == IMPL_CYCLE)
- id = (++blake3_current_id) % blake3_get_impl_count();
-
- /* 0..N for the real impl */
- for (i = 0, cid = 0; i < ARRAY_SIZE(blake3_impls); i++) {
- if (!blake3_impls[i]->is_supported()) continue;
- if (cid == id) {
- blake3_current_id = cid;
- blake3_selected_impl = blake3_impls[i];
- return;
- }
- cid++;
+ blake3_impl_init();
+ switch (id) {
+ case IMPL_FASTEST:
+ atomic_swap_32(&blake3_impl_chosen, IMPL_FASTEST);
+ break;
+ case IMPL_CYCLE:
+ atomic_swap_32(&blake3_impl_chosen, IMPL_CYCLE);
+ break;
+ default:
+ ASSERT3U(id, >=, 0);
+ ASSERT3U(id, <, blake3_supp_impls_cnt);
+ atomic_swap_32(&blake3_impl_chosen, id);
+ break;
}
}
/* set implementation by name */
int
-blake3_set_impl_name(const char *name)
+blake3_impl_setname(const char *val)
{
- int i, cid;
-
- if (strcmp(name, "fastest") == 0) {
- atomic_swap_32(&icp_blake3_impl, IMPL_FASTEST);
- blake3_set_impl_id(IMPL_FASTEST);
- return (0);
- } else if (strcmp(name, "cycle") == 0) {
- atomic_swap_32(&icp_blake3_impl, IMPL_CYCLE);
- blake3_set_impl_id(IMPL_CYCLE);
- return (0);
+ uint32_t impl = IMPL_READ(blake3_impl_chosen);
+ size_t val_len;
+ int i, err = -EINVAL;
+
+ blake3_impl_init();
+ val_len = strlen(val);
+ while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */
+ val_len--;
+
+ /* check mandatory implementations */
+ for (i = 0; i < ARRAY_SIZE(blake3_impl_selectors); i++) {
+ const char *name = blake3_impl_selectors[i].name;
+
+ if (val_len == strlen(name) &&
+ strncmp(val, name, val_len) == 0) {
+ impl = blake3_impl_selectors[i].sel;
+ err = 0;
+ break;
+ }
}
- for (i = 0, cid = 0; i < ARRAY_SIZE(blake3_impls); i++) {
- if (!blake3_impls[i]->is_supported()) continue;
- if (strcmp(name, blake3_impls[i]->name) == 0) {
- if (icp_blake3_impl == IMPL_PARAM) {
- blake3_param_id = cid;
- return (0);
+ if (err != 0 && blake3_initialized) {
+ /* check all supported implementations */
+ for (i = 0; i < blake3_supp_impls_cnt; i++) {
+ const char *name = blake3_supp_impls[i]->name;
+
+ if (val_len == strlen(name) &&
+ strncmp(val, name, val_len) == 0) {
+ impl = i;
+ err = 0;
+ break;
}
- blake3_selected_impl = blake3_impls[i];
- blake3_current_id = cid;
- return (0);
}
- cid++;
}
- return (-EINVAL);
+ if (err == 0) {
+ atomic_swap_32(&blake3_impl_chosen, impl);
+ }
+
+ return (err);
}
-/* setup implementation */
-void
-blake3_setup_impl(void)
+const blake3_ops_t *
+blake3_impl_get_ops(void)
{
- switch (IMPL_READ(icp_blake3_impl)) {
- case IMPL_PARAM:
- blake3_set_impl_id(blake3_param_id);
- atomic_swap_32(&icp_blake3_impl, IMPL_USER);
- break;
+ const blake3_ops_t *ops = NULL;
+ uint32_t impl = IMPL_READ(blake3_impl_chosen);
+
+ blake3_impl_init();
+ switch (impl) {
case IMPL_FASTEST:
- blake3_set_impl_id(IMPL_FASTEST);
+ ASSERT(blake3_initialized);
+ ops = &blake3_fastest_impl;
break;
case IMPL_CYCLE:
- blake3_set_impl_id(IMPL_CYCLE);
+ /* Cycle through supported implementations */
+ ASSERT(blake3_initialized);
+ ASSERT3U(blake3_supp_impls_cnt, >, 0);
+ static uint32_t cycle_count = 0;
+ uint32_t idx = (++cycle_count) % blake3_supp_impls_cnt;
+ ops = blake3_supp_impls[idx];
break;
default:
- blake3_set_impl_id(blake3_current_id);
+ ASSERT3U(blake3_supp_impls_cnt, >, 0);
+ ASSERT3U(impl, <, blake3_supp_impls_cnt);
+ ops = blake3_supp_impls[impl];
break;
}
-}
-/* return selected implementation */
-const blake3_impl_ops_t *
-blake3_impl_get_ops(void)
-{
- /* each call to ops will cycle */
- if (icp_blake3_impl == IMPL_CYCLE)
- blake3_set_impl_id(IMPL_CYCLE);
-
- return (blake3_selected_impl);
+ ASSERT3P(ops, !=, NULL);
+ return (ops);
}
#if defined(_KERNEL)
+
void **blake3_per_cpu_ctx;
void
blake3_per_cpu_ctx[i] = kmem_alloc(sizeof (BLAKE3_CTX),
KM_SLEEP);
}
+
+ /* init once in kernel mode */
+ blake3_impl_init();
}
void
memset(blake3_per_cpu_ctx, 0, max_ncpus * sizeof (void *));
kmem_free(blake3_per_cpu_ctx, max_ncpus * sizeof (void *));
}
-#endif
-#if defined(_KERNEL) && defined(__linux__)
+#define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
+
+#if defined(__linux__)
+
static int
-icp_blake3_impl_set(const char *name, zfs_kernel_param_t *kp)
+blake3_param_get(char *buffer, zfs_kernel_param_t *unused)
{
- char req_name[BLAKE3_IMPL_NAME_MAX];
- size_t i;
+ const uint32_t impl = IMPL_READ(blake3_impl_chosen);
+ char *fmt;
+ int cnt = 0;
- /* sanitize input */
- i = strnlen(name, BLAKE3_IMPL_NAME_MAX);
- if (i == 0 || i >= BLAKE3_IMPL_NAME_MAX)
- return (-EINVAL);
+ /* cycling */
+ fmt = IMPL_FMT(impl, IMPL_CYCLE);
+ cnt += sprintf(buffer + cnt, fmt, "cycle");
+
+ /* list fastest */
+ fmt = IMPL_FMT(impl, IMPL_FASTEST);
+ cnt += sprintf(buffer + cnt, fmt, "fastest");
+
+ /* list all supported implementations */
+ for (uint32_t i = 0; i < blake3_supp_impls_cnt; ++i) {
+ fmt = IMPL_FMT(impl, i);
+ cnt += sprintf(buffer + cnt, fmt,
+ blake3_supp_impls[i]->name);
+ }
- strlcpy(req_name, name, BLAKE3_IMPL_NAME_MAX);
- while (i > 0 && isspace(req_name[i-1]))
- i--;
- req_name[i] = '\0';
+ return (cnt);
+}
- atomic_swap_32(&icp_blake3_impl, IMPL_PARAM);
- return (blake3_set_impl_name(req_name));
+static int
+blake3_param_set(const char *val, zfs_kernel_param_t *unused)
+{
+ (void) unused;
+ return (blake3_impl_setname(val));
}
+#elif defined(__FreeBSD__)
+
+#include <sys/sbuf.h>
+
static int
-icp_blake3_impl_get(char *buffer, zfs_kernel_param_t *kp)
+blake3_param(ZFS_MODULE_PARAM_ARGS)
{
- int i, cid, cnt = 0;
- char *fmt;
+ int err;
- /* cycling */
- fmt = (icp_blake3_impl == IMPL_CYCLE) ? "[cycle] " : "cycle ";
- cnt += sprintf(buffer + cnt, fmt);
-
- /* fastest one */
- fmt = (icp_blake3_impl == IMPL_FASTEST) ? "[fastest] " : "fastest ";
- cnt += sprintf(buffer + cnt, fmt);
-
- /* user selected */
- for (i = 0, cid = 0; i < ARRAY_SIZE(blake3_impls); i++) {
- if (!blake3_impls[i]->is_supported()) continue;
- fmt = (icp_blake3_impl == IMPL_USER &&
- cid == blake3_current_id) ? "[%s] " : "%s ";
- cnt += sprintf(buffer + cnt, fmt, blake3_impls[i]->name);
- cid++;
+ if (req->newptr == NULL) {
+ const uint32_t impl = IMPL_READ(blake3_impl_chosen);
+ const int init_buflen = 64;
+ const char *fmt;
+ struct sbuf *s;
+
+ s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
+
+ /* cycling */
+ fmt = IMPL_FMT(impl, IMPL_CYCLE);
+ (void) sbuf_printf(s, fmt, "cycle");
+
+ /* list fastest */
+ fmt = IMPL_FMT(impl, IMPL_FASTEST);
+ (void) sbuf_printf(s, fmt, "fastest");
+
+ /* list all supported implementations */
+ for (uint32_t i = 0; i < blake3_supp_impls_cnt; ++i) {
+ fmt = IMPL_FMT(impl, i);
+ (void) sbuf_printf(s, fmt, blake3_supp_impls[i]->name);
+ }
+
+ err = sbuf_finish(s);
+ sbuf_delete(s);
+
+ return (err);
}
- buffer[cnt] = 0;
+ char buf[16];
- return (cnt);
+ err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
+ if (err) {
+ return (err);
+ }
+
+ return (-blake3_impl_setname(buf));
}
+#endif
+
+#undef IMPL_FMT
-module_param_call(icp_blake3_impl, icp_blake3_impl_set, icp_blake3_impl_get,
- NULL, 0644);
-MODULE_PARM_DESC(icp_blake3_impl, "Select BLAKE3 implementation.");
+ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, blake3_impl,
+ blake3_param_set, blake3_param_get, ZMOD_RW, \
+ "Select BLAKE3 implementation.");
#endif
projects / mirror_zfs.git / commitdiff