module/icp/algs/sha2/sha512_impl.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
  24  */
  25
  26 #include <sys/simd.h>
  27 #include <sys/zfs_context.h>
  28 #include <sys/zfs_impl.h>
  29 #include <sys/sha2.h>
  30
  31 #include <sha2/sha2_impl.h>
  32 #include <sys/asm_linkage.h>
  33
  34 #define TF(E, N) \
  35         extern void ASMABI E(uint64_t s[8], const void *, size_t); \
  36         static inline void N(uint64_t s[8], const void *d, size_t b) { \
  37         kfpu_begin(); E(s, d, b); kfpu_end(); \
  38 }
  39
  40 /* some implementation is always okay */
  41 static inline boolean_t sha2_is_supported(void)
  42 {
  43         return (B_TRUE);
  44 }
  45
  46 #if defined(__x86_64)
  47
  48 /* Users of ASMABI requires all calls to be from wrappers */
  49 extern void ASMABI
  50 zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
  51
  52 static inline void
  53 tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b)
  54 {
  55         zfs_sha512_transform_x64(s, d, b);
  56 }
  57 const sha512_ops_t sha512_x64_impl = {
  58         .is_supported = sha2_is_supported,
  59         .transform = tf_sha512_transform_x64,
  60         .name = "x64"
  61 };
  62
  63 #if defined(HAVE_AVX)
  64 static boolean_t sha2_have_avx(void)
  65 {
  66         return (kfpu_allowed() && zfs_avx_available());
  67 }
  68
  69 TF(zfs_sha512_transform_avx, tf_sha512_avx);
  70 const sha512_ops_t sha512_avx_impl = {
  71         .is_supported = sha2_have_avx,
  72         .transform = tf_sha512_avx,
  73         .name = "avx"
  74 };
  75 #endif
  76
  77 #if defined(HAVE_AVX2)
  78 static boolean_t sha2_have_avx2(void)
  79 {
  80         return (kfpu_allowed() && zfs_avx2_available());
  81 }
  82
  83 TF(zfs_sha512_transform_avx2, tf_sha512_avx2);
  84 const sha512_ops_t sha512_avx2_impl = {
  85         .is_supported = sha2_have_avx2,
  86         .transform = tf_sha512_avx2,
  87         .name = "avx2"
  88 };
  89 #endif
  90
  91 #elif defined(__aarch64__)
  92 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
  93 const sha512_ops_t sha512_armv7_impl = {
  94         .is_supported = sha2_is_supported,
  95         .transform = zfs_sha512_block_armv7,
  96         .name = "armv7"
  97 };
  98
  99 static boolean_t sha512_have_armv8ce(void)
 100 {
 101         return (kfpu_allowed() && zfs_sha512_available());
 102 }
 103
 104 TF(zfs_sha512_block_armv8, tf_sha512_armv8ce);
 105 const sha512_ops_t sha512_armv8_impl = {
 106         .is_supported = sha512_have_armv8ce,
 107         .transform = tf_sha512_armv8ce,
 108         .name = "armv8-ce"
 109 };
 110
 111 #elif defined(__arm__) && __ARM_ARCH > 6
 112 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
 113 const sha512_ops_t sha512_armv7_impl = {
 114         .is_supported = sha2_is_supported,
 115         .transform = zfs_sha512_block_armv7,
 116         .name = "armv7"
 117 };
 118
 119 static boolean_t sha512_have_neon(void)
 120 {
 121         return (kfpu_allowed() && zfs_neon_available());
 122 }
 123
 124 TF(zfs_sha512_block_neon, tf_sha512_neon);
 125 const sha512_ops_t sha512_neon_impl = {
 126         .is_supported = sha512_have_neon,
 127         .transform = tf_sha512_neon,
 128         .name = "neon"
 129 };
 130
 131 #elif defined(__PPC64__)
 132 TF(zfs_sha512_ppc, tf_sha512_ppc);
 133 const sha512_ops_t sha512_ppc_impl = {
 134         .is_supported = sha2_is_supported,
 135         .transform = tf_sha512_ppc,
 136         .name = "ppc"
 137 };
 138
 139 static boolean_t sha512_have_isa207(void)
 140 {
 141         return (kfpu_allowed() && zfs_isa207_available());
 142 }
 143
 144 TF(zfs_sha512_power8, tf_sha512_power8);
 145 const sha512_ops_t sha512_power8_impl = {
 146         .is_supported = sha512_have_isa207,
 147         .transform = tf_sha512_power8,
 148         .name = "power8"
 149 };
 150 #endif /* __PPC64__ */
 151
 152 /* the two generic ones */
 153 extern const sha512_ops_t sha512_generic_impl;
 154
 155 /* array with all sha512 implementations */
 156 static const sha512_ops_t *const sha512_impls[] = {
 157         &sha512_generic_impl,
 158 #if defined(__x86_64)
 159         &sha512_x64_impl,
 160 #endif
 161 #if defined(__x86_64) && defined(HAVE_AVX)
 162         &sha512_avx_impl,
 163 #endif
 164 #if defined(__x86_64) && defined(HAVE_AVX2)
 165         &sha512_avx2_impl,
 166 #endif
 167 #if defined(__aarch64__)
 168         &sha512_armv7_impl,
 169         &sha512_armv8_impl,
 170 #endif
 171 #if defined(__arm__) && __ARM_ARCH > 6
 172         &sha512_armv7_impl,
 173         &sha512_neon_impl,
 174 #endif
 175 #if defined(__PPC64__)
 176         &sha512_ppc_impl,
 177         &sha512_power8_impl,
 178 #endif /* __PPC64__ */
 179 };
 180
 181 /* use the generic implementation functions */
 182 #define IMPL_NAME               "sha512"
 183 #define IMPL_OPS_T              sha512_ops_t
 184 #define IMPL_ARRAY              sha512_impls
 185 #define IMPL_GET_OPS            sha512_get_ops
 186 #define ZFS_IMPL_OPS            zfs_sha512_ops
 187 #include <generic_impl.c>
 188
 189 #ifdef _KERNEL
 190
 191 #define IMPL_FMT(impl, i)       (((impl) == (i)) ? "[%s] " : "%s ")
 192
 193 #if defined(__linux__)
 194
 195 static int
 196 sha512_param_get(char *buffer, zfs_kernel_param_t *unused)
 197 {
 198         const uint32_t impl = IMPL_READ(generic_impl_chosen);
 199         char *fmt;
 200         int cnt = 0;
 201
 202         /* cycling */
 203         fmt = IMPL_FMT(impl, IMPL_CYCLE);
 204         cnt += sprintf(buffer + cnt, fmt, "cycle");
 205
 206         /* list fastest */
 207         fmt = IMPL_FMT(impl, IMPL_FASTEST);
 208         cnt += sprintf(buffer + cnt, fmt, "fastest");
 209
 210         /* list all supported implementations */
 211         generic_impl_init();
 212         for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
 213                 fmt = IMPL_FMT(impl, i);
 214                 cnt += sprintf(buffer + cnt, fmt,
 215                     generic_supp_impls[i]->name);
 216         }
 217
 218         return (cnt);
 219 }
 220
 221 static int
 222 sha512_param_set(const char *val, zfs_kernel_param_t *unused)
 223 {
 224         (void) unused;
 225         return (generic_impl_setname(val));
 226 }
 227
 228 #elif defined(__FreeBSD__)
 229
 230 #include <sys/sbuf.h>
 231
 232 static int
 233 sha512_param(ZFS_MODULE_PARAM_ARGS)
 234 {
 235         int err;
 236
 237         generic_impl_init();
 238         if (req->newptr == NULL) {
 239                 const uint32_t impl = IMPL_READ(generic_impl_chosen);
 240                 const int init_buflen = 64;
 241                 const char *fmt;
 242                 struct sbuf *s;
 243
 244                 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
 245
 246                 /* cycling */
 247                 fmt = IMPL_FMT(impl, IMPL_CYCLE);
 248                 (void) sbuf_printf(s, fmt, "cycle");
 249
 250                 /* list fastest */
 251                 fmt = IMPL_FMT(impl, IMPL_FASTEST);
 252                 (void) sbuf_printf(s, fmt, "fastest");
 253
 254                 /* list all supported implementations */
 255                 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
 256                         fmt = IMPL_FMT(impl, i);
 257                         (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
 258                 }
 259
 260                 err = sbuf_finish(s);
 261                 sbuf_delete(s);
 262
 263                 return (err);
 264         }
 265
 266         /* we got module parameter */
 267         char buf[16];
 268
 269         err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
 270         if (err) {
 271                 return (err);
 272         }
 273
 274         return (-generic_impl_setname(buf));
 275 }
 276 #endif
 277
 278 #undef IMPL_FMT
 279
 280 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl,
 281     sha512_param_set, sha512_param_get, ZMOD_RW, \
 282         "Select SHA512 implementation.");
 283 #endif
 284
 285 #undef TF