*.swp
.deps
.libs
+.dirstamp
.DS_Store
modules.order
Makefile
--- /dev/null
+dnl #
+dnl # Set the target arch for libspl atomic implementation and the icp
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_ARCH], [
+ AC_MSG_CHECKING(for target asm dir)
+ TARGET_ARCH=`echo ${target_cpu} | sed -e s/i.86/i386/`
+
+ case $TARGET_ARCH in
+ i386|x86_64)
+ TARGET_ASM_DIR=asm-${TARGET_ARCH}
+ ;;
+ *)
+ TARGET_ASM_DIR=asm-generic
+ ;;
+ esac
+
+ AC_SUBST([TARGET_ASM_DIR])
+ AM_CONDITIONAL([TARGET_ASM_X86_64], test $TARGET_ASM_DIR = asm-x86_64)
+ AM_CONDITIONAL([TARGET_ASM_I386], test $TARGET_ASM_DIR = asm-i386)
+ AM_CONDITIONAL([TARGET_ASM_GENERIC], test $TARGET_ASM_DIR = asm-generic)
+ AC_MSG_RESULT([$TARGET_ASM_DIR])
+])
+++ /dev/null
-dnl #
-dnl # Set the target arch for libspl atomic implementation
-dnl #
-AC_DEFUN([ZFS_AC_CONFIG_USER_ARCH], [
- AC_MSG_CHECKING(for target asm dir)
- TARGET_ARCH=`echo ${target_cpu} | sed -e s/i.86/i386/`
-
- case $TARGET_ARCH in
- i386|x86_64)
- TARGET_ASM_DIR=asm-${TARGET_ARCH}
- ;;
- *)
- TARGET_ASM_DIR=asm-generic
- ;;
- esac
-
- AC_SUBST([TARGET_ASM_DIR])
- AC_MSG_RESULT([$TARGET_ASM_DIR])
-])
ZFS_AC_CONFIG_USER_SYSTEMD
ZFS_AC_CONFIG_USER_SYSVINIT
ZFS_AC_CONFIG_USER_DRACUT
- ZFS_AC_CONFIG_USER_ARCH
ZFS_AC_CONFIG_USER_ZLIB
ZFS_AC_CONFIG_USER_LIBUUID
ZFS_AC_CONFIG_USER_LIBTIRPC
ZFS_AC_CONFIG_ALWAYS_NO_UNUSED_BUT_SET_VARIABLE
ZFS_AC_CONFIG_ALWAYS_NO_BOOL_COMPARE
ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD
+ ZFS_AC_CONFIG_ALWAYS_ARCH
])
AC_DEFUN([ZFS_AC_CONFIG], [
- TARGET_ASM_DIR=asm-generic
- AC_SUBST(TARGET_ASM_DIR)
-
ZFS_CONFIG=all
AC_ARG_WITH([config],
AS_HELP_STRING([--with-config=CONFIG],
AC_CANONICAL_SYSTEM
AM_MAINTAINER_MODE
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
-AM_INIT_AUTOMAKE
+AM_INIT_AUTOMAKE([subdir-objects])
AC_CONFIG_HEADERS([zfs_config.h], [
(mv zfs_config.h zfs_config.h.tmp &&
awk -f ${ac_srcdir}/config/config.awk zfs_config.h.tmp >zfs_config.h &&
lib/libspl/include/util/Makefile
lib/libavl/Makefile
lib/libefi/Makefile
+ lib/libicp/Makefile
lib/libnvpair/Makefile
lib/libunicode/Makefile
lib/libuutil/Makefile
module/zcommon/Makefile
module/zfs/Makefile
module/zpios/Makefile
+ module/icp/Makefile
include/Makefile
include/linux/Makefile
include/sys/Makefile
include/sys/fs/Makefile
include/sys/fm/Makefile
include/sys/fm/fs/Makefile
+ include/sys/crypto/Makefile
scripts/Makefile
scripts/zpios-profile/Makefile
scripts/zpios-test/Makefile
cp --recursive module "$KERNEL_DIR/fs/zfs"
cp zfs_config.h "$KERNEL_DIR/"
-adjust_obj_paths()
-{
- local FILE="$1"
- local LINE OBJPATH
-
- while IFS='' read -r LINE
- do
- OBJPATH="${LINE#\$(MODULE)-objs += }"
- if [ "$OBJPATH" = "$LINE" ]
- then
- echo "$LINE"
- else
- echo "\$(MODULE)-objs += ${OBJPATH##*/}"
- fi
- done < "$FILE" > "$FILE.new"
- mv "$FILE.new" "$FILE"
-}
-
for MODULE in "${MODULES[@]}"
do
- adjust_obj_paths "$KERNEL_DIR/fs/zfs/$MODULE/Makefile"
sed -i.bak '/obj =/d' "$KERNEL_DIR/fs/zfs/$MODULE/Makefile"
sed -i.bak '/src =/d' "$KERNEL_DIR/fs/zfs/$MODULE/Makefile"
done
-SUBDIRS = fm fs
+SUBDIRS = fm fs crypto
COMMON_H = \
$(top_srcdir)/include/sys/arc.h \
--- /dev/null
+COMMON_H = \
+ $(top_srcdir)/include/sys/crypto/api.h \
+ $(top_srcdir)/include/sys/crypto/common.h \
+ $(top_srcdir)/include/sys/crypto/icp.h
+
+KERNEL_H =
+
+USER_H =
+
+EXTRA_DIST = $(COMMON_H) $(KERNEL_H) $(USER_H)
+
+if CONFIG_USER
+libzfsdir = $(includedir)/libzfs/sys/crypto
+libzfs_HEADERS = $(COMMON_H) $(USER_H)
+endif
+
+if CONFIG_KERNEL
+kerneldir = @prefix@/src/zfs-$(VERSION)/include/sys/crypto
+kernel_HEADERS = $(COMMON_H) $(KERNEL_H)
+endif
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_CRYPTO_API_H
+#define _SYS_CRYPTO_API_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+
+typedef long crypto_req_id_t;
+typedef void *crypto_bc_t;
+typedef void *crypto_context_t;
+typedef void *crypto_ctx_template_t;
+
+typedef uint32_t crypto_call_flag_t;
+
+/* crypto_call_flag's values */
+#define CRYPTO_ALWAYS_QUEUE 0x00000001 /* ALWAYS queue the req. */
+#define CRYPTO_NOTIFY_OPDONE 0x00000002 /* Notify intermediate steps */
+#define CRYPTO_SKIP_REQID 0x00000004 /* Skip request ID generation */
+#define CRYPTO_RESTRICTED 0x00000008 /* cannot use restricted prov */
+
+typedef struct {
+ crypto_call_flag_t cr_flag;
+ void (*cr_callback_func)(void *, int);
+ void *cr_callback_arg;
+ crypto_req_id_t cr_reqid;
+} crypto_call_req_t;
+
+/*
+ * Returns the mechanism type corresponding to a mechanism name.
+ */
+
+#define CRYPTO_MECH_INVALID ((uint64_t)-1)
+extern crypto_mech_type_t crypto_mech2id(crypto_mech_name_t name);
+
+/*
+ * Create and destroy context templates.
+ */
+extern int crypto_create_ctx_template(crypto_mechanism_t *mech,
+ crypto_key_t *key, crypto_ctx_template_t *tmpl, int kmflag);
+extern void crypto_destroy_ctx_template(crypto_ctx_template_t tmpl);
+
+/*
+ * Single and multi-part digest operations.
+ */
+extern int crypto_digest(crypto_mechanism_t *mech, crypto_data_t *data,
+ crypto_data_t *digest, crypto_call_req_t *cr);
+extern int crypto_digest_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_data_t *, crypto_data_t *,
+ crypto_call_req_t *);
+extern int crypto_digest_init(crypto_mechanism_t *mech, crypto_context_t *ctxp,
+ crypto_call_req_t *cr);
+extern int crypto_digest_init_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_context_t *, crypto_call_req_t *);
+extern int crypto_digest_update(crypto_context_t ctx, crypto_data_t *data,
+ crypto_call_req_t *cr);
+extern int crypto_digest_final(crypto_context_t ctx, crypto_data_t *digest,
+ crypto_call_req_t *cr);
+
+/*
+ * Single and multi-part MAC operations.
+ */
+extern int crypto_mac(crypto_mechanism_t *mech, crypto_data_t *data,
+ crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *mac,
+ crypto_call_req_t *cr);
+extern int crypto_mac_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_data_t *, crypto_key_t *,
+ crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *);
+extern int crypto_mac_verify(crypto_mechanism_t *mech, crypto_data_t *data,
+ crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *mac,
+ crypto_call_req_t *cr);
+extern int crypto_mac_verify_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_data_t *, crypto_key_t *,
+ crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *);
+extern int crypto_mac_init(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr);
+extern int crypto_mac_init_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t,
+ crypto_context_t *, crypto_call_req_t *);
+extern int crypto_mac_update(crypto_context_t ctx, crypto_data_t *data,
+ crypto_call_req_t *cr);
+extern int crypto_mac_final(crypto_context_t ctx, crypto_data_t *data,
+ crypto_call_req_t *cr);
+
+/*
+ * Single and multi-part sign with private key operations.
+ */
+extern int crypto_sign(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_data_t *data, crypto_ctx_template_t tmpl,
+ crypto_data_t *signature, crypto_call_req_t *cr);
+extern int crypto_sign_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *);
+extern int crypto_sign_init(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr);
+extern int crypto_sign_init_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t,
+ crypto_context_t *, crypto_call_req_t *);
+extern int crypto_sign_update(crypto_context_t ctx, crypto_data_t *data,
+ crypto_call_req_t *cr);
+extern int crypto_sign_final(crypto_context_t ctx, crypto_data_t *signature,
+ crypto_call_req_t *cr);
+extern int crypto_sign_recover_init_prov(crypto_provider_t,
+ crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
+ crypto_ctx_template_t tmpl, crypto_context_t *, crypto_call_req_t *);
+extern int crypto_sign_recover(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_data_t *data, crypto_ctx_template_t tmpl, crypto_data_t *signature,
+ crypto_call_req_t *cr);
+extern int crypto_sign_recover_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *);
+
+/*
+ * Single and multi-part verify with public key operations.
+ */
+extern int crypto_verify(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_data_t *data, crypto_ctx_template_t tmpl, crypto_data_t *signature,
+ crypto_call_req_t *cr);
+extern int crypto_verify_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *);
+extern int crypto_verify_init(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr);
+extern int crypto_verify_init_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t,
+ crypto_context_t *, crypto_call_req_t *);
+extern int crypto_verify_update(crypto_context_t ctx, crypto_data_t *data,
+ crypto_call_req_t *cr);
+extern int crypto_verify_final(crypto_context_t ctx, crypto_data_t *signature,
+ crypto_call_req_t *cr);
+extern int crypto_verify_recover_init_prov(crypto_provider_t,
+ crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
+ crypto_ctx_template_t tmpl, crypto_context_t *, crypto_call_req_t *);
+extern int crypto_verify_recover(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_data_t *signature, crypto_ctx_template_t tmpl, crypto_data_t *data,
+ crypto_call_req_t *cr);
+extern int crypto_verify_recover_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *);
+
+/*
+ * Single and multi-part encryption operations.
+ */
+extern int crypto_encrypt(crypto_mechanism_t *mech, crypto_data_t *plaintext,
+ crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *ciphertext,
+ crypto_call_req_t *cr);
+extern int crypto_encrypt_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_data_t *, crypto_key_t *,
+ crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *);
+extern int crypto_encrypt_init(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr);
+extern int crypto_encrypt_init_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t,
+ crypto_context_t *, crypto_call_req_t *);
+extern int crypto_encrypt_update(crypto_context_t ctx,
+ crypto_data_t *plaintext, crypto_data_t *ciphertext,
+ crypto_call_req_t *cr);
+extern int crypto_encrypt_final(crypto_context_t ctx,
+ crypto_data_t *ciphertext, crypto_call_req_t *cr);
+
+/*
+ * Single and multi-part decryption operations.
+ */
+extern int crypto_decrypt(crypto_mechanism_t *mech, crypto_data_t *ciphertext,
+ crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *plaintext,
+ crypto_call_req_t *cr);
+extern int crypto_decrypt_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_data_t *, crypto_key_t *,
+ crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *);
+extern int crypto_decrypt_init(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *cr);
+extern int crypto_decrypt_init_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t,
+ crypto_context_t *, crypto_call_req_t *);
+extern int crypto_decrypt_update(crypto_context_t ctx,
+ crypto_data_t *ciphertext, crypto_data_t *plaintext,
+ crypto_call_req_t *cr);
+extern int crypto_decrypt_final(crypto_context_t ctx, crypto_data_t *plaintext,
+ crypto_call_req_t *cr);
+
+/*
+ * Single and multi-part encrypt/MAC dual operations.
+ */
+extern int crypto_encrypt_mac(crypto_mechanism_t *encr_mech,
+ crypto_mechanism_t *mac_mech, crypto_data_t *pt,
+ crypto_key_t *encr_key, crypto_key_t *mac_key,
+ crypto_ctx_template_t encr_tmpl, crypto_ctx_template_t mac_tmpl,
+ crypto_dual_data_t *ct, crypto_data_t *mac, crypto_call_req_t *cr);
+extern int crypto_encrypt_mac_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_mechanism_t *, crypto_data_t *,
+ crypto_key_t *, crypto_key_t *, crypto_ctx_template_t,
+ crypto_ctx_template_t, crypto_dual_data_t *, crypto_data_t *,
+ crypto_call_req_t *);
+extern int crypto_encrypt_mac_init(crypto_mechanism_t *encr_mech,
+ crypto_mechanism_t *mac_mech, crypto_key_t *encr_key,
+ crypto_key_t *mac_key, crypto_ctx_template_t encr_tmpl,
+ crypto_ctx_template_t mac_tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *cr);
+extern int crypto_encrypt_mac_init_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_mechanism_t *, crypto_key_t *, crypto_key_t *,
+ crypto_ctx_template_t, crypto_ctx_template_t, crypto_context_t *,
+ crypto_call_req_t *);
+extern int crypto_encrypt_mac_update(crypto_context_t ctx,
+ crypto_data_t *pt, crypto_dual_data_t *ct, crypto_call_req_t *cr);
+extern int crypto_encrypt_mac_final(crypto_context_t ctx,
+ crypto_dual_data_t *ct, crypto_data_t *mac, crypto_call_req_t *cr);
+
+/*
+ * Single and multi-part MAC/decrypt dual operations.
+ */
+extern int crypto_mac_decrypt(crypto_mechanism_t *mac_mech,
+ crypto_mechanism_t *decr_mech, crypto_dual_data_t *ct,
+ crypto_key_t *mac_key, crypto_key_t *decr_key,
+ crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl,
+ crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr);
+extern int crypto_mac_decrypt_prov(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *mac_mech, crypto_mechanism_t *decr_mech,
+ crypto_dual_data_t *ct, crypto_key_t *mac_key, crypto_key_t *decr_key,
+ crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl,
+ crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr);
+extern int crypto_mac_verify_decrypt(crypto_mechanism_t *mac_mech,
+ crypto_mechanism_t *decr_mech, crypto_dual_data_t *ct,
+ crypto_key_t *mac_key, crypto_key_t *decr_key,
+ crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl,
+ crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr);
+extern int crypto_mac_verify_decrypt_prov(crypto_provider_t,
+ crypto_session_id_t, crypto_mechanism_t *mac_mech,
+ crypto_mechanism_t *decr_mech, crypto_dual_data_t *ct,
+ crypto_key_t *mac_key, crypto_key_t *decr_key,
+ crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl,
+ crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr);
+extern int crypto_mac_decrypt_init(crypto_mechanism_t *mac_mech,
+ crypto_mechanism_t *decr_mech, crypto_key_t *mac_key,
+ crypto_key_t *decr_key, crypto_ctx_template_t mac_tmpl,
+ crypto_ctx_template_t decr_tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *cr);
+extern int crypto_mac_decrypt_init_prov(crypto_provider_t,
+ crypto_session_id_t, crypto_mechanism_t *mac_mech,
+ crypto_mechanism_t *decr_mech, crypto_key_t *mac_key,
+ crypto_key_t *decr_key, crypto_ctx_template_t mac_tmpl,
+ crypto_ctx_template_t decr_tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *cr);
+extern int crypto_mac_decrypt_update(crypto_context_t ctx,
+ crypto_dual_data_t *ct, crypto_data_t *pt, crypto_call_req_t *cr);
+extern int crypto_mac_decrypt_final(crypto_context_t ctx, crypto_data_t *mac,
+ crypto_data_t *pt, crypto_call_req_t *cr);
+
+/* Session Management */
+extern int crypto_session_open(crypto_provider_t, crypto_session_id_t *,
+ crypto_call_req_t *);
+extern int crypto_session_close(crypto_provider_t, crypto_session_id_t,
+ crypto_call_req_t *);
+extern int crypto_session_login(crypto_provider_t, crypto_session_id_t,
+ crypto_user_type_t, char *, size_t, crypto_call_req_t *);
+extern int crypto_session_logout(crypto_provider_t, crypto_session_id_t,
+ crypto_call_req_t *);
+
+/* Object Management */
+extern int crypto_object_copy(crypto_provider_t, crypto_session_id_t,
+ crypto_object_id_t, crypto_object_attribute_t *, uint_t,
+ crypto_object_id_t *, crypto_call_req_t *);
+extern int crypto_object_create(crypto_provider_t, crypto_session_id_t,
+ crypto_object_attribute_t *, uint_t, crypto_object_id_t *,
+ crypto_call_req_t *);
+extern int crypto_object_destroy(crypto_provider_t, crypto_session_id_t,
+ crypto_object_id_t, crypto_call_req_t *);
+extern int crypto_object_get_attribute_value(crypto_provider_t,
+ crypto_session_id_t, crypto_object_id_t, crypto_object_attribute_t *,
+ uint_t, crypto_call_req_t *);
+extern int crypto_object_get_size(crypto_provider_t, crypto_session_id_t,
+ crypto_object_id_t, size_t *, crypto_call_req_t *);
+extern int crypto_object_find_final(crypto_provider_t, void *,
+ crypto_call_req_t *);
+extern int crypto_object_find_init(crypto_provider_t, crypto_session_id_t,
+ crypto_object_attribute_t *, uint_t, void **, crypto_call_req_t *);
+extern int crypto_object_find(crypto_provider_t, void *, crypto_object_id_t *,
+ uint_t *, uint_t, crypto_call_req_t *);
+extern int crypto_object_set_attribute_value(crypto_provider_t,
+ crypto_session_id_t, crypto_object_id_t, crypto_object_attribute_t *,
+ uint_t, crypto_call_req_t *);
+
+/* Key Management */
+extern int crypto_key_derive(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_object_attribute_t *,
+ uint_t, crypto_object_id_t *, crypto_call_req_t *);
+extern int crypto_key_generate(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_object_attribute_t *, uint_t,
+ crypto_object_id_t *, crypto_call_req_t *);
+extern int crypto_key_generate_pair(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_object_attribute_t *, uint_t,
+ crypto_object_attribute_t *, uint_t, crypto_object_id_t *,
+ crypto_object_id_t *, crypto_call_req_t *);
+extern int crypto_key_unwrap(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, uchar_t *, size_t *,
+ crypto_object_attribute_t *, uint_t, crypto_object_id_t *,
+ crypto_call_req_t *);
+extern int crypto_key_wrap(crypto_provider_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_object_id_t *, uchar_t *,
+ size_t *, crypto_call_req_t *);
+extern int crypto_key_check_prov(crypto_provider_t, crypto_mechanism_t *mech,
+ crypto_key_t *key);
+extern int crypto_key_check(crypto_mechanism_t *mech, crypto_key_t *key);
+
+
+/*
+ * Routines to cancel a single asynchronous request or all asynchronous
+ * requests associated with a particular context.
+ */
+extern void crypto_cancel_req(crypto_req_id_t req);
+extern void crypto_cancel_ctx(crypto_context_t ctx);
+
+/*
+ * crypto_get_mech_list(9F) allocates and returns the list of currently
+ * supported cryptographic mechanisms.
+ */
+extern crypto_mech_name_t *crypto_get_mech_list(uint_t *count, int kmflag);
+extern void crypto_free_mech_list(crypto_mech_name_t *mech_names,
+ uint_t count);
+
+extern crypto_provider_t crypto_get_provider(char *, char *, char *);
+extern int crypto_get_provinfo(crypto_provider_t, crypto_provider_ext_info_t *);
+extern void crypto_release_provider(crypto_provider_t);
+
+/*
+ * A kernel consumer can request to be notified when some particular event
+ * occurs. The valid events, callback function type, and functions to
+ * be called to register or unregister for notification are defined below.
+ */
+
+#define CRYPTO_EVENT_MECHS_CHANGED 0x00000001
+#define CRYPTO_EVENT_PROVIDER_REGISTERED 0x00000002
+#define CRYPTO_EVENT_PROVIDER_UNREGISTERED 0x00000004
+
+typedef enum {
+ CRYPTO_MECH_ADDED = 1,
+ CRYPTO_MECH_REMOVED
+} crypto_event_change_t;
+
+/* The event_arg argument structure for CRYPTO_EVENT_PROVIDERS_CHANGE event */
+typedef struct crypto_notify_event_change {
+ crypto_mech_name_t ec_mech_name;
+ crypto_provider_type_t ec_provider_type;
+ crypto_event_change_t ec_change;
+} crypto_notify_event_change_t;
+
+typedef void *crypto_notify_handle_t;
+typedef void (*crypto_notify_callback_t)(uint32_t event_mask, void *event_arg);
+
+extern crypto_notify_handle_t crypto_notify_events(
+ crypto_notify_callback_t nf, uint32_t event_mask);
+extern void crypto_unnotify_events(crypto_notify_handle_t);
+
+/*
+ * crypto_bufcall(9F) group of routines.
+ */
+extern crypto_bc_t crypto_bufcall_alloc(void);
+extern int crypto_bufcall_free(crypto_bc_t bc);
+extern int crypto_bufcall(crypto_bc_t bc, void (*func)(void *arg), void *arg);
+extern int crypto_unbufcall(crypto_bc_t bc);
+
+/*
+ * To obtain the list of key size ranges supported by a mechanism.
+ */
+
+#define CRYPTO_MECH_USAGE_ENCRYPT 0x00000001
+#define CRYPTO_MECH_USAGE_DECRYPT 0x00000002
+#define CRYPTO_MECH_USAGE_MAC 0x00000004
+
+typedef uint32_t crypto_mech_usage_t;
+
+typedef struct crypto_mechanism_info {
+ size_t mi_min_key_size;
+ size_t mi_max_key_size;
+ crypto_keysize_unit_t mi_keysize_unit; /* for mi_xxx_key_size */
+ crypto_mech_usage_t mi_usage;
+} crypto_mechanism_info_t;
+
+#ifdef _SYSCALL32
+
+typedef struct crypto_mechanism_info32 {
+ size32_t mi_min_key_size;
+ size32_t mi_max_key_size;
+ crypto_keysize_unit_t mi_keysize_unit; /* for mi_xxx_key_size */
+ crypto_mech_usage_t mi_usage;
+} crypto_mechanism_info32_t;
+
+#endif /* _SYSCALL32 */
+
+extern int crypto_get_all_mech_info(crypto_mech_type_t,
+ crypto_mechanism_info_t **, uint_t *, int);
+extern void crypto_free_all_mech_info(crypto_mechanism_info_t *, uint_t);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_CRYPTO_API_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+/*
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
+ */
+
+#ifndef _SYS_CRYPTO_COMMON_H
+#define _SYS_CRYPTO_COMMON_H
+
+/*
+ * Header file for the common data structures of the cryptographic framework
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+
+/* Cryptographic Mechanisms */
+
+#define CRYPTO_MAX_MECH_NAME 32
+typedef char crypto_mech_name_t[CRYPTO_MAX_MECH_NAME];
+
+typedef uint64_t crypto_mech_type_t;
+
+typedef struct crypto_mechanism {
+ crypto_mech_type_t cm_type; /* mechanism type */
+ caddr_t cm_param; /* mech. parameter */
+ size_t cm_param_len; /* mech. parameter len */
+} crypto_mechanism_t;
+
+#ifdef _SYSCALL32
+
+typedef struct crypto_mechanism32 {
+ crypto_mech_type_t cm_type; /* mechanism type */
+ caddr32_t cm_param; /* mech. parameter */
+ size32_t cm_param_len; /* mech. parameter len */
+} crypto_mechanism32_t;
+
+#endif /* _SYSCALL32 */
+
+/* CK_AES_CTR_PARAMS provides parameters to the CKM_AES_CTR mechanism */
+typedef struct CK_AES_CTR_PARAMS {
+ ulong_t ulCounterBits;
+ uint8_t cb[16];
+} CK_AES_CTR_PARAMS;
+
+/* CK_AES_CCM_PARAMS provides parameters to the CKM_AES_CCM mechanism */
+typedef struct CK_AES_CCM_PARAMS {
+ ulong_t ulMACSize;
+ ulong_t ulNonceSize;
+ ulong_t ulAuthDataSize;
+ ulong_t ulDataSize; /* used for plaintext or ciphertext */
+ uchar_t *nonce;
+ uchar_t *authData;
+} CK_AES_CCM_PARAMS;
+
+/* CK_AES_GCM_PARAMS provides parameters to the CKM_AES_GCM mechanism */
+typedef struct CK_AES_GCM_PARAMS {
+ uchar_t *pIv;
+ ulong_t ulIvLen;
+ ulong_t ulIvBits;
+ uchar_t *pAAD;
+ ulong_t ulAADLen;
+ ulong_t ulTagBits;
+} CK_AES_GCM_PARAMS;
+
+/* CK_AES_GMAC_PARAMS provides parameters to the CKM_AES_GMAC mechanism */
+typedef struct CK_AES_GMAC_PARAMS {
+ uchar_t *pIv;
+ uchar_t *pAAD;
+ ulong_t ulAADLen;
+} CK_AES_GMAC_PARAMS;
+
+/*
+ * CK_ECDH1_DERIVE_PARAMS provides the parameters to the
+ * CKM_ECDH1_KEY_DERIVE mechanism
+ */
+typedef struct CK_ECDH1_DERIVE_PARAMS {
+ ulong_t kdf;
+ ulong_t ulSharedDataLen;
+ uchar_t *pSharedData;
+ ulong_t ulPublicDataLen;
+ uchar_t *pPublicData;
+} CK_ECDH1_DERIVE_PARAMS;
+
+#ifdef _SYSCALL32
+
+/* needed for 32-bit applications running on 64-bit kernels */
+typedef struct CK_AES_CTR_PARAMS32 {
+ uint32_t ulCounterBits;
+ uint8_t cb[16];
+} CK_AES_CTR_PARAMS32;
+
+/* needed for 32-bit applications running on 64-bit kernels */
+typedef struct CK_AES_CCM_PARAMS32 {
+ uint32_t ulMACSize;
+ uint32_t ulNonceSize;
+ uint32_t ulAuthDataSize;
+ uint32_t ulDataSize;
+ caddr32_t nonce;
+ caddr32_t authData;
+} CK_AES_CCM_PARAMS32;
+
+/* needed for 32-bit applications running on 64-bit kernels */
+typedef struct CK_AES_GCM_PARAMS32 {
+ caddr32_t pIv;
+ uint32_t ulIvLen;
+ uint32_t ulIvBits;
+ caddr32_t pAAD;
+ uint32_t ulAADLen;
+ uint32_t ulTagBits;
+} CK_AES_GCM_PARAMS32;
+
+/* needed for 32-bit applications running on 64-bit kernels */
+typedef struct CK_AES_GMAC_PARAMS32 {
+ caddr32_t pIv;
+ caddr32_t pAAD;
+ uint32_t ulAADLen;
+} CK_AES_GMAC_PARAMS32;
+
+typedef struct CK_ECDH1_DERIVE_PARAMS32 {
+ uint32_t kdf;
+ uint32_t ulSharedDataLen;
+ caddr32_t pSharedData;
+ uint32_t ulPublicDataLen;
+ caddr32_t pPublicData;
+} CK_ECDH1_DERIVE_PARAMS32;
+
+#endif /* _SYSCALL32 */
+
+/*
+ * The measurement unit bit flag for a mechanism's minimum or maximum key size.
+ * The unit are mechanism dependent. It can be in bits or in bytes.
+ */
+typedef uint32_t crypto_keysize_unit_t;
+
+/*
+ * The following bit flags are valid in cm_mech_flags field in
+ * the crypto_mech_info_t structure of the SPI.
+ *
+ * Only the first two bit flags are valid in mi_keysize_unit
+ * field in the crypto_mechanism_info_t structure of the API.
+ */
+#define CRYPTO_KEYSIZE_UNIT_IN_BITS 0x00000001
+#define CRYPTO_KEYSIZE_UNIT_IN_BYTES 0x00000002
+#define CRYPTO_CAN_SHARE_OPSTATE 0x00000004 /* supports sharing */
+
+
+/* Mechanisms supported out-of-the-box */
+#define SUN_CKM_MD4 "CKM_MD4"
+#define SUN_CKM_MD5 "CKM_MD5"
+#define SUN_CKM_MD5_HMAC "CKM_MD5_HMAC"
+#define SUN_CKM_MD5_HMAC_GENERAL "CKM_MD5_HMAC_GENERAL"
+#define SUN_CKM_SHA1 "CKM_SHA_1"
+#define SUN_CKM_SHA1_HMAC "CKM_SHA_1_HMAC"
+#define SUN_CKM_SHA1_HMAC_GENERAL "CKM_SHA_1_HMAC_GENERAL"
+#define SUN_CKM_SHA256 "CKM_SHA256"
+#define SUN_CKM_SHA256_HMAC "CKM_SHA256_HMAC"
+#define SUN_CKM_SHA256_HMAC_GENERAL "CKM_SHA256_HMAC_GENERAL"
+#define SUN_CKM_SHA384 "CKM_SHA384"
+#define SUN_CKM_SHA384_HMAC "CKM_SHA384_HMAC"
+#define SUN_CKM_SHA384_HMAC_GENERAL "CKM_SHA384_HMAC_GENERAL"
+#define SUN_CKM_SHA512 "CKM_SHA512"
+#define SUN_CKM_SHA512_HMAC "CKM_SHA512_HMAC"
+#define SUN_CKM_SHA512_HMAC_GENERAL "CKM_SHA512_HMAC_GENERAL"
+#define SUN_CKM_SHA512_224 "CKM_SHA512_224"
+#define SUN_CKM_SHA512_256 "CKM_SHA512_256"
+#define SUN_CKM_DES_CBC "CKM_DES_CBC"
+#define SUN_CKM_DES3_CBC "CKM_DES3_CBC"
+#define SUN_CKM_DES_ECB "CKM_DES_ECB"
+#define SUN_CKM_DES3_ECB "CKM_DES3_ECB"
+#define SUN_CKM_BLOWFISH_CBC "CKM_BLOWFISH_CBC"
+#define SUN_CKM_BLOWFISH_ECB "CKM_BLOWFISH_ECB"
+#define SUN_CKM_AES_CBC "CKM_AES_CBC"
+#define SUN_CKM_AES_ECB "CKM_AES_ECB"
+#define SUN_CKM_AES_CTR "CKM_AES_CTR"
+#define SUN_CKM_AES_CCM "CKM_AES_CCM"
+#define SUN_CKM_AES_GCM "CKM_AES_GCM"
+#define SUN_CKM_AES_GMAC "CKM_AES_GMAC"
+#define SUN_CKM_AES_CFB128 "CKM_AES_CFB128"
+#define SUN_CKM_RC4 "CKM_RC4"
+#define SUN_CKM_RSA_PKCS "CKM_RSA_PKCS"
+#define SUN_CKM_RSA_X_509 "CKM_RSA_X_509"
+#define SUN_CKM_MD5_RSA_PKCS "CKM_MD5_RSA_PKCS"
+#define SUN_CKM_SHA1_RSA_PKCS "CKM_SHA1_RSA_PKCS"
+#define SUN_CKM_SHA256_RSA_PKCS "CKM_SHA256_RSA_PKCS"
+#define SUN_CKM_SHA384_RSA_PKCS "CKM_SHA384_RSA_PKCS"
+#define SUN_CKM_SHA512_RSA_PKCS "CKM_SHA512_RSA_PKCS"
+#define SUN_CKM_EC_KEY_PAIR_GEN "CKM_EC_KEY_PAIR_GEN"
+#define SUN_CKM_ECDH1_DERIVE "CKM_ECDH1_DERIVE"
+#define SUN_CKM_ECDSA_SHA1 "CKM_ECDSA_SHA1"
+#define SUN_CKM_ECDSA "CKM_ECDSA"
+
+/* Shared operation context format for CKM_RC4 */
+typedef struct {
+#if defined(__amd64)
+ uint32_t i, j;
+ uint32_t arr[256];
+ uint32_t flag;
+#else
+ uchar_t arr[256];
+ uchar_t i, j;
+#endif /* __amd64 */
+ uint64_t pad; /* For 64-bit alignment */
+} arcfour_state_t;
+
+/* Data arguments of cryptographic operations */
+
+typedef enum crypto_data_format {
+ CRYPTO_DATA_RAW = 1,
+ CRYPTO_DATA_UIO,
+} crypto_data_format_t;
+
+typedef struct crypto_data {
+ crypto_data_format_t cd_format; /* Format identifier */
+ off_t cd_offset; /* Offset from the beginning */
+ size_t cd_length; /* # of bytes in use */
+ caddr_t cd_miscdata; /* ancillary data */
+ union {
+ /* Raw format */
+ iovec_t cdu_raw; /* Pointer and length */
+
+ /* uio scatter-gather format */
+ uio_t *cdu_uio;
+
+ } cdu; /* Crypto Data Union */
+} crypto_data_t;
+
+#define cd_raw cdu.cdu_raw
+#define cd_uio cdu.cdu_uio
+#define cd_mp cdu.cdu_mp
+
+typedef struct crypto_dual_data {
+ crypto_data_t dd_data; /* The data */
+ off_t dd_offset2; /* Used by dual operation */
+ size_t dd_len2; /* # of bytes to take */
+} crypto_dual_data_t;
+
+#define dd_format dd_data.cd_format
+#define dd_offset1 dd_data.cd_offset
+#define dd_len1 dd_data.cd_length
+#define dd_miscdata dd_data.cd_miscdata
+#define dd_raw dd_data.cd_raw
+#define dd_uio dd_data.cd_uio
+#define dd_mp dd_data.cd_mp
+
+/* The keys, and their contents */
+
+typedef enum {
+ CRYPTO_KEY_RAW = 1, /* ck_data is a cleartext key */
+ CRYPTO_KEY_REFERENCE, /* ck_obj_id is an opaque reference */
+ CRYPTO_KEY_ATTR_LIST /* ck_attrs is a list of object attributes */
+} crypto_key_format_t;
+
+typedef uint64_t crypto_attr_type_t;
+
+/* Attribute types to use for passing a RSA public key or a private key. */
+#define SUN_CKA_MODULUS 0x00000120
+#define SUN_CKA_MODULUS_BITS 0x00000121
+#define SUN_CKA_PUBLIC_EXPONENT 0x00000122
+#define SUN_CKA_PRIVATE_EXPONENT 0x00000123
+#define SUN_CKA_PRIME_1 0x00000124
+#define SUN_CKA_PRIME_2 0x00000125
+#define SUN_CKA_EXPONENT_1 0x00000126
+#define SUN_CKA_EXPONENT_2 0x00000127
+#define SUN_CKA_COEFFICIENT 0x00000128
+#define SUN_CKA_PRIME 0x00000130
+#define SUN_CKA_SUBPRIME 0x00000131
+#define SUN_CKA_BASE 0x00000132
+
+#define CKK_EC 0x00000003
+#define CKK_GENERIC_SECRET 0x00000010
+#define CKK_RC4 0x00000012
+#define CKK_AES 0x0000001F
+#define CKK_DES 0x00000013
+#define CKK_DES2 0x00000014
+#define CKK_DES3 0x00000015
+
+#define CKO_PUBLIC_KEY 0x00000002
+#define CKO_PRIVATE_KEY 0x00000003
+#define CKA_CLASS 0x00000000
+#define CKA_VALUE 0x00000011
+#define CKA_KEY_TYPE 0x00000100
+#define CKA_VALUE_LEN 0x00000161
+#define CKA_EC_PARAMS 0x00000180
+#define CKA_EC_POINT 0x00000181
+
+typedef uint32_t crypto_object_id_t;
+
+typedef struct crypto_object_attribute {
+ crypto_attr_type_t oa_type; /* attribute type */
+ caddr_t oa_value; /* attribute value */
+ ssize_t oa_value_len; /* length of attribute value */
+} crypto_object_attribute_t;
+
+typedef struct crypto_key {
+ crypto_key_format_t ck_format; /* format identifier */
+ union {
+ /* for CRYPTO_KEY_RAW ck_format */
+ struct {
+ uint_t cku_v_length; /* # of bits in ck_data */
+ void *cku_v_data; /* ptr to key value */
+ } cku_key_value;
+
+ /* for CRYPTO_KEY_REFERENCE ck_format */
+ crypto_object_id_t cku_key_id; /* reference to object key */
+
+ /* for CRYPTO_KEY_ATTR_LIST ck_format */
+ struct {
+ uint_t cku_a_count; /* number of attributes */
+ crypto_object_attribute_t *cku_a_oattr;
+ } cku_key_attrs;
+ } cku_data; /* Crypto Key union */
+} crypto_key_t;
+
+#ifdef _SYSCALL32
+
+typedef struct crypto_object_attribute32 {
+ uint64_t oa_type; /* attribute type */
+ caddr32_t oa_value; /* attribute value */
+ ssize32_t oa_value_len; /* length of attribute value */
+} crypto_object_attribute32_t;
+
+typedef struct crypto_key32 {
+ crypto_key_format_t ck_format; /* format identifier */
+ union {
+ /* for CRYPTO_KEY_RAW ck_format */
+ struct {
+ uint32_t cku_v_length; /* # of bytes in ck_data */
+ caddr32_t cku_v_data; /* ptr to key value */
+ } cku_key_value;
+
+ /* for CRYPTO_KEY_REFERENCE ck_format */
+ crypto_object_id_t cku_key_id; /* reference to object key */
+
+ /* for CRYPTO_KEY_ATTR_LIST ck_format */
+ struct {
+ uint32_t cku_a_count; /* number of attributes */
+ caddr32_t cku_a_oattr;
+ } cku_key_attrs;
+ } cku_data; /* Crypto Key union */
+} crypto_key32_t;
+
+#endif /* _SYSCALL32 */
+
+#define ck_data cku_data.cku_key_value.cku_v_data
+#define ck_length cku_data.cku_key_value.cku_v_length
+#define ck_obj_id cku_data.cku_key_id
+#define ck_count cku_data.cku_key_attrs.cku_a_count
+#define ck_attrs cku_data.cku_key_attrs.cku_a_oattr
+
+/*
+ * Raw key lengths are expressed in number of bits.
+ * The following macro returns the minimum number of
+ * bytes that can contain the specified number of bits.
+ * Round up without overflowing the integer type.
+ */
+#define CRYPTO_BITS2BYTES(n) ((n) == 0 ? 0 : (((n) - 1) >> 3) + 1)
+#define CRYPTO_BYTES2BITS(n) ((n) << 3)
+
+/* Providers */
+
+typedef enum {
+ CRYPTO_HW_PROVIDER = 0,
+ CRYPTO_SW_PROVIDER,
+ CRYPTO_LOGICAL_PROVIDER
+} crypto_provider_type_t;
+
+typedef uint32_t crypto_provider_id_t;
+#define KCF_PROVID_INVALID ((uint32_t)-1)
+
+typedef struct crypto_provider_entry {
+ crypto_provider_id_t pe_provider_id;
+ uint_t pe_mechanism_count;
+} crypto_provider_entry_t;
+
+typedef struct crypto_dev_list_entry {
+ char le_dev_name[MAXNAMELEN];
+ uint_t le_dev_instance;
+ uint_t le_mechanism_count;
+} crypto_dev_list_entry_t;
+
+/* User type for authentication ioctls and SPI entry points */
+
+typedef enum crypto_user_type {
+ CRYPTO_SO = 0,
+ CRYPTO_USER
+} crypto_user_type_t;
+
+/* Version for provider management ioctls and SPI entry points */
+
+typedef struct crypto_version {
+ uchar_t cv_major;
+ uchar_t cv_minor;
+} crypto_version_t;
+
+/* session data structure opaque to the consumer */
+typedef void *crypto_session_t;
+
+/* provider data structure opaque to the consumer */
+typedef void *crypto_provider_t;
+
+/* Limits used by both consumers and providers */
+#define CRYPTO_EXT_SIZE_LABEL 32
+#define CRYPTO_EXT_SIZE_MANUF 32
+#define CRYPTO_EXT_SIZE_MODEL 16
+#define CRYPTO_EXT_SIZE_SERIAL 16
+#define CRYPTO_EXT_SIZE_TIME 16
+
+typedef struct crypto_provider_ext_info {
+ uchar_t ei_label[CRYPTO_EXT_SIZE_LABEL];
+ uchar_t ei_manufacturerID[CRYPTO_EXT_SIZE_MANUF];
+ uchar_t ei_model[CRYPTO_EXT_SIZE_MODEL];
+ uchar_t ei_serial_number[CRYPTO_EXT_SIZE_SERIAL];
+ ulong_t ei_flags;
+ ulong_t ei_max_session_count;
+ ulong_t ei_max_pin_len;
+ ulong_t ei_min_pin_len;
+ ulong_t ei_total_public_memory;
+ ulong_t ei_free_public_memory;
+ ulong_t ei_total_private_memory;
+ ulong_t ei_free_private_memory;
+ crypto_version_t ei_hardware_version;
+ crypto_version_t ei_firmware_version;
+ uchar_t ei_time[CRYPTO_EXT_SIZE_TIME];
+ int ei_hash_max_input_len;
+ int ei_hmac_max_input_len;
+} crypto_provider_ext_info_t;
+
+typedef uint_t crypto_session_id_t;
+
+typedef enum cmd_type {
+ COPY_FROM_DATA,
+ COPY_TO_DATA,
+ COMPARE_TO_DATA,
+ MD5_DIGEST_DATA,
+ SHA1_DIGEST_DATA,
+ SHA2_DIGEST_DATA,
+ GHASH_DATA
+} cmd_type_t;
+
+#define CRYPTO_DO_UPDATE 0x01
+#define CRYPTO_DO_FINAL 0x02
+#define CRYPTO_DO_MD5 0x04
+#define CRYPTO_DO_SHA1 0x08
+#define CRYPTO_DO_SIGN 0x10
+#define CRYPTO_DO_VERIFY 0x20
+#define CRYPTO_DO_SHA2 0x40
+
+#define PROVIDER_OWNS_KEY_SCHEDULE 0x00000001
+
+/*
+ * Common cryptographic status and error codes.
+ */
+#define CRYPTO_SUCCESS 0x00000000
+#define CRYPTO_CANCEL 0x00000001
+#define CRYPTO_HOST_MEMORY 0x00000002
+#define CRYPTO_GENERAL_ERROR 0x00000003
+#define CRYPTO_FAILED 0x00000004
+#define CRYPTO_ARGUMENTS_BAD 0x00000005
+#define CRYPTO_ATTRIBUTE_READ_ONLY 0x00000006
+#define CRYPTO_ATTRIBUTE_SENSITIVE 0x00000007
+#define CRYPTO_ATTRIBUTE_TYPE_INVALID 0x00000008
+#define CRYPTO_ATTRIBUTE_VALUE_INVALID 0x00000009
+#define CRYPTO_CANCELED 0x0000000A
+#define CRYPTO_DATA_INVALID 0x0000000B
+#define CRYPTO_DATA_LEN_RANGE 0x0000000C
+#define CRYPTO_DEVICE_ERROR 0x0000000D
+#define CRYPTO_DEVICE_MEMORY 0x0000000E
+#define CRYPTO_DEVICE_REMOVED 0x0000000F
+#define CRYPTO_ENCRYPTED_DATA_INVALID 0x00000010
+#define CRYPTO_ENCRYPTED_DATA_LEN_RANGE 0x00000011
+#define CRYPTO_KEY_HANDLE_INVALID 0x00000012
+#define CRYPTO_KEY_SIZE_RANGE 0x00000013
+#define CRYPTO_KEY_TYPE_INCONSISTENT 0x00000014
+#define CRYPTO_KEY_NOT_NEEDED 0x00000015
+#define CRYPTO_KEY_CHANGED 0x00000016
+#define CRYPTO_KEY_NEEDED 0x00000017
+#define CRYPTO_KEY_INDIGESTIBLE 0x00000018
+#define CRYPTO_KEY_FUNCTION_NOT_PERMITTED 0x00000019
+#define CRYPTO_KEY_NOT_WRAPPABLE 0x0000001A
+#define CRYPTO_KEY_UNEXTRACTABLE 0x0000001B
+#define CRYPTO_MECHANISM_INVALID 0x0000001C
+#define CRYPTO_MECHANISM_PARAM_INVALID 0x0000001D
+#define CRYPTO_OBJECT_HANDLE_INVALID 0x0000001E
+#define CRYPTO_OPERATION_IS_ACTIVE 0x0000001F
+#define CRYPTO_OPERATION_NOT_INITIALIZED 0x00000020
+#define CRYPTO_PIN_INCORRECT 0x00000021
+#define CRYPTO_PIN_INVALID 0x00000022
+#define CRYPTO_PIN_LEN_RANGE 0x00000023
+#define CRYPTO_PIN_EXPIRED 0x00000024
+#define CRYPTO_PIN_LOCKED 0x00000025
+#define CRYPTO_SESSION_CLOSED 0x00000026
+#define CRYPTO_SESSION_COUNT 0x00000027
+#define CRYPTO_SESSION_HANDLE_INVALID 0x00000028
+#define CRYPTO_SESSION_READ_ONLY 0x00000029
+#define CRYPTO_SESSION_EXISTS 0x0000002A
+#define CRYPTO_SESSION_READ_ONLY_EXISTS 0x0000002B
+#define CRYPTO_SESSION_READ_WRITE_SO_EXISTS 0x0000002C
+#define CRYPTO_SIGNATURE_INVALID 0x0000002D
+#define CRYPTO_SIGNATURE_LEN_RANGE 0x0000002E
+#define CRYPTO_TEMPLATE_INCOMPLETE 0x0000002F
+#define CRYPTO_TEMPLATE_INCONSISTENT 0x00000030
+#define CRYPTO_UNWRAPPING_KEY_HANDLE_INVALID 0x00000031
+#define CRYPTO_UNWRAPPING_KEY_SIZE_RANGE 0x00000032
+#define CRYPTO_UNWRAPPING_KEY_TYPE_INCONSISTENT 0x00000033
+#define CRYPTO_USER_ALREADY_LOGGED_IN 0x00000034
+#define CRYPTO_USER_NOT_LOGGED_IN 0x00000035
+#define CRYPTO_USER_PIN_NOT_INITIALIZED 0x00000036
+#define CRYPTO_USER_TYPE_INVALID 0x00000037
+#define CRYPTO_USER_ANOTHER_ALREADY_LOGGED_IN 0x00000038
+#define CRYPTO_USER_TOO_MANY_TYPES 0x00000039
+#define CRYPTO_WRAPPED_KEY_INVALID 0x0000003A
+#define CRYPTO_WRAPPED_KEY_LEN_RANGE 0x0000003B
+#define CRYPTO_WRAPPING_KEY_HANDLE_INVALID 0x0000003C
+#define CRYPTO_WRAPPING_KEY_SIZE_RANGE 0x0000003D
+#define CRYPTO_WRAPPING_KEY_TYPE_INCONSISTENT 0x0000003E
+#define CRYPTO_RANDOM_SEED_NOT_SUPPORTED 0x0000003F
+#define CRYPTO_RANDOM_NO_RNG 0x00000040
+#define CRYPTO_DOMAIN_PARAMS_INVALID 0x00000041
+#define CRYPTO_BUFFER_TOO_SMALL 0x00000042
+#define CRYPTO_INFORMATION_SENSITIVE 0x00000043
+#define CRYPTO_NOT_SUPPORTED 0x00000044
+
+#define CRYPTO_QUEUED 0x00000045
+#define CRYPTO_BUFFER_TOO_BIG 0x00000046
+#define CRYPTO_INVALID_CONTEXT 0x00000047
+#define CRYPTO_INVALID_MAC 0x00000048
+#define CRYPTO_MECH_NOT_SUPPORTED 0x00000049
+#define CRYPTO_INCONSISTENT_ATTRIBUTE 0x0000004A
+#define CRYPTO_NO_PERMISSION 0x0000004B
+#define CRYPTO_INVALID_PROVIDER_ID 0x0000004C
+#define CRYPTO_VERSION_MISMATCH 0x0000004D
+#define CRYPTO_BUSY 0x0000004E
+#define CRYPTO_UNKNOWN_PROVIDER 0x0000004F
+#define CRYPTO_MODVERIFICATION_FAILED 0x00000050
+#define CRYPTO_OLD_CTX_TEMPLATE 0x00000051
+#define CRYPTO_WEAK_KEY 0x00000052
+#define CRYPTO_FIPS140_ERROR 0x00000053
+/*
+ * Don't forget to update CRYPTO_LAST_ERROR and the error_number_table[]
+ * in kernelUtil.c when new error code is added.
+ */
+#define CRYPTO_LAST_ERROR 0x00000053
+
+/*
+ * Special values that can be used to indicate that information is unavailable
+ * or that there is not practical limit. These values can be used
+ * by fields of the SPI crypto_provider_ext_info(9S) structure.
+ * The value of CRYPTO_UNAVAILABLE_INFO should be the same as
+ * CK_UNAVAILABLE_INFO in the PKCS#11 spec.
+ */
+#define CRYPTO_UNAVAILABLE_INFO ((ulong_t)(-1))
+#define CRYPTO_EFFECTIVELY_INFINITE 0x0
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_CRYPTO_COMMON_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2016, Datto, Inc. All rights reserved.
+ */
+
+#ifndef _SYS_CRYPTO_ALGS_H
+#define _SYS_CRYPTO_ALGS_H
+
+int aes_mod_init(void);
+int aes_mod_fini(void);
+
+int sha1_mod_init(void);
+int sha1_mod_fini(void);
+
+int sha2_mod_init(void);
+int sha2_mod_fini(void);
+
+int icp_init(void);
+void icp_fini(void);
+
+#endif /* _SYS_CRYPTO_ALGS_H */
#include <vm/seg_kmem.h>
#include <sys/zone.h>
#include <sys/sdt.h>
+#include <sys/kstat.h>
#include <sys/zfs_debug.h>
#include <sys/zfs_delay.h>
#include <sys/fm/fs/zfs.h>
/*
* Task queues
*/
-typedef struct taskq taskq_t;
+
+#define TASKQ_NAMELEN 31
+
typedef uintptr_t taskqid_t;
typedef void (task_func_t)(void *);
uintptr_t tqent_flags;
} taskq_ent_t;
+typedef struct taskq {
+ char tq_name[TASKQ_NAMELEN + 1];
+ kmutex_t tq_lock;
+ krwlock_t tq_threadlock;
+ kcondvar_t tq_dispatch_cv;
+ kcondvar_t tq_wait_cv;
+ kthread_t **tq_threadlist;
+ int tq_flags;
+ int tq_active;
+ int tq_nthreads;
+ int tq_nalloc;
+ int tq_minalloc;
+ int tq_maxalloc;
+ kcondvar_t tq_maxalloc_cv;
+ int tq_maxalloc_wait;
+ taskq_ent_t *tq_freelist;
+ taskq_ent_t tq_task;
+} taskq_t;
+
#define TQENT_FLAG_PREALLOC 0x1 /* taskq_dispatch_ent used */
#define TASKQ_PREPOPULATE 0x0001
extern int highbit64(uint64_t i);
extern int lowbit64(uint64_t i);
+extern int highbit(ulong_t i);
+extern int lowbit(ulong_t i);
extern int random_get_bytes(uint8_t *ptr, size_t len);
extern int random_get_pseudo_bytes(uint8_t *ptr, size_t len);
extern void kernel_fini(void);
extern void thread_init(void);
extern void thread_fini(void);
+extern void random_init(void);
+extern void random_fini(void);
struct spa;
extern void nicenum(uint64_t num, char *buf);
# NB: GNU Automake Manual, Chapter 8.3.5: Libtool Convenience Libraries
-# These five libraries are intermediary build components.
-SUBDIRS = libspl libavl libefi libshare libunicode
+# These six libraries are intermediary build components.
+SUBDIRS = libspl libavl libefi libshare libunicode libicp
# These four libraries, which are installed as the final build product,
-# incorporate the five convenience libraries given above.
+# incorporate the six convenience libraries given above.
SUBDIRS += libuutil libnvpair libzpool libzfs_core libzfs
--- /dev/null
+include $(top_srcdir)/config/Rules.am
+
+VPATH = \
+ $(top_srcdir)/module/icp \
+ $(top_srcdir)/lib/libicp
+
+AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN)
+
+DEFAULT_INCLUDES += \
+ -I$(top_srcdir)/include \
+ -I$(top_srcdir)/module/icp/include \
+ -I$(top_srcdir)/lib/libspl/include
+
+noinst_LTLIBRARIES = libicp.la
+
+if TARGET_ASM_X86_64
+ASM_SOURCES_C = asm-x86_64/aes/aeskey.c
+ASM_SOURCES_AS = \
+ asm-x86_64/aes/aes_amd64.S \
+ asm-x86_64/aes/aes_intel.S \
+ asm-x86_64/modes/gcm_intel.S \
+ asm-x86_64/sha1/sha1-x86_64.S \
+ asm-x86_64/sha2/sha256_impl.S
+endif
+
+if TARGET_ASM_I386
+ASM_SOURCES_C =
+ASM_SOURCES_AS =
+endif
+
+if TARGET_ASM_GENERIC
+ASM_SOURCES_C =
+ASM_SOURCES_AS =
+endif
+
+USER_C =
+
+USER_ASM =
+
+KERNEL_C = \
+ spi/kcf_spi.c \
+ api/kcf_ctxops.c \
+ api/kcf_digest.c \
+ api/kcf_cipher.c \
+ api/kcf_miscapi.c \
+ api/kcf_mac.c \
+ algs/aes/aes_impl.c \
+ algs/aes/aes_modes.c \
+ algs/modes/modes.c \
+ algs/modes/cbc.c \
+ algs/modes/gcm.c \
+ algs/modes/ctr.c \
+ algs/modes/ccm.c \
+ algs/modes/ecb.c \
+ algs/sha1/sha1.c \
+ algs/sha2/sha2.c \
+ illumos-crypto.c \
+ io/aes.c \
+ io/sha1_mod.c \
+ io/sha2_mod.c \
+ os/modhash.c \
+ os/modconf.c \
+ core/kcf_sched.c \
+ core/kcf_prov_lib.c \
+ core/kcf_callprov.c \
+ core/kcf_mech_tabs.c \
+ core/kcf_prov_tabs.c \
+ $(ASM_SOURCES_C)
+
+KERNEL_ASM = $(ASM_SOURCES_AS)
+
+nodist_libicp_la_SOURCES = \
+ $(USER_C) \
+ $(USER_ASM) \
+ $(KERNEL_C) \
+ $(KERNEL_ASM)
+
+libicp_la_LIBADD = -lrt
#define BE_64(x) BSWAP_64(x)
#endif
+#ifdef _BIG_ENDIAN
+static __inline__ uint64_t
+htonll(uint64_t n) {
+ return (n);
+}
+
+static __inline__ uint64_t
+ntohll(uint64_t n) {
+ return (n);
+}
+#else
+static __inline__ uint64_t
+htonll(uint64_t n) {
+ return ((((uint64_t)htonl(n)) << 32) + htonl(n >> 32));
+}
+
+static __inline__ uint64_t
+ntohll(uint64_t n) {
+ return ((((uint64_t)ntohl(n)) << 32) + ntohl(n >> 32));
+}
+#endif
+
/*
* Macros to read unaligned values from a specific byte order to
* native byte order
#define FREAD 1
#define FWRITE 2
-// #define FAPPEND 8
+// #define FAPPEND 8
#define FCREAT O_CREAT
#define FTRUNC O_TRUNC
typedef int minor_t;
typedef ushort_t o_mode_t; /* old file attribute type */
+typedef short index_t;
/*
* Definitions remaining from previous partial support for 64-bit file
libzpool_la_LIBADD = \
$(top_builddir)/lib/libunicode/libunicode.la \
$(top_builddir)/lib/libuutil/libuutil.la \
- $(top_builddir)/lib/libnvpair/libnvpair.la
+ $(top_builddir)/lib/libnvpair/libnvpair.la \
+ $(top_builddir)/lib/libicp/libicp.la
libzpool_la_LIBADD += $(ZLIB)
libzpool_la_LDFLAGS = -version-info 2:0:0
#include <sys/time.h>
#include <sys/systeminfo.h>
#include <zfs_fletcher.h>
+#include <sys/crypto/icp.h>
/*
* Emulation of kernel services in userland.
return (h);
}
+/*
+ * Find highest one bit set.
+ * Returns bit number + 1 of highest bit that is set, otherwise returns 0.
+ * High order bit is 31 (or 63 in _LP64 kernel).
+ */
+int
+highbit(ulong_t i)
+{
+register int h = 1;
+
+ if (i == 0)
+ return (0);
+#ifdef _LP64
+ if (i & 0xffffffff00000000ul) {
+ h += 32; i >>= 32;
+ }
+#endif
+ if (i & 0xffff0000) {
+ h += 16; i >>= 16;
+ }
+ if (i & 0xff00) {
+ h += 8; i >>= 8;
+ }
+ if (i & 0xf0) {
+ h += 4; i >>= 4;
+ }
+ if (i & 0xc) {
+ h += 2; i >>= 2;
+ }
+ if (i & 0x2) {
+ h += 1;
+ }
+ return (h);
+}
+
+/*
+ * Find lowest one bit set.
+ * Returns bit number + 1 of lowest bit that is set, otherwise returns 0.
+ * Low order bit is 0.
+ */
+int
+lowbit(ulong_t i)
+{
+ register int h = 1;
+
+ if (i == 0)
+ return (0);
+
+#ifdef _LP64
+ if (!(i & 0xffffffff)) {
+ h += 32; i >>= 32;
+ }
+#endif
+ if (!(i & 0xffff)) {
+ h += 16; i >>= 16;
+ }
+ if (!(i & 0xff)) {
+ h += 8; i >>= 8;
+ }
+ if (!(i & 0xf)) {
+ h += 4; i >>= 4;
+ }
+ if (!(i & 0x3)) {
+ h += 2; i >>= 2;
+ }
+ if (!(i & 0x1)) {
+ h += 1;
+ }
+ return (h);
+}
static int random_fd = -1, urandom_fd = -1;
+void
+random_init(void)
+{
+ VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
+ VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
+}
+
+void
+random_fini(void)
+{
+ close(random_fd);
+ close(urandom_fd);
+
+ random_fd = -1;
+ urandom_fd = -1;
+}
+
static int
random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
{
(void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
(mode & FWRITE) ? get_system_hostid() : 0);
- VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
- VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
+ random_init();
+
VERIFY0(uname(&hw_utsname));
thread_init();
system_taskq_init();
+ icp_init();
spa_init(mode);
fletcher_4_fini();
spa_fini();
+ icp_fini();
system_taskq_fini();
thread_fini();
- close(random_fd);
- close(urandom_fd);
-
- random_fd = -1;
- urandom_fd = -1;
+ random_fini();
}
uid_t
taskq_t *system_taskq;
#define TASKQ_ACTIVE 0x00010000
-#define TASKQ_NAMELEN 31
-
-struct taskq {
- char tq_name[TASKQ_NAMELEN + 1];
- kmutex_t tq_lock;
- krwlock_t tq_threadlock;
- kcondvar_t tq_dispatch_cv;
- kcondvar_t tq_wait_cv;
- kthread_t **tq_threadlist;
- int tq_flags;
- int tq_active;
- int tq_nthreads;
- int tq_nalloc;
- int tq_minalloc;
- int tq_maxalloc;
- kcondvar_t tq_maxalloc_cv;
- int tq_maxalloc_wait;
- taskq_ent_t *tq_freelist;
- taskq_ent_t tq_task;
-};
static taskq_ent_t *
task_alloc(taskq_t *tq, int tqflags)
subdir-m += zcommon
subdir-m += zfs
subdir-m += zpios
+subdir-m += icp
INSTALL_MOD_DIR ?= extra
ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include -I@SPL@/include -I@SPL@
export ZFS_MODULE_CFLAGS
+SUBDIR_TARGETS = icp
+
modules:
@# Make the exported SPL symbols available to these modules.
@# They may be in the root of SPL_OBJ when building against
"*** - @SPL_OBJ@/module/@SPL_SYMBOLS@\n"; \
exit 1; \
fi
+ list='$(SUBDIR_TARGETS)'; for targetdir in $$list; do \
+ $(MAKE) -C $$targetdir; \
+ done
$(MAKE) -C @LINUX_OBJ@ SUBDIRS=`pwd` @KERNELMAKE_PARAMS@ CONFIG_ZFS=m $@
clean:
distdir:
list='$(subdir-m)'; for subdir in $$list; do \
- (find @top_srcdir@/module/$$subdir -name '*.c' -o -name '*.h' |\
- xargs /bin/cp -t $$distdir/$$subdir); \
+ (cd @top_srcdir@/module && find $$subdir -name '*.c' -o -name '*.h' -o -name '*.S' |\
+ xargs /bin/cp --parents -t $$distdir); \
done
distclean maintainer-clean: clean
--- /dev/null
+src = @abs_top_srcdir@/module/icp
+obj = @abs_builddir@
+
+MODULE := icp
+
+TARGET_ASM_DIR = @TARGET_ASM_DIR@
+
+ifeq ($(TARGET_ASM_DIR), asm-x86_64)
+ASM_SOURCES := asm-x86_64/aes/aeskey.o
+ASM_SOURCES += asm-x86_64/aes/aes_amd64.o
+ASM_SOURCES += asm-x86_64/aes/aes_intel.o
+ASM_SOURCES += asm-x86_64/modes/gcm_intel.o
+ASM_SOURCES += asm-x86_64/sha1/sha1-x86_64.o
+ASM_SOURCES += asm-x86_64/sha2/sha256_impl.o
+endif
+
+ifeq ($(TARGET_ASM_DIR), asm-i386)
+ASM_SOURCES :=
+endif
+
+ifeq ($(TARGET_ASM_DIR), asm-generic)
+ASM_SOURCES :=
+endif
+
+EXTRA_CFLAGS = $(ZFS_MODULE_CFLAGS) @KERNELCPPFLAGS@
+
+obj-$(CONFIG_ZFS) := $(MODULE).o
+
+ccflags-y += -I$(src)/include
+asflags-y += -I$(src)/include
+asflags-y += $(ZFS_MODULE_CFLAGS)
+
+$(MODULE)-objs += illumos-crypto.o
+$(MODULE)-objs += api/kcf_cipher.o
+$(MODULE)-objs += api/kcf_digest.o
+$(MODULE)-objs += api/kcf_mac.o
+$(MODULE)-objs += api/kcf_miscapi.o
+$(MODULE)-objs += api/kcf_ctxops.o
+$(MODULE)-objs += core/kcf_callprov.o
+$(MODULE)-objs += core/kcf_prov_tabs.o
+$(MODULE)-objs += core/kcf_sched.o
+$(MODULE)-objs += core/kcf_mech_tabs.o
+$(MODULE)-objs += core/kcf_prov_lib.o
+$(MODULE)-objs += spi/kcf_spi.o
+$(MODULE)-objs += io/aes.o
+$(MODULE)-objs += io/sha1_mod.o
+$(MODULE)-objs += io/sha2_mod.o
+$(MODULE)-objs += os/modhash.o
+$(MODULE)-objs += os/modconf.o
+$(MODULE)-objs += algs/modes/cbc.o
+$(MODULE)-objs += algs/modes/ccm.o
+$(MODULE)-objs += algs/modes/ctr.o
+$(MODULE)-objs += algs/modes/ecb.o
+$(MODULE)-objs += algs/modes/gcm.o
+$(MODULE)-objs += algs/modes/modes.o
+$(MODULE)-objs += algs/aes/aes_impl.o
+$(MODULE)-objs += algs/aes/aes_modes.o
+$(MODULE)-objs += algs/sha1/sha1.o
+$(MODULE)-objs += algs/sha2/sha2.o
+$(MODULE)-objs += $(ASM_SOURCES)
+
+ICP_DIRS = \
+ api \
+ core \
+ spi \
+ io \
+ os \
+ algs \
+ algs/aes \
+ algs/modes \
+ algs/sha1 \
+ algs/sha2 \
+ asm-x86_64 \
+ asm-x86_64/aes \
+ asm-x86_64/modes \
+ asm-x86_64/sha1 \
+ asm-x86_64/sha2 \
+ asm-i386 \
+ asm-generic
+
+all:
+ mkdir -p $(ICP_DIRS)
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/spi.h>
+#include <modes/modes.h>
+#include <aes/aes_impl.h>
+
+#ifdef __amd64
+
+#ifdef _KERNEL
+/* Workaround for no XMM kernel thread save/restore */
+#define KPREEMPT_DISABLE kpreempt_disable()
+#define KPREEMPT_ENABLE kpreempt_enable()
+
+#else
+#define KPREEMPT_DISABLE
+#define KPREEMPT_ENABLE
+#endif /* _KERNEL */
+#endif /* __amd64 */
+
+
+/*
+ * This file is derived from the file rijndael-alg-fst.c taken from the
+ * "optimized C code v3.0" on the "rijndael home page"
+ * http://www.iaik.tu-graz.ac.at/research/krypto/AES/old/~rijmen/rijndael/
+ * pointed by the NIST web-site http://csrc.nist.gov/archive/aes/
+ *
+ * The following note is from the original file:
+ */
+
+/*
+ * rijndael-alg-fst.c
+ *
+ * @version 3.0 (December 2000)
+ *
+ * Optimised ANSI C code for the Rijndael cipher (now AES)
+ *
+ * @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be>
+ * @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be>
+ * @author Paulo Barreto <paulo.barreto@terra.com.br>
+ *
+ * This code is hereby placed in the public domain.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#if defined(__amd64)
+
+/* These functions are used to execute amd64 instructions for AMD or Intel: */
+extern int rijndael_key_setup_enc_amd64(uint32_t rk[],
+ const uint32_t cipherKey[], int keyBits);
+extern int rijndael_key_setup_dec_amd64(uint32_t rk[],
+ const uint32_t cipherKey[], int keyBits);
+extern void aes_encrypt_amd64(const uint32_t rk[], int Nr,
+ const uint32_t pt[4], uint32_t ct[4]);
+extern void aes_decrypt_amd64(const uint32_t rk[], int Nr,
+ const uint32_t ct[4], uint32_t pt[4]);
+
+/* These functions are used to execute Intel-specific AES-NI instructions: */
+extern int rijndael_key_setup_enc_intel(uint32_t rk[],
+ const uint32_t cipherKey[], uint64_t keyBits);
+extern int rijndael_key_setup_dec_intel(uint32_t rk[],
+ const uint32_t cipherKey[], uint64_t keyBits);
+extern void aes_encrypt_intel(const uint32_t rk[], int Nr,
+ const uint32_t pt[4], uint32_t ct[4]);
+extern void aes_decrypt_intel(const uint32_t rk[], int Nr,
+ const uint32_t ct[4], uint32_t pt[4]);
+
+static int intel_aes_instructions_present(void);
+
+#define AES_ENCRYPT_IMPL(a, b, c, d, e) rijndael_encrypt(a, b, c, d, e)
+#define AES_DECRYPT_IMPL(a, b, c, d, e) rijndael_decrypt(a, b, c, d, e)
+
+#else /* Generic C implementation */
+
+#define AES_ENCRYPT_IMPL(a, b, c, d, e) rijndael_encrypt(a, b, c, d)
+#define AES_DECRYPT_IMPL(a, b, c, d, e) rijndael_decrypt(a, b, c, d)
+#define rijndael_key_setup_enc_raw rijndael_key_setup_enc
+#endif /* __amd64 */
+
+#if defined(_LITTLE_ENDIAN) && !defined(__amd64)
+#define AES_BYTE_SWAP
+#endif
+
+
+#if !defined(__amd64)
+/*
+ * Constant tables
+ */
+
+/*
+ * Te0[x] = S [x].[02, 01, 01, 03];
+ * Te1[x] = S [x].[03, 02, 01, 01];
+ * Te2[x] = S [x].[01, 03, 02, 01];
+ * Te3[x] = S [x].[01, 01, 03, 02];
+ * Te4[x] = S [x].[01, 01, 01, 01];
+ *
+ * Td0[x] = Si[x].[0e, 09, 0d, 0b];
+ * Td1[x] = Si[x].[0b, 0e, 09, 0d];
+ * Td2[x] = Si[x].[0d, 0b, 0e, 09];
+ * Td3[x] = Si[x].[09, 0d, 0b, 0e];
+ * Td4[x] = Si[x].[01, 01, 01, 01];
+ */
+
+/* Encrypt Sbox constants (for the substitute bytes operation) */
+
+static const uint32_t Te0[256] =
+{
+ 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
+ 0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
+ 0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
+ 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
+ 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
+ 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
+ 0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
+ 0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
+ 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
+ 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
+ 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
+ 0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
+ 0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
+ 0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
+ 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
+ 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
+ 0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
+ 0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
+ 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
+ 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
+ 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
+ 0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
+ 0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
+ 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
+ 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
+ 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
+ 0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
+ 0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
+ 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
+ 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
+ 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
+ 0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
+ 0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
+ 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
+ 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
+ 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
+ 0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
+ 0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
+ 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
+ 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
+ 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
+ 0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
+ 0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
+ 0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
+ 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
+ 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
+ 0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
+ 0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
+ 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
+ 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
+ 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
+ 0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
+ 0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
+ 0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
+ 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
+ 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
+ 0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
+ 0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
+ 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
+ 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
+ 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
+ 0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
+ 0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
+ 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU
+};
+
+
+static const uint32_t Te1[256] =
+{
+ 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
+ 0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
+ 0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
+ 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
+ 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
+ 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
+ 0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
+ 0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
+ 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
+ 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
+ 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
+ 0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
+ 0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
+ 0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
+ 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
+ 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
+ 0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
+ 0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
+ 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
+ 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
+ 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
+ 0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
+ 0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
+ 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
+ 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
+ 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
+ 0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
+ 0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
+ 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
+ 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
+ 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
+ 0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
+ 0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
+ 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
+ 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
+ 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
+ 0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
+ 0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
+ 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
+ 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
+ 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
+ 0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
+ 0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
+ 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
+ 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
+ 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
+ 0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
+ 0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
+ 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
+ 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
+ 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
+ 0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
+ 0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
+ 0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
+ 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
+ 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
+ 0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
+ 0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
+ 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
+ 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
+ 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
+ 0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
+ 0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
+ 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U
+};
+
+
+static const uint32_t Te2[256] =
+{
+ 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
+ 0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
+ 0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
+ 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
+ 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
+ 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
+ 0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
+ 0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
+ 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
+ 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
+ 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
+ 0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
+ 0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
+ 0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
+ 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
+ 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
+ 0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
+ 0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
+ 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
+ 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
+ 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
+ 0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
+ 0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
+ 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
+ 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
+ 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
+ 0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
+ 0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
+ 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
+ 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
+ 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
+ 0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
+ 0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
+ 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
+ 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
+ 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
+ 0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
+ 0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
+ 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
+ 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
+ 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
+ 0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
+ 0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
+ 0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
+ 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
+ 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
+ 0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
+ 0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
+ 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
+ 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
+ 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
+ 0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
+ 0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
+ 0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
+ 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
+ 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
+ 0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
+ 0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
+ 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
+ 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
+ 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
+ 0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
+ 0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
+ 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U
+};
+
+
+static const uint32_t Te3[256] =
+{
+ 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
+ 0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
+ 0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
+ 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
+ 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
+ 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
+ 0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
+ 0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
+ 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
+ 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
+ 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
+ 0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
+ 0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
+ 0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
+ 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
+ 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
+ 0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
+ 0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
+ 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
+ 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
+ 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
+ 0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
+ 0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
+ 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
+ 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
+ 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
+ 0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
+ 0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
+ 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
+ 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
+ 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
+ 0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
+ 0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
+ 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
+ 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
+ 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
+ 0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
+ 0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
+ 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
+ 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
+ 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
+ 0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
+ 0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
+ 0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
+ 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
+ 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
+ 0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
+ 0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
+ 0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
+ 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
+ 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
+ 0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
+ 0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
+ 0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
+ 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
+ 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
+ 0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
+ 0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
+ 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
+ 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
+ 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
+ 0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
+ 0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
+ 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU
+};
+
+static const uint32_t Te4[256] =
+{
+ 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU,
+ 0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U,
+ 0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU,
+ 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U,
+ 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU,
+ 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U,
+ 0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU,
+ 0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U,
+ 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U,
+ 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU,
+ 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U,
+ 0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U,
+ 0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U,
+ 0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU,
+ 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U,
+ 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U,
+ 0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU,
+ 0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U,
+ 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U,
+ 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U,
+ 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU,
+ 0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU,
+ 0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U,
+ 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU,
+ 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU,
+ 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U,
+ 0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU,
+ 0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U,
+ 0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU,
+ 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U,
+ 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U,
+ 0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U,
+ 0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU,
+ 0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U,
+ 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU,
+ 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U,
+ 0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU,
+ 0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U,
+ 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U,
+ 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU,
+ 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU,
+ 0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU,
+ 0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U,
+ 0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U,
+ 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU,
+ 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U,
+ 0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU,
+ 0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U,
+ 0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU,
+ 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U,
+ 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU,
+ 0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU,
+ 0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U,
+ 0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU,
+ 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U,
+ 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU,
+ 0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U,
+ 0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U,
+ 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U,
+ 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU,
+ 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU,
+ 0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U,
+ 0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU,
+ 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U
+};
+
+/* Decrypt Sbox constants (for the substitute bytes operation) */
+
+static const uint32_t Td0[256] =
+{
+ 0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
+ 0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
+ 0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
+ 0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU,
+ 0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U,
+ 0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U,
+ 0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU,
+ 0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U,
+ 0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU,
+ 0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U,
+ 0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U,
+ 0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U,
+ 0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U,
+ 0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU,
+ 0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U,
+ 0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU,
+ 0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U,
+ 0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU,
+ 0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U,
+ 0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U,
+ 0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U,
+ 0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU,
+ 0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U,
+ 0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU,
+ 0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U,
+ 0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU,
+ 0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U,
+ 0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU,
+ 0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU,
+ 0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U,
+ 0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU,
+ 0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U,
+ 0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU,
+ 0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U,
+ 0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U,
+ 0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U,
+ 0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU,
+ 0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U,
+ 0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U,
+ 0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU,
+ 0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U,
+ 0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U,
+ 0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U,
+ 0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U,
+ 0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U,
+ 0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU,
+ 0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U,
+ 0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U,
+ 0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U,
+ 0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U,
+ 0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U,
+ 0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU,
+ 0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU,
+ 0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU,
+ 0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU,
+ 0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U,
+ 0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U,
+ 0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU,
+ 0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU,
+ 0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U,
+ 0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU,
+ 0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U,
+ 0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
+ 0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U
+};
+
+static const uint32_t Td1[256] =
+{
+ 0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
+ 0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
+ 0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
+ 0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U,
+ 0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U,
+ 0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U,
+ 0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U,
+ 0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U,
+ 0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U,
+ 0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU,
+ 0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU,
+ 0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU,
+ 0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U,
+ 0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU,
+ 0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U,
+ 0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U,
+ 0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U,
+ 0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU,
+ 0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU,
+ 0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U,
+ 0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU,
+ 0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U,
+ 0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU,
+ 0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU,
+ 0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U,
+ 0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U,
+ 0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U,
+ 0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU,
+ 0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U,
+ 0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU,
+ 0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U,
+ 0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U,
+ 0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U,
+ 0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU,
+ 0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U,
+ 0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U,
+ 0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U,
+ 0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U,
+ 0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U,
+ 0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U,
+ 0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU,
+ 0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU,
+ 0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U,
+ 0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU,
+ 0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U,
+ 0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU,
+ 0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU,
+ 0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U,
+ 0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU,
+ 0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U,
+ 0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U,
+ 0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U,
+ 0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U,
+ 0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U,
+ 0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U,
+ 0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U,
+ 0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU,
+ 0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U,
+ 0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U,
+ 0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU,
+ 0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U,
+ 0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U,
+ 0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
+ 0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U
+};
+
+static const uint32_t Td2[256] =
+{
+ 0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
+ 0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
+ 0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
+ 0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U,
+ 0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU,
+ 0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U,
+ 0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U,
+ 0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U,
+ 0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U,
+ 0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU,
+ 0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U,
+ 0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U,
+ 0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU,
+ 0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U,
+ 0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U,
+ 0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U,
+ 0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U,
+ 0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U,
+ 0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U,
+ 0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU,
+ 0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U,
+ 0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U,
+ 0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U,
+ 0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U,
+ 0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U,
+ 0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU,
+ 0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU,
+ 0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U,
+ 0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU,
+ 0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U,
+ 0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU,
+ 0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU,
+ 0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU,
+ 0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU,
+ 0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U,
+ 0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U,
+ 0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U,
+ 0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U,
+ 0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U,
+ 0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U,
+ 0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U,
+ 0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU,
+ 0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU,
+ 0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U,
+ 0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U,
+ 0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU,
+ 0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU,
+ 0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U,
+ 0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U,
+ 0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U,
+ 0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U,
+ 0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U,
+ 0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U,
+ 0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U,
+ 0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU,
+ 0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U,
+ 0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U,
+ 0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U,
+ 0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U,
+ 0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U,
+ 0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U,
+ 0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU,
+ 0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
+ 0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U
+};
+
+static const uint32_t Td3[256] =
+{
+ 0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
+ 0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
+ 0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
+ 0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U,
+ 0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU,
+ 0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU,
+ 0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U,
+ 0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU,
+ 0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U,
+ 0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU,
+ 0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U,
+ 0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U,
+ 0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U,
+ 0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U,
+ 0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U,
+ 0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU,
+ 0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU,
+ 0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U,
+ 0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U,
+ 0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU,
+ 0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU,
+ 0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U,
+ 0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U,
+ 0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U,
+ 0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U,
+ 0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU,
+ 0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U,
+ 0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U,
+ 0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU,
+ 0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU,
+ 0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U,
+ 0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U,
+ 0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U,
+ 0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU,
+ 0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U,
+ 0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U,
+ 0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U,
+ 0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U,
+ 0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U,
+ 0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U,
+ 0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U,
+ 0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU,
+ 0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U,
+ 0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U,
+ 0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU,
+ 0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU,
+ 0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U,
+ 0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU,
+ 0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U,
+ 0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U,
+ 0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U,
+ 0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U,
+ 0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U,
+ 0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U,
+ 0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU,
+ 0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU,
+ 0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU,
+ 0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU,
+ 0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U,
+ 0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U,
+ 0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U,
+ 0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU,
+ 0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
+ 0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U
+};
+
+static const uint32_t Td4[256] =
+{
+ 0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U,
+ 0x30303030U, 0x36363636U, 0xa5a5a5a5U, 0x38383838U,
+ 0xbfbfbfbfU, 0x40404040U, 0xa3a3a3a3U, 0x9e9e9e9eU,
+ 0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, 0xfbfbfbfbU,
+ 0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U,
+ 0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U,
+ 0x34343434U, 0x8e8e8e8eU, 0x43434343U, 0x44444444U,
+ 0xc4c4c4c4U, 0xdedededeU, 0xe9e9e9e9U, 0xcbcbcbcbU,
+ 0x54545454U, 0x7b7b7b7bU, 0x94949494U, 0x32323232U,
+ 0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU,
+ 0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU,
+ 0x42424242U, 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU,
+ 0x08080808U, 0x2e2e2e2eU, 0xa1a1a1a1U, 0x66666666U,
+ 0x28282828U, 0xd9d9d9d9U, 0x24242424U, 0xb2b2b2b2U,
+ 0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U,
+ 0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U,
+ 0x72727272U, 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U,
+ 0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U,
+ 0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, 0xccccccccU,
+ 0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U,
+ 0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U,
+ 0xfdfdfdfdU, 0xededededU, 0xb9b9b9b9U, 0xdadadadaU,
+ 0x5e5e5e5eU, 0x15151515U, 0x46464646U, 0x57575757U,
+ 0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, 0x84848484U,
+ 0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U,
+ 0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU,
+ 0xf7f7f7f7U, 0xe4e4e4e4U, 0x58585858U, 0x05050505U,
+ 0xb8b8b8b8U, 0xb3b3b3b3U, 0x45454545U, 0x06060606U,
+ 0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, 0x8f8f8f8fU,
+ 0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U,
+ 0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U,
+ 0x01010101U, 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU,
+ 0x3a3a3a3aU, 0x91919191U, 0x11111111U, 0x41414141U,
+ 0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, 0xeaeaeaeaU,
+ 0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU,
+ 0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U,
+ 0x96969696U, 0xacacacacU, 0x74747474U, 0x22222222U,
+ 0xe7e7e7e7U, 0xadadadadU, 0x35353535U, 0x85858585U,
+ 0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, 0xe8e8e8e8U,
+ 0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU,
+ 0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U,
+ 0x1d1d1d1dU, 0x29292929U, 0xc5c5c5c5U, 0x89898989U,
+ 0x6f6f6f6fU, 0xb7b7b7b7U, 0x62626262U, 0x0e0e0e0eU,
+ 0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, 0x1b1b1b1bU,
+ 0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU,
+ 0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U,
+ 0x9a9a9a9aU, 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU,
+ 0x78787878U, 0xcdcdcdcdU, 0x5a5a5a5aU, 0xf4f4f4f4U,
+ 0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, 0x33333333U,
+ 0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U,
+ 0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U,
+ 0x27272727U, 0x80808080U, 0xececececU, 0x5f5f5f5fU,
+ 0x60606060U, 0x51515151U, 0x7f7f7f7fU, 0xa9a9a9a9U,
+ 0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, 0x0d0d0d0dU,
+ 0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU,
+ 0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU,
+ 0xa0a0a0a0U, 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU,
+ 0xaeaeaeaeU, 0x2a2a2a2aU, 0xf5f5f5f5U, 0xb0b0b0b0U,
+ 0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, 0x3c3c3c3cU,
+ 0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U,
+ 0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU,
+ 0xbabababaU, 0x77777777U, 0xd6d6d6d6U, 0x26262626U,
+ 0xe1e1e1e1U, 0x69696969U, 0x14141414U, 0x63636363U,
+ 0x55555555U, 0x21212121U, 0x0c0c0c0cU, 0x7d7d7d7dU
+};
+
+/* Rcon is Round Constant; used for encryption key expansion */
+static const uint32_t rcon[RC_LENGTH] =
+{
+ /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
+ 0x01000000, 0x02000000, 0x04000000, 0x08000000,
+ 0x10000000, 0x20000000, 0x40000000, 0x80000000,
+ 0x1B000000, 0x36000000
+};
+
+
+/*
+ * Expand the cipher key into the encryption key schedule.
+ *
+ * Return the number of rounds for the given cipher key size.
+ * The size of the key schedule depends on the number of rounds
+ * (which can be computed from the size of the key), i.e. 4*(Nr + 1).
+ *
+ * Parameters:
+ * rk AES key schedule 32-bit array to be initialized
+ * cipherKey User key
+ * keyBits AES key size (128, 192, or 256 bits)
+ */
+static int
+rijndael_key_setup_enc_raw(uint32_t rk[], const uint32_t cipherKey[],
+ int keyBits)
+{
+ int i = 0;
+ uint32_t temp;
+
+ rk[0] = cipherKey[0];
+ rk[1] = cipherKey[1];
+ rk[2] = cipherKey[2];
+ rk[3] = cipherKey[3];
+
+ if (keyBits == 128) {
+ for (;;) {
+ temp = rk[3];
+ rk[4] = rk[0] ^
+ (Te4[(temp >> 16) & 0xff] & 0xff000000) ^
+ (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^
+ (Te4[temp & 0xff] & 0x0000ff00) ^
+ (Te4[temp >> 24] & 0x000000ff) ^
+ rcon[i];
+ rk[5] = rk[1] ^ rk[4];
+ rk[6] = rk[2] ^ rk[5];
+ rk[7] = rk[3] ^ rk[6];
+
+ if (++i == 10) {
+ return (10);
+ }
+ rk += 4;
+ }
+ }
+
+ rk[4] = cipherKey[4];
+ rk[5] = cipherKey[5];
+
+ if (keyBits == 192) {
+ for (;;) {
+ temp = rk[5];
+ rk[6] = rk[0] ^
+ (Te4[(temp >> 16) & 0xff] & 0xff000000) ^
+ (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^
+ (Te4[temp & 0xff] & 0x0000ff00) ^
+ (Te4[temp >> 24] & 0x000000ff) ^
+ rcon[i];
+ rk[7] = rk[1] ^ rk[6];
+ rk[8] = rk[2] ^ rk[7];
+ rk[9] = rk[3] ^ rk[8];
+
+ if (++i == 8) {
+ return (12);
+ }
+
+ rk[10] = rk[4] ^ rk[9];
+ rk[11] = rk[5] ^ rk[10];
+ rk += 6;
+ }
+ }
+
+ rk[6] = cipherKey[6];
+ rk[7] = cipherKey[7];
+
+ if (keyBits == 256) {
+ for (;;) {
+ temp = rk[7];
+ rk[8] = rk[0] ^
+ (Te4[(temp >> 16) & 0xff] & 0xff000000) ^
+ (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^
+ (Te4[temp & 0xff] & 0x0000ff00) ^
+ (Te4[temp >> 24] & 0x000000ff) ^
+ rcon[i];
+ rk[9] = rk[1] ^ rk[8];
+ rk[10] = rk[2] ^ rk[9];
+ rk[11] = rk[3] ^ rk[10];
+
+ if (++i == 7) {
+ return (14);
+ }
+ temp = rk[11];
+ rk[12] = rk[4] ^
+ (Te4[temp >> 24] & 0xff000000) ^
+ (Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^
+ (Te4[(temp >> 8) & 0xff] & 0x0000ff00) ^
+ (Te4[temp & 0xff] & 0x000000ff);
+ rk[13] = rk[5] ^ rk[12];
+ rk[14] = rk[6] ^ rk[13];
+ rk[15] = rk[7] ^ rk[14];
+
+ rk += 8;
+ }
+ }
+
+ return (0);
+}
+#endif /* !__amd64 */
+
+#if defined(__amd64)
+
+/*
+ * Expand the 32-bit AES cipher key array into the encryption and decryption
+ * key schedules.
+ *
+ * Parameters:
+ * key AES key schedule to be initialized
+ * keyarr32 User key
+ * keyBits AES key size (128, 192, or 256 bits)
+ */
+static void
+aes_setupkeys(aes_key_t *key, const uint32_t *keyarr32, int keybits)
+{
+ if (intel_aes_instructions_present()) {
+ key->flags = INTEL_AES_NI_CAPABLE;
+ KPREEMPT_DISABLE;
+ key->nr = rijndael_key_setup_enc_intel(&(key->encr_ks.ks32[0]),
+ keyarr32, keybits);
+ key->nr = rijndael_key_setup_dec_intel(&(key->decr_ks.ks32[0]),
+ keyarr32, keybits);
+ KPREEMPT_ENABLE;
+ } else {
+ key->flags = 0;
+ key->nr = rijndael_key_setup_enc_amd64(&(key->encr_ks.ks32[0]),
+ keyarr32, keybits);
+ key->nr = rijndael_key_setup_dec_amd64(&(key->decr_ks.ks32[0]),
+ keyarr32, keybits);
+ }
+
+ key->type = AES_32BIT_KS;
+}
+
+/*
+ * Encrypt one block of data. The block is assumed to be an array
+ * of four uint32_t values, so copy for alignment (and byte-order
+ * reversal for little endian systems might be necessary on the
+ * input and output byte streams.
+ * The size of the key schedule depends on the number of rounds
+ * (which can be computed from the size of the key), i.e. 4*(Nr + 1).
+ *
+ * Parameters:
+ * rk Key schedule, of aes_ks_t (60 32-bit integers)
+ * Nr Number of rounds
+ * pt Input block (plain text)
+ * ct Output block (crypto text). Can overlap with pt
+ * flags Indicates whether we're on Intel AES-NI-capable hardware
+ */
+static void
+rijndael_encrypt(const uint32_t rk[], int Nr, const uint32_t pt[4],
+ uint32_t ct[4], int flags) {
+ if (flags & INTEL_AES_NI_CAPABLE) {
+ KPREEMPT_DISABLE;
+ aes_encrypt_intel(rk, Nr, pt, ct);
+ KPREEMPT_ENABLE;
+ } else {
+ aes_encrypt_amd64(rk, Nr, pt, ct);
+ }
+}
+
+/*
+ * Decrypt one block of data. The block is assumed to be an array
+ * of four uint32_t values, so copy for alignment (and byte-order
+ * reversal for little endian systems might be necessary on the
+ * input and output byte streams.
+ * The size of the key schedule depends on the number of rounds
+ * (which can be computed from the size of the key), i.e. 4*(Nr + 1).
+ *
+ * Parameters:
+ * rk Key schedule, of aes_ks_t (60 32-bit integers)
+ * Nr Number of rounds
+ * ct Input block (crypto text)
+ * pt Output block (plain text). Can overlap with pt
+ * flags Indicates whether we're on Intel AES-NI-capable hardware
+ */
+static void
+rijndael_decrypt(const uint32_t rk[], int Nr, const uint32_t ct[4],
+ uint32_t pt[4], int flags) {
+ if (flags & INTEL_AES_NI_CAPABLE) {
+ KPREEMPT_DISABLE;
+ aes_decrypt_intel(rk, Nr, ct, pt);
+ KPREEMPT_ENABLE;
+ } else {
+ aes_decrypt_amd64(rk, Nr, ct, pt);
+ }
+}
+
+
+#else /* generic C implementation */
+
+/*
+ * Expand the cipher key into the decryption key schedule.
+ * Return the number of rounds for the given cipher key size.
+ * The size of the key schedule depends on the number of rounds
+ * (which can be computed from the size of the key), i.e. 4*(Nr + 1).
+ *
+ * Parameters:
+ * rk AES key schedule 32-bit array to be initialized
+ * cipherKey User key
+ * keyBits AES key size (128, 192, or 256 bits)
+ */
+static int
+rijndael_key_setup_dec(uint32_t rk[], const uint32_t cipherKey[], int keyBits)
+{
+ int Nr, i, j;
+ uint32_t temp;
+
+ /* expand the cipher key: */
+ Nr = rijndael_key_setup_enc_raw(rk, cipherKey, keyBits);
+
+ /* invert the order of the round keys: */
+ for (i = 0, j = 4 * Nr; i < j; i += 4, j -= 4) {
+ temp = rk[i];
+ rk[i] = rk[j];
+ rk[j] = temp;
+ temp = rk[i + 1];
+ rk[i + 1] = rk[j + 1];
+ rk[j + 1] = temp;
+ temp = rk[i + 2];
+ rk[i + 2] = rk[j + 2];
+ rk[j + 2] = temp;
+ temp = rk[i + 3];
+ rk[i + 3] = rk[j + 3];
+ rk[j + 3] = temp;
+ }
+
+ /*
+ * apply the inverse MixColumn transform to all
+ * round keys but the first and the last:
+ */
+ for (i = 1; i < Nr; i++) {
+ rk += 4;
+ rk[0] = Td0[Te4[rk[0] >> 24] & 0xff] ^
+ Td1[Te4[(rk[0] >> 16) & 0xff] & 0xff] ^
+ Td2[Te4[(rk[0] >> 8) & 0xff] & 0xff] ^
+ Td3[Te4[rk[0] & 0xff] & 0xff];
+ rk[1] = Td0[Te4[rk[1] >> 24] & 0xff] ^
+ Td1[Te4[(rk[1] >> 16) & 0xff] & 0xff] ^
+ Td2[Te4[(rk[1] >> 8) & 0xff] & 0xff] ^
+ Td3[Te4[rk[1] & 0xff] & 0xff];
+ rk[2] = Td0[Te4[rk[2] >> 24] & 0xff] ^
+ Td1[Te4[(rk[2] >> 16) & 0xff] & 0xff] ^
+ Td2[Te4[(rk[2] >> 8) & 0xff] & 0xff] ^
+ Td3[Te4[rk[2] & 0xff] & 0xff];
+ rk[3] = Td0[Te4[rk[3] >> 24] & 0xff] ^
+ Td1[Te4[(rk[3] >> 16) & 0xff] & 0xff] ^
+ Td2[Te4[(rk[3] >> 8) & 0xff] & 0xff] ^
+ Td3[Te4[rk[3] & 0xff] & 0xff];
+ }
+
+ return (Nr);
+}
+
+
+/*
+ * Expand the 32-bit AES cipher key array into the encryption and decryption
+ * key schedules.
+ *
+ * Parameters:
+ * key AES key schedule to be initialized
+ * keyarr32 User key
+ * keyBits AES key size (128, 192, or 256 bits)
+ */
+static void
+aes_setupkeys(aes_key_t *key, const uint32_t *keyarr32, int keybits)
+{
+ key->nr = rijndael_key_setup_enc(&(key->encr_ks.ks32[0]), keyarr32,
+ keybits);
+ key->nr = rijndael_key_setup_dec(&(key->decr_ks.ks32[0]), keyarr32,
+ keybits);
+ key->type = AES_32BIT_KS;
+}
+
+
+/*
+ * Encrypt one block of data. The block is assumed to be an array
+ * of four uint32_t values, so copy for alignment (and byte-order
+ * reversal for little endian systems might be necessary on the
+ * input and output byte streams.
+ * The size of the key schedule depends on the number of rounds
+ * (which can be computed from the size of the key), i.e. 4*(Nr + 1).
+ *
+ * Parameters:
+ * rk Key schedule, of aes_ks_t (60 32-bit integers)
+ * Nr Number of rounds
+ * pt Input block (plain text)
+ * ct Output block (crypto text). Can overlap with pt
+ */
+static void
+rijndael_encrypt(const uint32_t rk[], int Nr, const uint32_t pt[4],
+ uint32_t ct[4])
+{
+ uint32_t s0, s1, s2, s3, t0, t1, t2, t3;
+ int r;
+
+ /*
+ * map byte array block to cipher state
+ * and add initial round key:
+ */
+
+ s0 = pt[0] ^ rk[0];
+ s1 = pt[1] ^ rk[1];
+ s2 = pt[2] ^ rk[2];
+ s3 = pt[3] ^ rk[3];
+
+ /*
+ * Nr - 1 full rounds:
+ */
+
+ r = Nr >> 1;
+
+ for (;;) {
+ t0 = Te0[s0 >> 24] ^
+ Te1[(s1 >> 16) & 0xff] ^
+ Te2[(s2 >> 8) & 0xff] ^
+ Te3[s3 & 0xff] ^
+ rk[4];
+
+ t1 = Te0[s1 >> 24] ^
+ Te1[(s2 >> 16) & 0xff] ^
+ Te2[(s3 >> 8) & 0xff] ^
+ Te3[s0 & 0xff] ^
+ rk[5];
+
+ t2 = Te0[s2 >> 24] ^
+ Te1[(s3 >> 16) & 0xff] ^
+ Te2[(s0 >> 8) & 0xff] ^
+ Te3[s1 & 0xff] ^
+ rk[6];
+
+ t3 = Te0[s3 >> 24] ^
+ Te1[(s0 >> 16) & 0xff] ^
+ Te2[(s1 >> 8) & 0xff] ^
+ Te3[s2 & 0xff] ^
+ rk[7];
+
+ rk += 8;
+
+ if (--r == 0) {
+ break;
+ }
+
+ s0 = Te0[t0 >> 24] ^
+ Te1[(t1 >> 16) & 0xff] ^
+ Te2[(t2 >> 8) & 0xff] ^
+ Te3[t3 & 0xff] ^
+ rk[0];
+
+ s1 = Te0[t1 >> 24] ^
+ Te1[(t2 >> 16) & 0xff] ^
+ Te2[(t3 >> 8) & 0xff] ^
+ Te3[t0 & 0xff] ^
+ rk[1];
+
+ s2 = Te0[t2 >> 24] ^
+ Te1[(t3 >> 16) & 0xff] ^
+ Te2[(t0 >> 8) & 0xff] ^
+ Te3[t1 & 0xff] ^
+ rk[2];
+
+ s3 = Te0[t3 >> 24] ^
+ Te1[(t0 >> 16) & 0xff] ^
+ Te2[(t1 >> 8) & 0xff] ^
+ Te3[t2 & 0xff] ^
+ rk[3];
+ }
+
+ /*
+ * apply last round and
+ * map cipher state to byte array block:
+ */
+
+ s0 = (Te4[(t0 >> 24)] & 0xff000000) ^
+ (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
+ (Te4[(t2 >> 8) & 0xff] & 0x0000ff00) ^
+ (Te4[t3 & 0xff] & 0x000000ff) ^
+ rk[0];
+ ct[0] = s0;
+
+ s1 = (Te4[(t1 >> 24)] & 0xff000000) ^
+ (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
+ (Te4[(t3 >> 8) & 0xff] & 0x0000ff00) ^
+ (Te4[t0 & 0xff] & 0x000000ff) ^
+ rk[1];
+ ct[1] = s1;
+
+ s2 = (Te4[(t2 >> 24)] & 0xff000000) ^
+ (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
+ (Te4[(t0 >> 8) & 0xff] & 0x0000ff00) ^
+ (Te4[t1 & 0xff] & 0x000000ff) ^
+ rk[2];
+ ct[2] = s2;
+
+ s3 = (Te4[(t3 >> 24)] & 0xff000000) ^
+ (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
+ (Te4[(t1 >> 8) & 0xff] & 0x0000ff00) ^
+ (Te4[t2 & 0xff] & 0x000000ff) ^
+ rk[3];
+ ct[3] = s3;
+}
+
+
+/*
+ * Decrypt one block of data. The block is assumed to be an array
+ * of four uint32_t values, so copy for alignment (and byte-order
+ * reversal for little endian systems might be necessary on the
+ * input and output byte streams.
+ * The size of the key schedule depends on the number of rounds
+ * (which can be computed from the size of the key), i.e. 4*(Nr + 1).
+ *
+ * Parameters:
+ * rk Key schedule, of aes_ks_t (60 32-bit integers)
+ * Nr Number of rounds
+ * ct Input block (crypto text)
+ * pt Output block (plain text). Can overlap with pt
+ */
+static void
+rijndael_decrypt(const uint32_t rk[], int Nr, const uint32_t ct[4],
+ uint32_t pt[4])
+{
+ uint32_t s0, s1, s2, s3, t0, t1, t2, t3;
+ int r;
+
+ /*
+ * map byte array block to cipher state
+ * and add initial round key:
+ */
+ s0 = ct[0] ^ rk[0];
+ s1 = ct[1] ^ rk[1];
+ s2 = ct[2] ^ rk[2];
+ s3 = ct[3] ^ rk[3];
+
+ /*
+ * Nr - 1 full rounds:
+ */
+
+ r = Nr >> 1;
+
+ for (;;) {
+ t0 = Td0[s0 >> 24] ^
+ Td1[(s3 >> 16) & 0xff] ^
+ Td2[(s2 >> 8) & 0xff] ^
+ Td3[s1 & 0xff] ^
+ rk[4];
+
+ t1 = Td0[s1 >> 24] ^
+ Td1[(s0 >> 16) & 0xff] ^
+ Td2[(s3 >> 8) & 0xff] ^
+ Td3[s2 & 0xff] ^
+ rk[5];
+
+ t2 = Td0[s2 >> 24] ^
+ Td1[(s1 >> 16) & 0xff] ^
+ Td2[(s0 >> 8) & 0xff] ^
+ Td3[s3 & 0xff] ^
+ rk[6];
+
+ t3 = Td0[s3 >> 24] ^
+ Td1[(s2 >> 16) & 0xff] ^
+ Td2[(s1 >> 8) & 0xff] ^
+ Td3[s0 & 0xff] ^
+ rk[7];
+
+ rk += 8;
+
+ if (--r == 0) {
+ break;
+ }
+
+ s0 = Td0[t0 >> 24] ^
+ Td1[(t3 >> 16) & 0xff] ^
+ Td2[(t2 >> 8) & 0xff] ^
+ Td3[t1 & 0xff] ^
+ rk[0];
+
+ s1 = Td0[t1 >> 24] ^
+ Td1[(t0 >> 16) & 0xff] ^
+ Td2[(t3 >> 8) & 0xff] ^
+ Td3[t2 & 0xff] ^
+ rk[1];
+
+ s2 = Td0[t2 >> 24] ^
+ Td1[(t1 >> 16) & 0xff] ^
+ Td2[(t0 >> 8) & 0xff] ^
+ Td3[t3 & 0xff] ^
+ rk[2];
+
+ s3 = Td0[t3 >> 24] ^
+ Td1[(t2 >> 16) & 0xff] ^
+ Td2[(t1 >> 8) & 0xff] ^
+ Td3[t0 & 0xff] ^
+ rk[3];
+ }
+
+ /*
+ * apply last round and
+ * map cipher state to byte array block:
+ */
+
+ s0 = (Td4[t0 >> 24] & 0xff000000) ^
+ (Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
+ (Td4[(t2 >> 8) & 0xff] & 0x0000ff00) ^
+ (Td4[t1 & 0xff] & 0x000000ff) ^
+ rk[0];
+ pt[0] = s0;
+
+ s1 = (Td4[t1 >> 24] & 0xff000000) ^
+ (Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
+ (Td4[(t3 >> 8) & 0xff] & 0x0000ff00) ^
+ (Td4[t2 & 0xff] & 0x000000ff) ^
+ rk[1];
+ pt[1] = s1;
+
+ s2 = (Td4[t2 >> 24] & 0xff000000) ^
+ (Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
+ (Td4[(t0 >> 8) & 0xff] & 0x0000ff00) ^
+ (Td4[t3 & 0xff] & 0x000000ff) ^
+ rk[2];
+ pt[2] = s2;
+
+ s3 = (Td4[t3 >> 24] & 0xff000000) ^
+ (Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
+ (Td4[(t1 >> 8) & 0xff] & 0x0000ff00) ^
+ (Td4[t0 & 0xff] & 0x000000ff) ^
+ rk[3];
+ pt[3] = s3;
+}
+#endif /* __amd64 */
+
+
+/*
+ * Initialize AES encryption and decryption key schedules.
+ *
+ * Parameters:
+ * cipherKey User key
+ * keyBits AES key size (128, 192, or 256 bits)
+ * keysched AES key schedule to be initialized, of type aes_key_t.
+ * Allocated by aes_alloc_keysched().
+ */
+void
+aes_init_keysched(const uint8_t *cipherKey, uint_t keyBits, void *keysched)
+{
+ aes_key_t *newbie = keysched;
+ uint_t keysize, i, j;
+ union {
+ uint64_t ka64[4];
+ uint32_t ka32[8];
+ } keyarr;
+
+ switch (keyBits) {
+ case 128:
+ newbie->nr = 10;
+ break;
+
+ case 192:
+ newbie->nr = 12;
+ break;
+
+ case 256:
+ newbie->nr = 14;
+ break;
+
+ default:
+ /* should never get here */
+ return;
+ }
+ keysize = CRYPTO_BITS2BYTES(keyBits);
+
+ /*
+ * For _LITTLE_ENDIAN machines (except AMD64), reverse every
+ * 4 bytes in the key. On _BIG_ENDIAN and AMD64, copy the key
+ * without reversing bytes.
+ * For AMD64, do not byte swap for aes_setupkeys().
+ *
+ * SPARCv8/v9 uses a key schedule array with 64-bit elements.
+ * X86/AMD64 uses a key schedule array with 32-bit elements.
+ */
+#ifndef AES_BYTE_SWAP
+ if (IS_P2ALIGNED(cipherKey, sizeof (uint64_t))) {
+ for (i = 0, j = 0; j < keysize; i++, j += 8) {
+ /* LINTED: pointer alignment */
+ keyarr.ka64[i] = *((uint64_t *)&cipherKey[j]);
+ }
+ } else {
+ bcopy(cipherKey, keyarr.ka32, keysize);
+ }
+
+#else /* byte swap */
+ for (i = 0, j = 0; j < keysize; i++, j += 4) {
+ keyarr.ka32[i] = htonl(*(uint32_t *)(void *)&cipherKey[j]);
+ }
+#endif
+
+ aes_setupkeys(newbie, keyarr.ka32, keyBits);
+}
+
+
+/*
+ * Encrypt one block using AES.
+ * Align if needed and (for x86 32-bit only) byte-swap.
+ *
+ * Parameters:
+ * ks Key schedule, of type aes_key_t
+ * pt Input block (plain text)
+ * ct Output block (crypto text). Can overlap with pt
+ */
+int
+aes_encrypt_block(const void *ks, const uint8_t *pt, uint8_t *ct)
+{
+ aes_key_t *ksch = (aes_key_t *)ks;
+
+#ifndef AES_BYTE_SWAP
+ if (IS_P2ALIGNED2(pt, ct, sizeof (uint32_t))) {
+ /* LINTED: pointer alignment */
+ AES_ENCRYPT_IMPL(&ksch->encr_ks.ks32[0], ksch->nr,
+ /* LINTED: pointer alignment */
+ (uint32_t *)pt, (uint32_t *)ct, ksch->flags);
+ } else {
+#endif
+ uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)];
+
+ /* Copy input block into buffer */
+#ifndef AES_BYTE_SWAP
+ bcopy(pt, &buffer, AES_BLOCK_LEN);
+
+#else /* byte swap */
+ buffer[0] = htonl(*(uint32_t *)(void *)&pt[0]);
+ buffer[1] = htonl(*(uint32_t *)(void *)&pt[4]);
+ buffer[2] = htonl(*(uint32_t *)(void *)&pt[8]);
+ buffer[3] = htonl(*(uint32_t *)(void *)&pt[12]);
+#endif
+
+ AES_ENCRYPT_IMPL(&ksch->encr_ks.ks32[0], ksch->nr,
+ buffer, buffer, ksch->flags);
+
+ /* Copy result from buffer to output block */
+#ifndef AES_BYTE_SWAP
+ bcopy(&buffer, ct, AES_BLOCK_LEN);
+ }
+
+#else /* byte swap */
+ *(uint32_t *)(void *)&ct[0] = htonl(buffer[0]);
+ *(uint32_t *)(void *)&ct[4] = htonl(buffer[1]);
+ *(uint32_t *)(void *)&ct[8] = htonl(buffer[2]);
+ *(uint32_t *)(void *)&ct[12] = htonl(buffer[3]);
+#endif
+ return (CRYPTO_SUCCESS);
+}
+
+
+/*
+ * Decrypt one block using AES.
+ * Align and byte-swap if needed.
+ *
+ * Parameters:
+ * ks Key schedule, of type aes_key_t
+ * ct Input block (crypto text)
+ * pt Output block (plain text). Can overlap with pt
+ */
+int
+aes_decrypt_block(const void *ks, const uint8_t *ct, uint8_t *pt)
+{
+ aes_key_t *ksch = (aes_key_t *)ks;
+
+#ifndef AES_BYTE_SWAP
+ if (IS_P2ALIGNED2(ct, pt, sizeof (uint32_t))) {
+ /* LINTED: pointer alignment */
+ AES_DECRYPT_IMPL(&ksch->decr_ks.ks32[0], ksch->nr,
+ /* LINTED: pointer alignment */
+ (uint32_t *)ct, (uint32_t *)pt, ksch->flags);
+ } else {
+#endif
+ uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)];
+
+ /* Copy input block into buffer */
+#ifndef AES_BYTE_SWAP
+ bcopy(ct, &buffer, AES_BLOCK_LEN);
+
+#else /* byte swap */
+ buffer[0] = htonl(*(uint32_t *)(void *)&ct[0]);
+ buffer[1] = htonl(*(uint32_t *)(void *)&ct[4]);
+ buffer[2] = htonl(*(uint32_t *)(void *)&ct[8]);
+ buffer[3] = htonl(*(uint32_t *)(void *)&ct[12]);
+#endif
+
+ AES_DECRYPT_IMPL(&ksch->decr_ks.ks32[0], ksch->nr,
+ buffer, buffer, ksch->flags);
+
+ /* Copy result from buffer to output block */
+#ifndef AES_BYTE_SWAP
+ bcopy(&buffer, pt, AES_BLOCK_LEN);
+ }
+
+#else /* byte swap */
+ *(uint32_t *)(void *)&pt[0] = htonl(buffer[0]);
+ *(uint32_t *)(void *)&pt[4] = htonl(buffer[1]);
+ *(uint32_t *)(void *)&pt[8] = htonl(buffer[2]);
+ *(uint32_t *)(void *)&pt[12] = htonl(buffer[3]);
+#endif
+
+ return (CRYPTO_SUCCESS);
+}
+
+
+/*
+ * Allocate key schedule for AES.
+ *
+ * Return the pointer and set size to the number of bytes allocated.
+ * Memory allocated must be freed by the caller when done.
+ *
+ * Parameters:
+ * size Size of key schedule allocated, in bytes
+ * kmflag Flag passed to kmem_alloc(9F); ignored in userland.
+ */
+/* ARGSUSED */
+void *
+aes_alloc_keysched(size_t *size, int kmflag)
+{
+ aes_key_t *keysched;
+
+ keysched = (aes_key_t *)kmem_alloc(sizeof (aes_key_t), kmflag);
+ if (keysched != NULL) {
+ *size = sizeof (aes_key_t);
+ return (keysched);
+ }
+ return (NULL);
+}
+
+
+#ifdef __amd64
+
+#define INTEL_AESNI_FLAG (1 << 25)
+
+/*
+ * Return 1 if executing on Intel with AES-NI instructions,
+ * otherwise 0 (i.e., Intel without AES-NI or AMD64).
+ * Cache the result, as the CPU can't change.
+ */
+static int
+intel_aes_instructions_present(void)
+{
+ static int cached_result = -1;
+ unsigned eax, ebx, ecx, edx;
+ unsigned func, subfunc;
+
+ if (cached_result == -1) { /* first time */
+ /* check for an intel cpu */
+ func = 0;
+ subfunc = 0;
+
+ __asm__ __volatile__(
+ "cpuid"
+ : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
+ : "a"(func), "c"(subfunc));
+
+ if (memcmp((char *) (&ebx), "Genu", 4) == 0 &&
+ memcmp((char *) (&edx), "ineI", 4) == 0 &&
+ memcmp((char *) (&ecx), "ntel", 4) == 0) {
+
+ func = 1;
+ subfunc = 0;
+
+ /* check for aes-ni instruction set */
+ __asm__ __volatile__(
+ "cpuid"
+ : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
+ : "a"(func), "c"(subfunc));
+
+ cached_result = !!(ecx & INTEL_AESNI_FLAG);
+ } else {
+ cached_result = 0;
+ }
+ }
+
+ return (cached_result);
+}
+
+#endif /* __amd64 */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <aes/aes_impl.h>
+
+/* Copy a 16-byte AES block from "in" to "out" */
+void
+aes_copy_block(uint8_t *in, uint8_t *out)
+{
+ if (IS_P2ALIGNED2(in, out, sizeof (uint32_t))) {
+ /* LINTED: pointer alignment */
+ *(uint32_t *)&out[0] = *(uint32_t *)&in[0];
+ /* LINTED: pointer alignment */
+ *(uint32_t *)&out[4] = *(uint32_t *)&in[4];
+ /* LINTED: pointer alignment */
+ *(uint32_t *)&out[8] = *(uint32_t *)&in[8];
+ /* LINTED: pointer alignment */
+ *(uint32_t *)&out[12] = *(uint32_t *)&in[12];
+ } else {
+ AES_COPY_BLOCK(in, out);
+ }
+}
+
+
+/* XOR a 16-byte AES block of data into dst */
+void
+aes_xor_block(uint8_t *data, uint8_t *dst)
+{
+ if (IS_P2ALIGNED2(dst, data, sizeof (uint32_t))) {
+ /* LINTED: pointer alignment */
+ *(uint32_t *)&dst[0] ^= *(uint32_t *)&data[0];
+ /* LINTED: pointer alignment */
+ *(uint32_t *)&dst[4] ^= *(uint32_t *)&data[4];
+ /* LINTED: pointer alignment */
+ *(uint32_t *)&dst[8] ^= *(uint32_t *)&data[8];
+ /* LINTED: pointer alignment */
+ *(uint32_t *)&dst[12] ^= *(uint32_t *)&data[12];
+ } else {
+ AES_XOR_BLOCK(data, dst);
+ }
+}
+
+
+/*
+ * Encrypt multiple blocks of data according to mode.
+ */
+int
+aes_encrypt_contiguous_blocks(void *ctx, char *data, size_t length,
+ crypto_data_t *out)
+{
+ aes_ctx_t *aes_ctx = ctx;
+ int rv;
+
+ if (aes_ctx->ac_flags & CTR_MODE) {
+ rv = ctr_mode_contiguous_blocks(ctx, data, length, out,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+ } else if (aes_ctx->ac_flags & CCM_MODE) {
+ rv = ccm_mode_encrypt_contiguous_blocks(ctx, data, length,
+ out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+ aes_xor_block);
+ } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
+ rv = gcm_mode_encrypt_contiguous_blocks(ctx, data, length,
+ out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+ aes_xor_block);
+ } else if (aes_ctx->ac_flags & CBC_MODE) {
+ rv = cbc_encrypt_contiguous_blocks(ctx,
+ data, length, out, AES_BLOCK_LEN, aes_encrypt_block,
+ aes_copy_block, aes_xor_block);
+ } else {
+ rv = ecb_cipher_contiguous_blocks(ctx, data, length, out,
+ AES_BLOCK_LEN, aes_encrypt_block);
+ }
+ return (rv);
+}
+
+
+/*
+ * Decrypt multiple blocks of data according to mode.
+ */
+int
+aes_decrypt_contiguous_blocks(void *ctx, char *data, size_t length,
+ crypto_data_t *out)
+{
+ aes_ctx_t *aes_ctx = ctx;
+ int rv;
+
+ if (aes_ctx->ac_flags & CTR_MODE) {
+ rv = ctr_mode_contiguous_blocks(ctx, data, length, out,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+ if (rv == CRYPTO_DATA_LEN_RANGE)
+ rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
+ } else if (aes_ctx->ac_flags & CCM_MODE) {
+ rv = ccm_mode_decrypt_contiguous_blocks(ctx, data, length,
+ out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+ aes_xor_block);
+ } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
+ rv = gcm_mode_decrypt_contiguous_blocks(ctx, data, length,
+ out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+ aes_xor_block);
+ } else if (aes_ctx->ac_flags & CBC_MODE) {
+ rv = cbc_decrypt_contiguous_blocks(ctx, data, length, out,
+ AES_BLOCK_LEN, aes_decrypt_block, aes_copy_block,
+ aes_xor_block);
+ } else {
+ rv = ecb_cipher_contiguous_blocks(ctx, data, length, out,
+ AES_BLOCK_LEN, aes_decrypt_block);
+ if (rv == CRYPTO_DATA_LEN_RANGE)
+ rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
+ }
+ return (rv);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+
+/*
+ * Algorithm independent CBC functions.
+ */
+int
+cbc_encrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length,
+ crypto_data_t *out, size_t block_size,
+ int (*encrypt)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ size_t remainder = length;
+ size_t need = 0;
+ uint8_t *datap = (uint8_t *)data;
+ uint8_t *blockp;
+ uint8_t *lastp;
+ void *iov_or_mp;
+ offset_t offset;
+ uint8_t *out_data_1;
+ uint8_t *out_data_2;
+ size_t out_data_1_len;
+
+ if (length + ctx->cbc_remainder_len < block_size) {
+ /* accumulate bytes here and return */
+ bcopy(datap,
+ (uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len,
+ length);
+ ctx->cbc_remainder_len += length;
+ ctx->cbc_copy_to = datap;
+ return (CRYPTO_SUCCESS);
+ }
+
+ lastp = (uint8_t *)ctx->cbc_iv;
+ if (out != NULL)
+ crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+ do {
+ /* Unprocessed data from last call. */
+ if (ctx->cbc_remainder_len > 0) {
+ need = block_size - ctx->cbc_remainder_len;
+
+ if (need > remainder)
+ return (CRYPTO_DATA_LEN_RANGE);
+
+ bcopy(datap, &((uint8_t *)ctx->cbc_remainder)
+ [ctx->cbc_remainder_len], need);
+
+ blockp = (uint8_t *)ctx->cbc_remainder;
+ } else {
+ blockp = datap;
+ }
+
+ if (out == NULL) {
+ /*
+ * XOR the previous cipher block or IV with the
+ * current clear block.
+ */
+ xor_block(lastp, blockp);
+ encrypt(ctx->cbc_keysched, blockp, blockp);
+
+ ctx->cbc_lastp = blockp;
+ lastp = blockp;
+
+ if (ctx->cbc_remainder_len > 0) {
+ bcopy(blockp, ctx->cbc_copy_to,
+ ctx->cbc_remainder_len);
+ bcopy(blockp + ctx->cbc_remainder_len, datap,
+ need);
+ }
+ } else {
+ /*
+ * XOR the previous cipher block or IV with the
+ * current clear block.
+ */
+ xor_block(blockp, lastp);
+ encrypt(ctx->cbc_keysched, lastp, lastp);
+ crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+ &out_data_1_len, &out_data_2, block_size);
+
+ /* copy block to where it belongs */
+ if (out_data_1_len == block_size) {
+ copy_block(lastp, out_data_1);
+ } else {
+ bcopy(lastp, out_data_1, out_data_1_len);
+ if (out_data_2 != NULL) {
+ bcopy(lastp + out_data_1_len,
+ out_data_2,
+ block_size - out_data_1_len);
+ }
+ }
+ /* update offset */
+ out->cd_offset += block_size;
+ }
+
+ /* Update pointer to next block of data to be processed. */
+ if (ctx->cbc_remainder_len != 0) {
+ datap += need;
+ ctx->cbc_remainder_len = 0;
+ } else {
+ datap += block_size;
+ }
+
+ remainder = (size_t)&data[length] - (size_t)datap;
+
+ /* Incomplete last block. */
+ if (remainder > 0 && remainder < block_size) {
+ bcopy(datap, ctx->cbc_remainder, remainder);
+ ctx->cbc_remainder_len = remainder;
+ ctx->cbc_copy_to = datap;
+ goto out;
+ }
+ ctx->cbc_copy_to = NULL;
+
+ } while (remainder > 0);
+
+out:
+ /*
+ * Save the last encrypted block in the context.
+ */
+ if (ctx->cbc_lastp != NULL) {
+ copy_block((uint8_t *)ctx->cbc_lastp, (uint8_t *)ctx->cbc_iv);
+ ctx->cbc_lastp = (uint8_t *)ctx->cbc_iv;
+ }
+
+ return (CRYPTO_SUCCESS);
+}
+
+#define OTHER(a, ctx) \
+ (((a) == (ctx)->cbc_lastblock) ? (ctx)->cbc_iv : (ctx)->cbc_lastblock)
+
+/* ARGSUSED */
+int
+cbc_decrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length,
+ crypto_data_t *out, size_t block_size,
+ int (*decrypt)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ size_t remainder = length;
+ size_t need = 0;
+ uint8_t *datap = (uint8_t *)data;
+ uint8_t *blockp;
+ uint8_t *lastp;
+ void *iov_or_mp;
+ offset_t offset;
+ uint8_t *out_data_1;
+ uint8_t *out_data_2;
+ size_t out_data_1_len;
+
+ if (length + ctx->cbc_remainder_len < block_size) {
+ /* accumulate bytes here and return */
+ bcopy(datap,
+ (uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len,
+ length);
+ ctx->cbc_remainder_len += length;
+ ctx->cbc_copy_to = datap;
+ return (CRYPTO_SUCCESS);
+ }
+
+ lastp = ctx->cbc_lastp;
+ if (out != NULL)
+ crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+ do {
+ /* Unprocessed data from last call. */
+ if (ctx->cbc_remainder_len > 0) {
+ need = block_size - ctx->cbc_remainder_len;
+
+ if (need > remainder)
+ return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+
+ bcopy(datap, &((uint8_t *)ctx->cbc_remainder)
+ [ctx->cbc_remainder_len], need);
+
+ blockp = (uint8_t *)ctx->cbc_remainder;
+ } else {
+ blockp = datap;
+ }
+
+ /* LINTED: pointer alignment */
+ copy_block(blockp, (uint8_t *)OTHER((uint64_t *)lastp, ctx));
+
+ if (out != NULL) {
+ decrypt(ctx->cbc_keysched, blockp,
+ (uint8_t *)ctx->cbc_remainder);
+ blockp = (uint8_t *)ctx->cbc_remainder;
+ } else {
+ decrypt(ctx->cbc_keysched, blockp, blockp);
+ }
+
+ /*
+ * XOR the previous cipher block or IV with the
+ * currently decrypted block.
+ */
+ xor_block(lastp, blockp);
+
+ /* LINTED: pointer alignment */
+ lastp = (uint8_t *)OTHER((uint64_t *)lastp, ctx);
+
+ if (out != NULL) {
+ crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+ &out_data_1_len, &out_data_2, block_size);
+
+ bcopy(blockp, out_data_1, out_data_1_len);
+ if (out_data_2 != NULL) {
+ bcopy(blockp + out_data_1_len, out_data_2,
+ block_size - out_data_1_len);
+ }
+
+ /* update offset */
+ out->cd_offset += block_size;
+
+ } else if (ctx->cbc_remainder_len > 0) {
+ /* copy temporary block to where it belongs */
+ bcopy(blockp, ctx->cbc_copy_to, ctx->cbc_remainder_len);
+ bcopy(blockp + ctx->cbc_remainder_len, datap, need);
+ }
+
+ /* Update pointer to next block of data to be processed. */
+ if (ctx->cbc_remainder_len != 0) {
+ datap += need;
+ ctx->cbc_remainder_len = 0;
+ } else {
+ datap += block_size;
+ }
+
+ remainder = (size_t)&data[length] - (size_t)datap;
+
+ /* Incomplete last block. */
+ if (remainder > 0 && remainder < block_size) {
+ bcopy(datap, ctx->cbc_remainder, remainder);
+ ctx->cbc_remainder_len = remainder;
+ ctx->cbc_lastp = lastp;
+ ctx->cbc_copy_to = datap;
+ return (CRYPTO_SUCCESS);
+ }
+ ctx->cbc_copy_to = NULL;
+
+ } while (remainder > 0);
+
+ ctx->cbc_lastp = lastp;
+ return (CRYPTO_SUCCESS);
+}
+
+int
+cbc_init_ctx(cbc_ctx_t *cbc_ctx, char *param, size_t param_len,
+ size_t block_size, void (*copy_block)(uint8_t *, uint64_t *))
+{
+ /*
+ * Copy IV into context.
+ *
+ * If cm_param == NULL then the IV comes from the
+ * cd_miscdata field in the crypto_data structure.
+ */
+ if (param != NULL) {
+ ASSERT(param_len == block_size);
+ copy_block((uchar_t *)param, cbc_ctx->cbc_iv);
+ }
+
+ cbc_ctx->cbc_lastp = (uint8_t *)&cbc_ctx->cbc_iv[0];
+ cbc_ctx->cbc_flags |= CBC_MODE;
+ return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+void *
+cbc_alloc_ctx(int kmflag)
+{
+ cbc_ctx_t *cbc_ctx;
+
+ if ((cbc_ctx = kmem_zalloc(sizeof (cbc_ctx_t), kmflag)) == NULL)
+ return (NULL);
+
+ cbc_ctx->cbc_flags = CBC_MODE;
+ return (cbc_ctx);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+
+#if defined(__i386) || defined(__amd64)
+#include <sys/byteorder.h>
+#define UNALIGNED_POINTERS_PERMITTED
+#endif
+
+/*
+ * Encrypt multiple blocks of data in CCM mode. Decrypt for CCM mode
+ * is done in another function.
+ */
+int
+ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
+ crypto_data_t *out, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ size_t remainder = length;
+ size_t need = 0;
+ uint8_t *datap = (uint8_t *)data;
+ uint8_t *blockp;
+ uint8_t *lastp;
+ void *iov_or_mp;
+ offset_t offset;
+ uint8_t *out_data_1;
+ uint8_t *out_data_2;
+ size_t out_data_1_len;
+ uint64_t counter;
+ uint8_t *mac_buf;
+
+ if (length + ctx->ccm_remainder_len < block_size) {
+ /* accumulate bytes here and return */
+ bcopy(datap,
+ (uint8_t *)ctx->ccm_remainder + ctx->ccm_remainder_len,
+ length);
+ ctx->ccm_remainder_len += length;
+ ctx->ccm_copy_to = datap;
+ return (CRYPTO_SUCCESS);
+ }
+
+ lastp = (uint8_t *)ctx->ccm_cb;
+ if (out != NULL)
+ crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+ mac_buf = (uint8_t *)ctx->ccm_mac_buf;
+
+ do {
+ /* Unprocessed data from last call. */
+ if (ctx->ccm_remainder_len > 0) {
+ need = block_size - ctx->ccm_remainder_len;
+
+ if (need > remainder)
+ return (CRYPTO_DATA_LEN_RANGE);
+
+ bcopy(datap, &((uint8_t *)ctx->ccm_remainder)
+ [ctx->ccm_remainder_len], need);
+
+ blockp = (uint8_t *)ctx->ccm_remainder;
+ } else {
+ blockp = datap;
+ }
+
+ /*
+ * do CBC MAC
+ *
+ * XOR the previous cipher block current clear block.
+ * mac_buf always contain previous cipher block.
+ */
+ xor_block(blockp, mac_buf);
+ encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+
+ /* ccm_cb is the counter block */
+ encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb,
+ (uint8_t *)ctx->ccm_tmp);
+
+ lastp = (uint8_t *)ctx->ccm_tmp;
+
+ /*
+ * Increment counter. Counter bits are confined
+ * to the bottom 64 bits of the counter block.
+ */
+#ifdef _LITTLE_ENDIAN
+ counter = ntohll(ctx->ccm_cb[1] & ctx->ccm_counter_mask);
+ counter = htonll(counter + 1);
+#else
+ counter = ctx->ccm_cb[1] & ctx->ccm_counter_mask;
+ counter++;
+#endif /* _LITTLE_ENDIAN */
+ counter &= ctx->ccm_counter_mask;
+ ctx->ccm_cb[1] =
+ (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;
+
+ /*
+ * XOR encrypted counter block with the current clear block.
+ */
+ xor_block(blockp, lastp);
+
+ ctx->ccm_processed_data_len += block_size;
+
+ if (out == NULL) {
+ if (ctx->ccm_remainder_len > 0) {
+ bcopy(blockp, ctx->ccm_copy_to,
+ ctx->ccm_remainder_len);
+ bcopy(blockp + ctx->ccm_remainder_len, datap,
+ need);
+ }
+ } else {
+ crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+ &out_data_1_len, &out_data_2, block_size);
+
+ /* copy block to where it belongs */
+ if (out_data_1_len == block_size) {
+ copy_block(lastp, out_data_1);
+ } else {
+ bcopy(lastp, out_data_1, out_data_1_len);
+ if (out_data_2 != NULL) {
+ bcopy(lastp + out_data_1_len,
+ out_data_2,
+ block_size - out_data_1_len);
+ }
+ }
+ /* update offset */
+ out->cd_offset += block_size;
+ }
+
+ /* Update pointer to next block of data to be processed. */
+ if (ctx->ccm_remainder_len != 0) {
+ datap += need;
+ ctx->ccm_remainder_len = 0;
+ } else {
+ datap += block_size;
+ }
+
+ remainder = (size_t)&data[length] - (size_t)datap;
+
+ /* Incomplete last block. */
+ if (remainder > 0 && remainder < block_size) {
+ bcopy(datap, ctx->ccm_remainder, remainder);
+ ctx->ccm_remainder_len = remainder;
+ ctx->ccm_copy_to = datap;
+ goto out;
+ }
+ ctx->ccm_copy_to = NULL;
+
+ } while (remainder > 0);
+
+out:
+ return (CRYPTO_SUCCESS);
+}
+
+void
+calculate_ccm_mac(ccm_ctx_t *ctx, uint8_t *ccm_mac,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *))
+{
+ uint64_t counter;
+ uint8_t *counterp, *mac_buf;
+ int i;
+
+ mac_buf = (uint8_t *)ctx->ccm_mac_buf;
+
+ /* first counter block start with index 0 */
+ counter = 0;
+ ctx->ccm_cb[1] = (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;
+
+ counterp = (uint8_t *)ctx->ccm_tmp;
+ encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp);
+
+ /* calculate XOR of MAC with first counter block */
+ for (i = 0; i < ctx->ccm_mac_len; i++) {
+ ccm_mac[i] = mac_buf[i] ^ counterp[i];
+ }
+}
+
+/* ARGSUSED */
+int
+ccm_encrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ uint8_t *lastp, *mac_buf, *ccm_mac_p, *macp = NULL;
+ void *iov_or_mp;
+ offset_t offset;
+ uint8_t *out_data_1;
+ uint8_t *out_data_2;
+ size_t out_data_1_len;
+ int i;
+
+ if (out->cd_length < (ctx->ccm_remainder_len + ctx->ccm_mac_len)) {
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ /*
+ * When we get here, the number of bytes of payload processed
+ * plus whatever data remains, if any,
+ * should be the same as the number of bytes that's being
+ * passed in the argument during init time.
+ */
+ if ((ctx->ccm_processed_data_len + ctx->ccm_remainder_len)
+ != (ctx->ccm_data_len)) {
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ mac_buf = (uint8_t *)ctx->ccm_mac_buf;
+
+ if (ctx->ccm_remainder_len > 0) {
+
+ /* ccm_mac_input_buf is not used for encryption */
+ macp = (uint8_t *)ctx->ccm_mac_input_buf;
+ bzero(macp, block_size);
+
+ /* copy remainder to temporary buffer */
+ bcopy(ctx->ccm_remainder, macp, ctx->ccm_remainder_len);
+
+ /* calculate the CBC MAC */
+ xor_block(macp, mac_buf);
+ encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+
+ /* calculate the counter mode */
+ lastp = (uint8_t *)ctx->ccm_tmp;
+ encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, lastp);
+
+ /* XOR with counter block */
+ for (i = 0; i < ctx->ccm_remainder_len; i++) {
+ macp[i] ^= lastp[i];
+ }
+ ctx->ccm_processed_data_len += ctx->ccm_remainder_len;
+ }
+
+ /* Calculate the CCM MAC */
+ ccm_mac_p = (uint8_t *)ctx->ccm_tmp;
+ calculate_ccm_mac(ctx, ccm_mac_p, encrypt_block);
+
+ crypto_init_ptrs(out, &iov_or_mp, &offset);
+ crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+ &out_data_1_len, &out_data_2,
+ ctx->ccm_remainder_len + ctx->ccm_mac_len);
+
+ if (ctx->ccm_remainder_len > 0) {
+
+ /* copy temporary block to where it belongs */
+ if (out_data_2 == NULL) {
+ /* everything will fit in out_data_1 */
+ bcopy(macp, out_data_1, ctx->ccm_remainder_len);
+ bcopy(ccm_mac_p, out_data_1 + ctx->ccm_remainder_len,
+ ctx->ccm_mac_len);
+ } else {
+
+ if (out_data_1_len < ctx->ccm_remainder_len) {
+
+ size_t data_2_len_used;
+
+ bcopy(macp, out_data_1, out_data_1_len);
+
+ data_2_len_used = ctx->ccm_remainder_len
+ - out_data_1_len;
+
+ bcopy((uint8_t *)macp + out_data_1_len,
+ out_data_2, data_2_len_used);
+ bcopy(ccm_mac_p, out_data_2 + data_2_len_used,
+ ctx->ccm_mac_len);
+ } else {
+ bcopy(macp, out_data_1, out_data_1_len);
+ if (out_data_1_len == ctx->ccm_remainder_len) {
+ /* mac will be in out_data_2 */
+ bcopy(ccm_mac_p, out_data_2,
+ ctx->ccm_mac_len);
+ } else {
+ size_t len_not_used = out_data_1_len -
+ ctx->ccm_remainder_len;
+ /*
+ * part of mac in will be in
+ * out_data_1, part of the mac will be
+ * in out_data_2
+ */
+ bcopy(ccm_mac_p,
+ out_data_1 + ctx->ccm_remainder_len,
+ len_not_used);
+ bcopy(ccm_mac_p + len_not_used,
+ out_data_2,
+ ctx->ccm_mac_len - len_not_used);
+
+ }
+ }
+ }
+ } else {
+ /* copy block to where it belongs */
+ bcopy(ccm_mac_p, out_data_1, out_data_1_len);
+ if (out_data_2 != NULL) {
+ bcopy(ccm_mac_p + out_data_1_len, out_data_2,
+ block_size - out_data_1_len);
+ }
+ }
+ out->cd_offset += ctx->ccm_remainder_len + ctx->ccm_mac_len;
+ ctx->ccm_remainder_len = 0;
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * This will only deal with decrypting the last block of the input that
+ * might not be a multiple of block length.
+ */
+void
+ccm_decrypt_incomplete_block(ccm_ctx_t *ctx,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *))
+{
+ uint8_t *datap, *outp, *counterp;
+ int i;
+
+ datap = (uint8_t *)ctx->ccm_remainder;
+ outp = &((ctx->ccm_pt_buf)[ctx->ccm_processed_data_len]);
+
+ counterp = (uint8_t *)ctx->ccm_tmp;
+ encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp);
+
+ /* XOR with counter block */
+ for (i = 0; i < ctx->ccm_remainder_len; i++) {
+ outp[i] = datap[i] ^ counterp[i];
+ }
+}
+
+/*
+ * This will decrypt the cipher text. However, the plaintext won't be
+ * returned to the caller. It will be returned when decrypt_final() is
+ * called if the MAC matches
+ */
+/* ARGSUSED */
+int
+ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
+ crypto_data_t *out, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ size_t remainder = length;
+ size_t need = 0;
+ uint8_t *datap = (uint8_t *)data;
+ uint8_t *blockp;
+ uint8_t *cbp;
+ uint64_t counter;
+ size_t pt_len, total_decrypted_len, mac_len, pm_len, pd_len;
+ uint8_t *resultp;
+
+
+ pm_len = ctx->ccm_processed_mac_len;
+
+ if (pm_len > 0) {
+ uint8_t *tmp;
+ /*
+ * all ciphertext has been processed, just waiting for
+ * part of the value of the mac
+ */
+ if ((pm_len + length) > ctx->ccm_mac_len) {
+ return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+ }
+ tmp = (uint8_t *)ctx->ccm_mac_input_buf;
+
+ bcopy(datap, tmp + pm_len, length);
+
+ ctx->ccm_processed_mac_len += length;
+ return (CRYPTO_SUCCESS);
+ }
+
+ /*
+ * If we decrypt the given data, what total amount of data would
+ * have been decrypted?
+ */
+ pd_len = ctx->ccm_processed_data_len;
+ total_decrypted_len = pd_len + length + ctx->ccm_remainder_len;
+
+ if (total_decrypted_len >
+ (ctx->ccm_data_len + ctx->ccm_mac_len)) {
+ return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+ }
+
+ pt_len = ctx->ccm_data_len;
+
+ if (total_decrypted_len > pt_len) {
+ /*
+ * part of the input will be the MAC, need to isolate that
+ * to be dealt with later. The left-over data in
+ * ccm_remainder_len from last time will not be part of the
+ * MAC. Otherwise, it would have already been taken out
+ * when this call is made last time.
+ */
+ size_t pt_part = pt_len - pd_len - ctx->ccm_remainder_len;
+
+ mac_len = length - pt_part;
+
+ ctx->ccm_processed_mac_len = mac_len;
+ bcopy(data + pt_part, ctx->ccm_mac_input_buf, mac_len);
+
+ if (pt_part + ctx->ccm_remainder_len < block_size) {
+ /*
+ * since this is last of the ciphertext, will
+ * just decrypt with it here
+ */
+ bcopy(datap, &((uint8_t *)ctx->ccm_remainder)
+ [ctx->ccm_remainder_len], pt_part);
+ ctx->ccm_remainder_len += pt_part;
+ ccm_decrypt_incomplete_block(ctx, encrypt_block);
+ ctx->ccm_processed_data_len += ctx->ccm_remainder_len;
+ ctx->ccm_remainder_len = 0;
+ return (CRYPTO_SUCCESS);
+ } else {
+ /* let rest of the code handle this */
+ length = pt_part;
+ }
+ } else if (length + ctx->ccm_remainder_len < block_size) {
+ /* accumulate bytes here and return */
+ bcopy(datap,
+ (uint8_t *)ctx->ccm_remainder + ctx->ccm_remainder_len,
+ length);
+ ctx->ccm_remainder_len += length;
+ ctx->ccm_copy_to = datap;
+ return (CRYPTO_SUCCESS);
+ }
+
+ do {
+ /* Unprocessed data from last call. */
+ if (ctx->ccm_remainder_len > 0) {
+ need = block_size - ctx->ccm_remainder_len;
+
+ if (need > remainder)
+ return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+
+ bcopy(datap, &((uint8_t *)ctx->ccm_remainder)
+ [ctx->ccm_remainder_len], need);
+
+ blockp = (uint8_t *)ctx->ccm_remainder;
+ } else {
+ blockp = datap;
+ }
+
+ /* Calculate the counter mode, ccm_cb is the counter block */
+ cbp = (uint8_t *)ctx->ccm_tmp;
+ encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, cbp);
+
+ /*
+ * Increment counter.
+ * Counter bits are confined to the bottom 64 bits
+ */
+#ifdef _LITTLE_ENDIAN
+ counter = ntohll(ctx->ccm_cb[1] & ctx->ccm_counter_mask);
+ counter = htonll(counter + 1);
+#else
+ counter = ctx->ccm_cb[1] & ctx->ccm_counter_mask;
+ counter++;
+#endif /* _LITTLE_ENDIAN */
+ counter &= ctx->ccm_counter_mask;
+ ctx->ccm_cb[1] =
+ (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;
+
+ /* XOR with the ciphertext */
+ xor_block(blockp, cbp);
+
+ /* Copy the plaintext to the "holding buffer" */
+ resultp = (uint8_t *)ctx->ccm_pt_buf +
+ ctx->ccm_processed_data_len;
+ copy_block(cbp, resultp);
+
+ ctx->ccm_processed_data_len += block_size;
+
+ ctx->ccm_lastp = blockp;
+
+ /* Update pointer to next block of data to be processed. */
+ if (ctx->ccm_remainder_len != 0) {
+ datap += need;
+ ctx->ccm_remainder_len = 0;
+ } else {
+ datap += block_size;
+ }
+
+ remainder = (size_t)&data[length] - (size_t)datap;
+
+ /* Incomplete last block */
+ if (remainder > 0 && remainder < block_size) {
+ bcopy(datap, ctx->ccm_remainder, remainder);
+ ctx->ccm_remainder_len = remainder;
+ ctx->ccm_copy_to = datap;
+ if (ctx->ccm_processed_mac_len > 0) {
+ /*
+ * not expecting anymore ciphertext, just
+ * compute plaintext for the remaining input
+ */
+ ccm_decrypt_incomplete_block(ctx,
+ encrypt_block);
+ ctx->ccm_processed_data_len += remainder;
+ ctx->ccm_remainder_len = 0;
+ }
+ goto out;
+ }
+ ctx->ccm_copy_to = NULL;
+
+ } while (remainder > 0);
+
+out:
+ return (CRYPTO_SUCCESS);
+}
+
+int
+ccm_decrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ size_t mac_remain, pt_len;
+ uint8_t *pt, *mac_buf, *macp, *ccm_mac_p;
+ int rv;
+
+ pt_len = ctx->ccm_data_len;
+
+ /* Make sure output buffer can fit all of the plaintext */
+ if (out->cd_length < pt_len) {
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ pt = ctx->ccm_pt_buf;
+ mac_remain = ctx->ccm_processed_data_len;
+ mac_buf = (uint8_t *)ctx->ccm_mac_buf;
+
+ macp = (uint8_t *)ctx->ccm_tmp;
+
+ while (mac_remain > 0) {
+
+ if (mac_remain < block_size) {
+ bzero(macp, block_size);
+ bcopy(pt, macp, mac_remain);
+ mac_remain = 0;
+ } else {
+ copy_block(pt, macp);
+ mac_remain -= block_size;
+ pt += block_size;
+ }
+
+ /* calculate the CBC MAC */
+ xor_block(macp, mac_buf);
+ encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+ }
+
+ /* Calculate the CCM MAC */
+ ccm_mac_p = (uint8_t *)ctx->ccm_tmp;
+ calculate_ccm_mac((ccm_ctx_t *)ctx, ccm_mac_p, encrypt_block);
+
+ /* compare the input CCM MAC value with what we calculated */
+ if (bcmp(ctx->ccm_mac_input_buf, ccm_mac_p, ctx->ccm_mac_len)) {
+ /* They don't match */
+ return (CRYPTO_INVALID_MAC);
+ } else {
+ rv = crypto_put_output_data(ctx->ccm_pt_buf, out, pt_len);
+ if (rv != CRYPTO_SUCCESS)
+ return (rv);
+ out->cd_offset += pt_len;
+ }
+ return (CRYPTO_SUCCESS);
+}
+
+int
+ccm_validate_args(CK_AES_CCM_PARAMS *ccm_param, boolean_t is_encrypt_init)
+{
+ size_t macSize, nonceSize;
+ uint8_t q;
+ uint64_t maxValue;
+
+ /*
+ * Check the length of the MAC. The only valid
+ * lengths for the MAC are: 4, 6, 8, 10, 12, 14, 16
+ */
+ macSize = ccm_param->ulMACSize;
+ if ((macSize < 4) || (macSize > 16) || ((macSize % 2) != 0)) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+
+ /* Check the nonce length. Valid values are 7, 8, 9, 10, 11, 12, 13 */
+ nonceSize = ccm_param->ulNonceSize;
+ if ((nonceSize < 7) || (nonceSize > 13)) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+
+ /* q is the length of the field storing the length, in bytes */
+ q = (uint8_t)((15 - nonceSize) & 0xFF);
+
+
+ /*
+ * If it is decrypt, need to make sure size of ciphertext is at least
+ * bigger than MAC len
+ */
+ if ((!is_encrypt_init) && (ccm_param->ulDataSize < macSize)) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+
+ /*
+ * Check to make sure the length of the payload is within the
+ * range of values allowed by q
+ */
+ if (q < 8) {
+ maxValue = (1ULL << (q * 8)) - 1;
+ } else {
+ maxValue = ULONG_MAX;
+ }
+
+ if (ccm_param->ulDataSize > maxValue) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Format the first block used in CBC-MAC (B0) and the initial counter
+ * block based on formatting functions and counter generation functions
+ * specified in RFC 3610 and NIST publication 800-38C, appendix A
+ *
+ * b0 is the first block used in CBC-MAC
+ * cb0 is the first counter block
+ *
+ * It's assumed that the arguments b0 and cb0 are preallocated AES blocks
+ *
+ */
+static void
+ccm_format_initial_blocks(uchar_t *nonce, ulong_t nonceSize,
+ ulong_t authDataSize, uint8_t *b0, ccm_ctx_t *aes_ctx)
+{
+ uint64_t payloadSize;
+ uint8_t t, q, have_adata = 0;
+ size_t limit;
+ int i, j, k;
+ uint64_t mask = 0;
+ uint8_t *cb;
+
+ q = (uint8_t)((15 - nonceSize) & 0xFF);
+ t = (uint8_t)((aes_ctx->ccm_mac_len) & 0xFF);
+
+ /* Construct the first octet of b0 */
+ if (authDataSize > 0) {
+ have_adata = 1;
+ }
+ b0[0] = (have_adata << 6) | (((t - 2) / 2) << 3) | (q - 1);
+
+ /* copy the nonce value into b0 */
+ bcopy(nonce, &(b0[1]), nonceSize);
+
+ /* store the length of the payload into b0 */
+ bzero(&(b0[1+nonceSize]), q);
+
+ payloadSize = aes_ctx->ccm_data_len;
+ limit = 8 < q ? 8 : q;
+
+ for (i = 0, j = 0, k = 15; i < limit; i++, j += 8, k--) {
+ b0[k] = (uint8_t)((payloadSize >> j) & 0xFF);
+ }
+
+ /* format the counter block */
+
+ cb = (uint8_t *)aes_ctx->ccm_cb;
+
+ cb[0] = 0x07 & (q-1); /* first byte */
+
+ /* copy the nonce value into the counter block */
+ bcopy(nonce, &(cb[1]), nonceSize);
+
+ bzero(&(cb[1+nonceSize]), q);
+
+ /* Create the mask for the counter field based on the size of nonce */
+ q <<= 3;
+ while (q-- > 0) {
+ mask |= (1ULL << q);
+ }
+
+#ifdef _LITTLE_ENDIAN
+ mask = htonll(mask);
+#endif
+ aes_ctx->ccm_counter_mask = mask;
+
+ /*
+ * During calculation, we start using counter block 1, we will
+ * set it up right here.
+ * We can just set the last byte to have the value 1, because
+ * even with the biggest nonce of 13, the last byte of the
+ * counter block will be used for the counter value.
+ */
+ cb[15] = 0x01;
+}
+
+/*
+ * Encode the length of the associated data as
+ * specified in RFC 3610 and NIST publication 800-38C, appendix A
+ */
+static void
+encode_adata_len(ulong_t auth_data_len, uint8_t *encoded, size_t *encoded_len)
+{
+#ifdef UNALIGNED_POINTERS_PERMITTED
+ uint32_t *lencoded_ptr;
+#ifdef _LP64
+ uint64_t *llencoded_ptr;
+#endif
+#endif /* UNALIGNED_POINTERS_PERMITTED */
+
+ if (auth_data_len < ((1ULL<<16) - (1ULL<<8))) {
+ /* 0 < a < (2^16-2^8) */
+ *encoded_len = 2;
+ encoded[0] = (auth_data_len & 0xff00) >> 8;
+ encoded[1] = auth_data_len & 0xff;
+
+ } else if ((auth_data_len >= ((1ULL<<16) - (1ULL<<8))) &&
+ (auth_data_len < (1ULL << 31))) {
+ /* (2^16-2^8) <= a < 2^32 */
+ *encoded_len = 6;
+ encoded[0] = 0xff;
+ encoded[1] = 0xfe;
+#ifdef UNALIGNED_POINTERS_PERMITTED
+ lencoded_ptr = (uint32_t *)&encoded[2];
+ *lencoded_ptr = htonl(auth_data_len);
+#else
+ encoded[2] = (auth_data_len & 0xff000000) >> 24;
+ encoded[3] = (auth_data_len & 0xff0000) >> 16;
+ encoded[4] = (auth_data_len & 0xff00) >> 8;
+ encoded[5] = auth_data_len & 0xff;
+#endif /* UNALIGNED_POINTERS_PERMITTED */
+
+#ifdef _LP64
+ } else {
+ /* 2^32 <= a < 2^64 */
+ *encoded_len = 10;
+ encoded[0] = 0xff;
+ encoded[1] = 0xff;
+#ifdef UNALIGNED_POINTERS_PERMITTED
+ llencoded_ptr = (uint64_t *)&encoded[2];
+ *llencoded_ptr = htonl(auth_data_len);
+#else
+ encoded[2] = (auth_data_len & 0xff00000000000000) >> 56;
+ encoded[3] = (auth_data_len & 0xff000000000000) >> 48;
+ encoded[4] = (auth_data_len & 0xff0000000000) >> 40;
+ encoded[5] = (auth_data_len & 0xff00000000) >> 32;
+ encoded[6] = (auth_data_len & 0xff000000) >> 24;
+ encoded[7] = (auth_data_len & 0xff0000) >> 16;
+ encoded[8] = (auth_data_len & 0xff00) >> 8;
+ encoded[9] = auth_data_len & 0xff;
+#endif /* UNALIGNED_POINTERS_PERMITTED */
+#endif /* _LP64 */
+ }
+}
+
+/*
+ * The following function should be call at encrypt or decrypt init time
+ * for AES CCM mode.
+ */
+int
+ccm_init(ccm_ctx_t *ctx, unsigned char *nonce, size_t nonce_len,
+ unsigned char *auth_data, size_t auth_data_len, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ uint8_t *mac_buf, *datap, *ivp, *authp;
+ size_t remainder, processed;
+ uint8_t encoded_a[10]; /* max encoded auth data length is 10 octets */
+ size_t encoded_a_len = 0;
+
+ mac_buf = (uint8_t *)&(ctx->ccm_mac_buf);
+
+ /*
+ * Format the 1st block for CBC-MAC and construct the
+ * 1st counter block.
+ *
+ * aes_ctx->ccm_iv is used for storing the counter block
+ * mac_buf will store b0 at this time.
+ */
+ ccm_format_initial_blocks(nonce, nonce_len,
+ auth_data_len, mac_buf, ctx);
+
+ /* The IV for CBC MAC for AES CCM mode is always zero */
+ ivp = (uint8_t *)ctx->ccm_tmp;
+ bzero(ivp, block_size);
+
+ xor_block(ivp, mac_buf);
+
+ /* encrypt the nonce */
+ encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+
+ /* take care of the associated data, if any */
+ if (auth_data_len == 0) {
+ return (CRYPTO_SUCCESS);
+ }
+
+ encode_adata_len(auth_data_len, encoded_a, &encoded_a_len);
+
+ remainder = auth_data_len;
+
+ /* 1st block: it contains encoded associated data, and some data */
+ authp = (uint8_t *)ctx->ccm_tmp;
+ bzero(authp, block_size);
+ bcopy(encoded_a, authp, encoded_a_len);
+ processed = block_size - encoded_a_len;
+ if (processed > auth_data_len) {
+ /* in case auth_data is very small */
+ processed = auth_data_len;
+ }
+ bcopy(auth_data, authp+encoded_a_len, processed);
+ /* xor with previous buffer */
+ xor_block(authp, mac_buf);
+ encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+ remainder -= processed;
+ if (remainder == 0) {
+ /* a small amount of associated data, it's all done now */
+ return (CRYPTO_SUCCESS);
+ }
+
+ do {
+ if (remainder < block_size) {
+ /*
+ * There's not a block full of data, pad rest of
+ * buffer with zero
+ */
+ bzero(authp, block_size);
+ bcopy(&(auth_data[processed]), authp, remainder);
+ datap = (uint8_t *)authp;
+ remainder = 0;
+ } else {
+ datap = (uint8_t *)(&(auth_data[processed]));
+ processed += block_size;
+ remainder -= block_size;
+ }
+
+ xor_block(datap, mac_buf);
+ encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+
+ } while (remainder > 0);
+
+ return (CRYPTO_SUCCESS);
+}
+
+int
+ccm_init_ctx(ccm_ctx_t *ccm_ctx, char *param, int kmflag,
+ boolean_t is_encrypt_init, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ int rv;
+ CK_AES_CCM_PARAMS *ccm_param;
+
+ if (param != NULL) {
+ ccm_param = (CK_AES_CCM_PARAMS *)param;
+
+ if ((rv = ccm_validate_args(ccm_param,
+ is_encrypt_init)) != 0) {
+ return (rv);
+ }
+
+ ccm_ctx->ccm_mac_len = ccm_param->ulMACSize;
+ if (is_encrypt_init) {
+ ccm_ctx->ccm_data_len = ccm_param->ulDataSize;
+ } else {
+ ccm_ctx->ccm_data_len =
+ ccm_param->ulDataSize - ccm_ctx->ccm_mac_len;
+ ccm_ctx->ccm_processed_mac_len = 0;
+ }
+ ccm_ctx->ccm_processed_data_len = 0;
+
+ ccm_ctx->ccm_flags |= CCM_MODE;
+ } else {
+ rv = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto out;
+ }
+
+ if (ccm_init(ccm_ctx, ccm_param->nonce, ccm_param->ulNonceSize,
+ ccm_param->authData, ccm_param->ulAuthDataSize, block_size,
+ encrypt_block, xor_block) != 0) {
+ rv = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto out;
+ }
+ if (!is_encrypt_init) {
+ /* allocate buffer for storing decrypted plaintext */
+ ccm_ctx->ccm_pt_buf = vmem_alloc(ccm_ctx->ccm_data_len,
+ kmflag);
+ if (ccm_ctx->ccm_pt_buf == NULL) {
+ rv = CRYPTO_HOST_MEMORY;
+ }
+ }
+out:
+ return (rv);
+}
+
+void *
+ccm_alloc_ctx(int kmflag)
+{
+ ccm_ctx_t *ccm_ctx;
+
+ if ((ccm_ctx = kmem_zalloc(sizeof (ccm_ctx_t), kmflag)) == NULL)
+ return (NULL);
+
+ ccm_ctx->ccm_flags = CCM_MODE;
+ return (ccm_ctx);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/byteorder.h>
+
+/*
+ * Encrypt and decrypt multiple blocks of data in counter mode.
+ */
+int
+ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *data, size_t length,
+ crypto_data_t *out, size_t block_size,
+ int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ size_t remainder = length;
+ size_t need = 0;
+ uint8_t *datap = (uint8_t *)data;
+ uint8_t *blockp;
+ uint8_t *lastp;
+ void *iov_or_mp;
+ offset_t offset;
+ uint8_t *out_data_1;
+ uint8_t *out_data_2;
+ size_t out_data_1_len;
+ uint64_t lower_counter, upper_counter;
+
+ if (length + ctx->ctr_remainder_len < block_size) {
+ /* accumulate bytes here and return */
+ bcopy(datap,
+ (uint8_t *)ctx->ctr_remainder + ctx->ctr_remainder_len,
+ length);
+ ctx->ctr_remainder_len += length;
+ ctx->ctr_copy_to = datap;
+ return (CRYPTO_SUCCESS);
+ }
+
+ lastp = (uint8_t *)ctx->ctr_cb;
+ if (out != NULL)
+ crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+ do {
+ /* Unprocessed data from last call. */
+ if (ctx->ctr_remainder_len > 0) {
+ need = block_size - ctx->ctr_remainder_len;
+
+ if (need > remainder)
+ return (CRYPTO_DATA_LEN_RANGE);
+
+ bcopy(datap, &((uint8_t *)ctx->ctr_remainder)
+ [ctx->ctr_remainder_len], need);
+
+ blockp = (uint8_t *)ctx->ctr_remainder;
+ } else {
+ blockp = datap;
+ }
+
+ /* ctr_cb is the counter block */
+ cipher(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb,
+ (uint8_t *)ctx->ctr_tmp);
+
+ lastp = (uint8_t *)ctx->ctr_tmp;
+
+ /*
+ * Increment Counter.
+ */
+ lower_counter = ntohll(ctx->ctr_cb[1] & ctx->ctr_lower_mask);
+ lower_counter = htonll(lower_counter + 1);
+ lower_counter &= ctx->ctr_lower_mask;
+ ctx->ctr_cb[1] = (ctx->ctr_cb[1] & ~(ctx->ctr_lower_mask)) |
+ lower_counter;
+
+ /* wrap around */
+ if (lower_counter == 0) {
+ upper_counter =
+ ntohll(ctx->ctr_cb[0] & ctx->ctr_upper_mask);
+ upper_counter = htonll(upper_counter + 1);
+ upper_counter &= ctx->ctr_upper_mask;
+ ctx->ctr_cb[0] =
+ (ctx->ctr_cb[0] & ~(ctx->ctr_upper_mask)) |
+ upper_counter;
+ }
+
+ /*
+ * XOR encrypted counter block with the current clear block.
+ */
+ xor_block(blockp, lastp);
+
+ if (out == NULL) {
+ if (ctx->ctr_remainder_len > 0) {
+ bcopy(lastp, ctx->ctr_copy_to,
+ ctx->ctr_remainder_len);
+ bcopy(lastp + ctx->ctr_remainder_len, datap,
+ need);
+ }
+ } else {
+ crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+ &out_data_1_len, &out_data_2, block_size);
+
+ /* copy block to where it belongs */
+ bcopy(lastp, out_data_1, out_data_1_len);
+ if (out_data_2 != NULL) {
+ bcopy(lastp + out_data_1_len, out_data_2,
+ block_size - out_data_1_len);
+ }
+ /* update offset */
+ out->cd_offset += block_size;
+ }
+
+ /* Update pointer to next block of data to be processed. */
+ if (ctx->ctr_remainder_len != 0) {
+ datap += need;
+ ctx->ctr_remainder_len = 0;
+ } else {
+ datap += block_size;
+ }
+
+ remainder = (size_t)&data[length] - (size_t)datap;
+
+ /* Incomplete last block. */
+ if (remainder > 0 && remainder < block_size) {
+ bcopy(datap, ctx->ctr_remainder, remainder);
+ ctx->ctr_remainder_len = remainder;
+ ctx->ctr_copy_to = datap;
+ goto out;
+ }
+ ctx->ctr_copy_to = NULL;
+
+ } while (remainder > 0);
+
+out:
+ return (CRYPTO_SUCCESS);
+}
+
+int
+ctr_mode_final(ctr_ctx_t *ctx, crypto_data_t *out,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *))
+{
+ uint8_t *lastp;
+ void *iov_or_mp;
+ offset_t offset;
+ uint8_t *out_data_1;
+ uint8_t *out_data_2;
+ size_t out_data_1_len;
+ uint8_t *p;
+ int i;
+
+ if (out->cd_length < ctx->ctr_remainder_len)
+ return (CRYPTO_DATA_LEN_RANGE);
+
+ encrypt_block(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb,
+ (uint8_t *)ctx->ctr_tmp);
+
+ lastp = (uint8_t *)ctx->ctr_tmp;
+ p = (uint8_t *)ctx->ctr_remainder;
+ for (i = 0; i < ctx->ctr_remainder_len; i++) {
+ p[i] ^= lastp[i];
+ }
+
+ crypto_init_ptrs(out, &iov_or_mp, &offset);
+ crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+ &out_data_1_len, &out_data_2, ctx->ctr_remainder_len);
+
+ bcopy(p, out_data_1, out_data_1_len);
+ if (out_data_2 != NULL) {
+ bcopy((uint8_t *)p + out_data_1_len,
+ out_data_2, ctx->ctr_remainder_len - out_data_1_len);
+ }
+ out->cd_offset += ctx->ctr_remainder_len;
+ ctx->ctr_remainder_len = 0;
+ return (CRYPTO_SUCCESS);
+}
+
+int
+ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb,
+void (*copy_block)(uint8_t *, uint8_t *))
+{
+ uint64_t upper_mask = 0;
+ uint64_t lower_mask = 0;
+
+ if (count == 0 || count > 128) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+ /* upper 64 bits of the mask */
+ if (count >= 64) {
+ count -= 64;
+ upper_mask = (count == 64) ? UINT64_MAX : (1ULL << count) - 1;
+ lower_mask = UINT64_MAX;
+ } else {
+ /* now the lower 63 bits */
+ lower_mask = (1ULL << count) - 1;
+ }
+ ctr_ctx->ctr_lower_mask = htonll(lower_mask);
+ ctr_ctx->ctr_upper_mask = htonll(upper_mask);
+
+ copy_block(cb, (uchar_t *)ctr_ctx->ctr_cb);
+ ctr_ctx->ctr_lastp = (uint8_t *)&ctr_ctx->ctr_cb[0];
+ ctr_ctx->ctr_flags |= CTR_MODE;
+ return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+void *
+ctr_alloc_ctx(int kmflag)
+{
+ ctr_ctx_t *ctr_ctx;
+
+ if ((ctr_ctx = kmem_zalloc(sizeof (ctr_ctx_t), kmflag)) == NULL)
+ return (NULL);
+
+ ctr_ctx->ctr_flags = CTR_MODE;
+ return (ctr_ctx);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+
+/*
+ * Algorithm independent ECB functions.
+ */
+int
+ecb_cipher_contiguous_blocks(ecb_ctx_t *ctx, char *data, size_t length,
+ crypto_data_t *out, size_t block_size,
+ int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct))
+{
+ size_t remainder = length;
+ size_t need = 0;
+ uint8_t *datap = (uint8_t *)data;
+ uint8_t *blockp;
+ uint8_t *lastp;
+ void *iov_or_mp;
+ offset_t offset;
+ uint8_t *out_data_1;
+ uint8_t *out_data_2;
+ size_t out_data_1_len;
+
+ if (length + ctx->ecb_remainder_len < block_size) {
+ /* accumulate bytes here and return */
+ bcopy(datap,
+ (uint8_t *)ctx->ecb_remainder + ctx->ecb_remainder_len,
+ length);
+ ctx->ecb_remainder_len += length;
+ ctx->ecb_copy_to = datap;
+ return (CRYPTO_SUCCESS);
+ }
+
+ lastp = (uint8_t *)ctx->ecb_iv;
+ if (out != NULL)
+ crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+ do {
+ /* Unprocessed data from last call. */
+ if (ctx->ecb_remainder_len > 0) {
+ need = block_size - ctx->ecb_remainder_len;
+
+ if (need > remainder)
+ return (CRYPTO_DATA_LEN_RANGE);
+
+ bcopy(datap, &((uint8_t *)ctx->ecb_remainder)
+ [ctx->ecb_remainder_len], need);
+
+ blockp = (uint8_t *)ctx->ecb_remainder;
+ } else {
+ blockp = datap;
+ }
+
+ if (out == NULL) {
+ cipher(ctx->ecb_keysched, blockp, blockp);
+
+ ctx->ecb_lastp = blockp;
+ lastp = blockp;
+
+ if (ctx->ecb_remainder_len > 0) {
+ bcopy(blockp, ctx->ecb_copy_to,
+ ctx->ecb_remainder_len);
+ bcopy(blockp + ctx->ecb_remainder_len, datap,
+ need);
+ }
+ } else {
+ cipher(ctx->ecb_keysched, blockp, lastp);
+ crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+ &out_data_1_len, &out_data_2, block_size);
+
+ /* copy block to where it belongs */
+ bcopy(lastp, out_data_1, out_data_1_len);
+ if (out_data_2 != NULL) {
+ bcopy(lastp + out_data_1_len, out_data_2,
+ block_size - out_data_1_len);
+ }
+ /* update offset */
+ out->cd_offset += block_size;
+ }
+
+ /* Update pointer to next block of data to be processed. */
+ if (ctx->ecb_remainder_len != 0) {
+ datap += need;
+ ctx->ecb_remainder_len = 0;
+ } else {
+ datap += block_size;
+ }
+
+ remainder = (size_t)&data[length] - (size_t)datap;
+
+ /* Incomplete last block. */
+ if (remainder > 0 && remainder < block_size) {
+ bcopy(datap, ctx->ecb_remainder, remainder);
+ ctx->ecb_remainder_len = remainder;
+ ctx->ecb_copy_to = datap;
+ goto out;
+ }
+ ctx->ecb_copy_to = NULL;
+
+ } while (remainder > 0);
+
+out:
+ return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+void *
+ecb_alloc_ctx(int kmflag)
+{
+ ecb_ctx_t *ecb_ctx;
+
+ if ((ecb_ctx = kmem_zalloc(sizeof (ecb_ctx_t), kmflag)) == NULL)
+ return (NULL);
+
+ ecb_ctx->ecb_flags = ECB_MODE;
+ return (ecb_ctx);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/byteorder.h>
+
+#ifdef __amd64
+
+#ifdef _KERNEL
+/* Workaround for no XMM kernel thread save/restore */
+#define KPREEMPT_DISABLE kpreempt_disable()
+#define KPREEMPT_ENABLE kpreempt_enable()
+
+#else
+#define KPREEMPT_DISABLE
+#define KPREEMPT_ENABLE
+#endif /* _KERNEL */
+
+extern void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
+static int intel_pclmulqdq_instruction_present(void);
+#endif /* __amd64 */
+
+struct aes_block {
+ uint64_t a;
+ uint64_t b;
+};
+
+
+/*
+ * gcm_mul()
+ * Perform a carry-less multiplication (that is, use XOR instead of the
+ * multiply operator) on *x_in and *y and place the result in *res.
+ *
+ * Byte swap the input (*x_in and *y) and the output (*res).
+ *
+ * Note: x_in, y, and res all point to 16-byte numbers (an array of two
+ * 64-bit integers).
+ */
+void
+gcm_mul(uint64_t *x_in, uint64_t *y, uint64_t *res)
+{
+#ifdef __amd64
+ if (intel_pclmulqdq_instruction_present()) {
+ KPREEMPT_DISABLE;
+ gcm_mul_pclmulqdq(x_in, y, res);
+ KPREEMPT_ENABLE;
+ } else
+#endif /* __amd64 */
+ {
+ static const uint64_t R = 0xe100000000000000ULL;
+ struct aes_block z = {0, 0};
+ struct aes_block v;
+ uint64_t x;
+ int i, j;
+
+ v.a = ntohll(y[0]);
+ v.b = ntohll(y[1]);
+
+ for (j = 0; j < 2; j++) {
+ x = ntohll(x_in[j]);
+ for (i = 0; i < 64; i++, x <<= 1) {
+ if (x & 0x8000000000000000ULL) {
+ z.a ^= v.a;
+ z.b ^= v.b;
+ }
+ if (v.b & 1ULL) {
+ v.b = (v.a << 63)|(v.b >> 1);
+ v.a = (v.a >> 1) ^ R;
+ } else {
+ v.b = (v.a << 63)|(v.b >> 1);
+ v.a = v.a >> 1;
+ }
+ }
+ }
+ res[0] = htonll(z.a);
+ res[1] = htonll(z.b);
+ }
+}
+
+
+#define GHASH(c, d, t) \
+ xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \
+ gcm_mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \
+ (uint64_t *)(void *)(t));
+
+
+/*
+ * Encrypt multiple blocks of data in GCM mode. Decrypt for GCM mode
+ * is done in another function.
+ */
+int
+gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
+ crypto_data_t *out, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ size_t remainder = length;
+ size_t need = 0;
+ uint8_t *datap = (uint8_t *)data;
+ uint8_t *blockp;
+ uint8_t *lastp;
+ void *iov_or_mp;
+ offset_t offset;
+ uint8_t *out_data_1;
+ uint8_t *out_data_2;
+ size_t out_data_1_len;
+ uint64_t counter;
+ uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
+
+ if (length + ctx->gcm_remainder_len < block_size) {
+ /* accumulate bytes here and return */
+ bcopy(datap,
+ (uint8_t *)ctx->gcm_remainder + ctx->gcm_remainder_len,
+ length);
+ ctx->gcm_remainder_len += length;
+ ctx->gcm_copy_to = datap;
+ return (CRYPTO_SUCCESS);
+ }
+
+ lastp = (uint8_t *)ctx->gcm_cb;
+ if (out != NULL)
+ crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+ do {
+ /* Unprocessed data from last call. */
+ if (ctx->gcm_remainder_len > 0) {
+ need = block_size - ctx->gcm_remainder_len;
+
+ if (need > remainder)
+ return (CRYPTO_DATA_LEN_RANGE);
+
+ bcopy(datap, &((uint8_t *)ctx->gcm_remainder)
+ [ctx->gcm_remainder_len], need);
+
+ blockp = (uint8_t *)ctx->gcm_remainder;
+ } else {
+ blockp = datap;
+ }
+
+ /*
+ * Increment counter. Counter bits are confined
+ * to the bottom 32 bits of the counter block.
+ */
+ counter = ntohll(ctx->gcm_cb[1] & counter_mask);
+ counter = htonll(counter + 1);
+ counter &= counter_mask;
+ ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
+
+ encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
+ (uint8_t *)ctx->gcm_tmp);
+ xor_block(blockp, (uint8_t *)ctx->gcm_tmp);
+
+ lastp = (uint8_t *)ctx->gcm_tmp;
+
+ ctx->gcm_processed_data_len += block_size;
+
+ if (out == NULL) {
+ if (ctx->gcm_remainder_len > 0) {
+ bcopy(blockp, ctx->gcm_copy_to,
+ ctx->gcm_remainder_len);
+ bcopy(blockp + ctx->gcm_remainder_len, datap,
+ need);
+ }
+ } else {
+ crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+ &out_data_1_len, &out_data_2, block_size);
+
+ /* copy block to where it belongs */
+ if (out_data_1_len == block_size) {
+ copy_block(lastp, out_data_1);
+ } else {
+ bcopy(lastp, out_data_1, out_data_1_len);
+ if (out_data_2 != NULL) {
+ bcopy(lastp + out_data_1_len,
+ out_data_2,
+ block_size - out_data_1_len);
+ }
+ }
+ /* update offset */
+ out->cd_offset += block_size;
+ }
+
+ /* add ciphertext to the hash */
+ GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
+
+ /* Update pointer to next block of data to be processed. */
+ if (ctx->gcm_remainder_len != 0) {
+ datap += need;
+ ctx->gcm_remainder_len = 0;
+ } else {
+ datap += block_size;
+ }
+
+ remainder = (size_t)&data[length] - (size_t)datap;
+
+ /* Incomplete last block. */
+ if (remainder > 0 && remainder < block_size) {
+ bcopy(datap, ctx->gcm_remainder, remainder);
+ ctx->gcm_remainder_len = remainder;
+ ctx->gcm_copy_to = datap;
+ goto out;
+ }
+ ctx->gcm_copy_to = NULL;
+
+ } while (remainder > 0);
+out:
+ return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+int
+gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
+ uint8_t *ghash, *macp = NULL;
+ int i, rv;
+
+ if (out->cd_length <
+ (ctx->gcm_remainder_len + ctx->gcm_tag_len)) {
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ ghash = (uint8_t *)ctx->gcm_ghash;
+
+ if (ctx->gcm_remainder_len > 0) {
+ uint64_t counter;
+ uint8_t *tmpp = (uint8_t *)ctx->gcm_tmp;
+
+ /*
+ * Here is where we deal with data that is not a
+ * multiple of the block size.
+ */
+
+ /*
+ * Increment counter.
+ */
+ counter = ntohll(ctx->gcm_cb[1] & counter_mask);
+ counter = htonll(counter + 1);
+ counter &= counter_mask;
+ ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
+
+ encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
+ (uint8_t *)ctx->gcm_tmp);
+
+ macp = (uint8_t *)ctx->gcm_remainder;
+ bzero(macp + ctx->gcm_remainder_len,
+ block_size - ctx->gcm_remainder_len);
+
+ /* XOR with counter block */
+ for (i = 0; i < ctx->gcm_remainder_len; i++) {
+ macp[i] ^= tmpp[i];
+ }
+
+ /* add ciphertext to the hash */
+ GHASH(ctx, macp, ghash);
+
+ ctx->gcm_processed_data_len += ctx->gcm_remainder_len;
+ }
+
+ ctx->gcm_len_a_len_c[1] =
+ htonll(CRYPTO_BYTES2BITS(ctx->gcm_processed_data_len));
+ GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
+ encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
+ (uint8_t *)ctx->gcm_J0);
+ xor_block((uint8_t *)ctx->gcm_J0, ghash);
+
+ if (ctx->gcm_remainder_len > 0) {
+ rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len);
+ if (rv != CRYPTO_SUCCESS)
+ return (rv);
+ }
+ out->cd_offset += ctx->gcm_remainder_len;
+ ctx->gcm_remainder_len = 0;
+ rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len);
+ if (rv != CRYPTO_SUCCESS)
+ return (rv);
+ out->cd_offset += ctx->gcm_tag_len;
+
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * This will only deal with decrypting the last block of the input that
+ * might not be a multiple of block length.
+ */
+static void
+gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ uint8_t *datap, *outp, *counterp;
+ uint64_t counter;
+ uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
+ int i;
+
+ /*
+ * Increment counter.
+ * Counter bits are confined to the bottom 32 bits
+ */
+ counter = ntohll(ctx->gcm_cb[1] & counter_mask);
+ counter = htonll(counter + 1);
+ counter &= counter_mask;
+ ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
+
+ datap = (uint8_t *)ctx->gcm_remainder;
+ outp = &((ctx->gcm_pt_buf)[index]);
+ counterp = (uint8_t *)ctx->gcm_tmp;
+
+ /* authentication tag */
+ bzero((uint8_t *)ctx->gcm_tmp, block_size);
+ bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len);
+
+ /* add ciphertext to the hash */
+ GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
+
+ /* decrypt remaining ciphertext */
+ encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp);
+
+ /* XOR with counter block */
+ for (i = 0; i < ctx->gcm_remainder_len; i++) {
+ outp[i] = datap[i] ^ counterp[i];
+ }
+}
+
+/* ARGSUSED */
+int
+gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
+ crypto_data_t *out, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ size_t new_len;
+ uint8_t *new;
+
+ /*
+ * Copy contiguous ciphertext input blocks to plaintext buffer.
+ * Ciphertext will be decrypted in the final.
+ */
+ if (length > 0) {
+ new_len = ctx->gcm_pt_buf_len + length;
+ new = vmem_alloc(new_len, ctx->gcm_kmflag);
+ bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
+ vmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len);
+ if (new == NULL)
+ return (CRYPTO_HOST_MEMORY);
+
+ ctx->gcm_pt_buf = new;
+ ctx->gcm_pt_buf_len = new_len;
+ bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len],
+ length);
+ ctx->gcm_processed_data_len += length;
+ }
+
+ ctx->gcm_remainder_len = 0;
+ return (CRYPTO_SUCCESS);
+}
+
+int
+gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ size_t pt_len;
+ size_t remainder;
+ uint8_t *ghash;
+ uint8_t *blockp;
+ uint8_t *cbp;
+ uint64_t counter;
+ uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
+ int processed = 0, rv;
+
+ ASSERT(ctx->gcm_processed_data_len == ctx->gcm_pt_buf_len);
+
+ pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
+ ghash = (uint8_t *)ctx->gcm_ghash;
+ blockp = ctx->gcm_pt_buf;
+ remainder = pt_len;
+ while (remainder > 0) {
+ /* Incomplete last block */
+ if (remainder < block_size) {
+ bcopy(blockp, ctx->gcm_remainder, remainder);
+ ctx->gcm_remainder_len = remainder;
+ /*
+ * not expecting anymore ciphertext, just
+ * compute plaintext for the remaining input
+ */
+ gcm_decrypt_incomplete_block(ctx, block_size,
+ processed, encrypt_block, xor_block);
+ ctx->gcm_remainder_len = 0;
+ goto out;
+ }
+ /* add ciphertext to the hash */
+ GHASH(ctx, blockp, ghash);
+
+ /*
+ * Increment counter.
+ * Counter bits are confined to the bottom 32 bits
+ */
+ counter = ntohll(ctx->gcm_cb[1] & counter_mask);
+ counter = htonll(counter + 1);
+ counter &= counter_mask;
+ ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
+
+ cbp = (uint8_t *)ctx->gcm_tmp;
+ encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp);
+
+ /* XOR with ciphertext */
+ xor_block(cbp, blockp);
+
+ processed += block_size;
+ blockp += block_size;
+ remainder -= block_size;
+ }
+out:
+ ctx->gcm_len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(pt_len));
+ GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
+ encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
+ (uint8_t *)ctx->gcm_J0);
+ xor_block((uint8_t *)ctx->gcm_J0, ghash);
+
+ /* compare the input authentication tag with what we calculated */
+ if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) {
+ /* They don't match */
+ return (CRYPTO_INVALID_MAC);
+ } else {
+ rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len);
+ if (rv != CRYPTO_SUCCESS)
+ return (rv);
+ out->cd_offset += pt_len;
+ }
+ return (CRYPTO_SUCCESS);
+}
+
+static int
+gcm_validate_args(CK_AES_GCM_PARAMS *gcm_param)
+{
+ size_t tag_len;
+
+ /*
+ * Check the length of the authentication tag (in bits).
+ */
+ tag_len = gcm_param->ulTagBits;
+ switch (tag_len) {
+ case 32:
+ case 64:
+ case 96:
+ case 104:
+ case 112:
+ case 120:
+ case 128:
+ break;
+ default:
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+
+ if (gcm_param->ulIvLen == 0)
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+
+ return (CRYPTO_SUCCESS);
+}
+
+static void
+gcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len,
+ gcm_ctx_t *ctx, size_t block_size,
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ uint8_t *cb;
+ ulong_t remainder = iv_len;
+ ulong_t processed = 0;
+ uint8_t *datap, *ghash;
+ uint64_t len_a_len_c[2];
+
+ ghash = (uint8_t *)ctx->gcm_ghash;
+ cb = (uint8_t *)ctx->gcm_cb;
+ if (iv_len == 12) {
+ bcopy(iv, cb, 12);
+ cb[12] = 0;
+ cb[13] = 0;
+ cb[14] = 0;
+ cb[15] = 1;
+ /* J0 will be used again in the final */
+ copy_block(cb, (uint8_t *)ctx->gcm_J0);
+ } else {
+ /* GHASH the IV */
+ do {
+ if (remainder < block_size) {
+ bzero(cb, block_size);
+ bcopy(&(iv[processed]), cb, remainder);
+ datap = (uint8_t *)cb;
+ remainder = 0;
+ } else {
+ datap = (uint8_t *)(&(iv[processed]));
+ processed += block_size;
+ remainder -= block_size;
+ }
+ GHASH(ctx, datap, ghash);
+ } while (remainder > 0);
+
+ len_a_len_c[0] = 0;
+ len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(iv_len));
+ GHASH(ctx, len_a_len_c, ctx->gcm_J0);
+
+ /* J0 will be used again in the final */
+ copy_block((uint8_t *)ctx->gcm_J0, (uint8_t *)cb);
+ }
+}
+
+/*
+ * The following function is called at encrypt or decrypt init time
+ * for AES GCM mode.
+ */
+int
+gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len,
+ unsigned char *auth_data, size_t auth_data_len, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ uint8_t *ghash, *datap, *authp;
+ size_t remainder, processed;
+
+ /* encrypt zero block to get subkey H */
+ bzero(ctx->gcm_H, sizeof (ctx->gcm_H));
+ encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H,
+ (uint8_t *)ctx->gcm_H);
+
+ gcm_format_initial_blocks(iv, iv_len, ctx, block_size,
+ copy_block, xor_block);
+
+ authp = (uint8_t *)ctx->gcm_tmp;
+ ghash = (uint8_t *)ctx->gcm_ghash;
+ bzero(authp, block_size);
+ bzero(ghash, block_size);
+
+ processed = 0;
+ remainder = auth_data_len;
+ do {
+ if (remainder < block_size) {
+ /*
+ * There's not a block full of data, pad rest of
+ * buffer with zero
+ */
+ bzero(authp, block_size);
+ bcopy(&(auth_data[processed]), authp, remainder);
+ datap = (uint8_t *)authp;
+ remainder = 0;
+ } else {
+ datap = (uint8_t *)(&(auth_data[processed]));
+ processed += block_size;
+ remainder -= block_size;
+ }
+
+ /* add auth data to the hash */
+ GHASH(ctx, datap, ghash);
+
+ } while (remainder > 0);
+
+ return (CRYPTO_SUCCESS);
+}
+
+int
+gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ int rv;
+ CK_AES_GCM_PARAMS *gcm_param;
+
+ if (param != NULL) {
+ gcm_param = (CK_AES_GCM_PARAMS *)(void *)param;
+
+ if ((rv = gcm_validate_args(gcm_param)) != 0) {
+ return (rv);
+ }
+
+ gcm_ctx->gcm_tag_len = gcm_param->ulTagBits;
+ gcm_ctx->gcm_tag_len >>= 3;
+ gcm_ctx->gcm_processed_data_len = 0;
+
+ /* these values are in bits */
+ gcm_ctx->gcm_len_a_len_c[0]
+ = htonll(CRYPTO_BYTES2BITS(gcm_param->ulAADLen));
+
+ rv = CRYPTO_SUCCESS;
+ gcm_ctx->gcm_flags |= GCM_MODE;
+ } else {
+ rv = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto out;
+ }
+
+ if (gcm_init(gcm_ctx, gcm_param->pIv, gcm_param->ulIvLen,
+ gcm_param->pAAD, gcm_param->ulAADLen, block_size,
+ encrypt_block, copy_block, xor_block) != 0) {
+ rv = CRYPTO_MECHANISM_PARAM_INVALID;
+ }
+out:
+ return (rv);
+}
+
+int
+gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *))
+{
+ int rv;
+ CK_AES_GMAC_PARAMS *gmac_param;
+
+ if (param != NULL) {
+ gmac_param = (CK_AES_GMAC_PARAMS *)(void *)param;
+
+ gcm_ctx->gcm_tag_len = CRYPTO_BITS2BYTES(AES_GMAC_TAG_BITS);
+ gcm_ctx->gcm_processed_data_len = 0;
+
+ /* these values are in bits */
+ gcm_ctx->gcm_len_a_len_c[0]
+ = htonll(CRYPTO_BYTES2BITS(gmac_param->ulAADLen));
+
+ rv = CRYPTO_SUCCESS;
+ gcm_ctx->gcm_flags |= GMAC_MODE;
+ } else {
+ rv = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto out;
+ }
+
+ if (gcm_init(gcm_ctx, gmac_param->pIv, AES_GMAC_IV_LEN,
+ gmac_param->pAAD, gmac_param->ulAADLen, block_size,
+ encrypt_block, copy_block, xor_block) != 0) {
+ rv = CRYPTO_MECHANISM_PARAM_INVALID;
+ }
+out:
+ return (rv);
+}
+
+void *
+gcm_alloc_ctx(int kmflag)
+{
+ gcm_ctx_t *gcm_ctx;
+
+ if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL)
+ return (NULL);
+
+ gcm_ctx->gcm_flags = GCM_MODE;
+ return (gcm_ctx);
+}
+
+void *
+gmac_alloc_ctx(int kmflag)
+{
+ gcm_ctx_t *gcm_ctx;
+
+ if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL)
+ return (NULL);
+
+ gcm_ctx->gcm_flags = GMAC_MODE;
+ return (gcm_ctx);
+}
+
+void
+gcm_set_kmflag(gcm_ctx_t *ctx, int kmflag)
+{
+ ctx->gcm_kmflag = kmflag;
+}
+
+
+#ifdef __amd64
+
+#define INTEL_PCLMULQDQ_FLAG (1 << 1)
+
+/*
+ * Return 1 if executing on Intel with PCLMULQDQ instructions,
+ * otherwise 0 (i.e., Intel without PCLMULQDQ or AMD64).
+ * Cache the result, as the CPU can't change.
+ *
+ * Note: the userland version uses getisax(). The kernel version uses
+ * is_x86_featureset().
+ */
+static int
+intel_pclmulqdq_instruction_present(void)
+{
+ static int cached_result = -1;
+ unsigned eax, ebx, ecx, edx;
+ unsigned func, subfunc;
+
+ if (cached_result == -1) { /* first time */
+ /* check for an intel cpu */
+ func = 0;
+ subfunc = 0;
+
+ __asm__ __volatile__(
+ "cpuid"
+ : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
+ : "a"(func), "c"(subfunc));
+
+ if (memcmp((char *) (&ebx), "Genu", 4) == 0 &&
+ memcmp((char *) (&edx), "ineI", 4) == 0 &&
+ memcmp((char *) (&ecx), "ntel", 4) == 0) {
+
+ func = 1;
+ subfunc = 0;
+
+ /* check for aes-ni instruction set */
+ __asm__ __volatile__(
+ "cpuid"
+ : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
+ : "a"(func), "c"(subfunc));
+
+ cached_result = !!(ecx & INTEL_PCLMULQDQ_FLAG);
+ } else {
+ cached_result = 0;
+ }
+ }
+
+ return (cached_result);
+}
+
+#endif /* __amd64 */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+
+/*
+ * Initialize by setting iov_or_mp to point to the current iovec or mp,
+ * and by setting current_offset to an offset within the current iovec or mp.
+ */
+void
+crypto_init_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset)
+{
+ offset_t offset;
+
+ switch (out->cd_format) {
+ case CRYPTO_DATA_RAW:
+ *current_offset = out->cd_offset;
+ break;
+
+ case CRYPTO_DATA_UIO: {
+ uio_t *uiop = out->cd_uio;
+ uintptr_t vec_idx;
+
+ offset = out->cd_offset;
+ for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
+ offset >= uiop->uio_iov[vec_idx].iov_len;
+ offset -= uiop->uio_iov[vec_idx++].iov_len)
+ ;
+
+ *current_offset = offset;
+ *iov_or_mp = (void *)vec_idx;
+ break;
+ }
+ } /* end switch */
+}
+
+/*
+ * Get pointers for where in the output to copy a block of encrypted or
+ * decrypted data. The iov_or_mp argument stores a pointer to the current
+ * iovec or mp, and offset stores an offset into the current iovec or mp.
+ */
+void
+crypto_get_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset,
+ uint8_t **out_data_1, size_t *out_data_1_len, uint8_t **out_data_2,
+ size_t amt)
+{
+ offset_t offset;
+
+ switch (out->cd_format) {
+ case CRYPTO_DATA_RAW: {
+ iovec_t *iov;
+
+ offset = *current_offset;
+ iov = &out->cd_raw;
+ if ((offset + amt) <= iov->iov_len) {
+ /* one block fits */
+ *out_data_1 = (uint8_t *)iov->iov_base + offset;
+ *out_data_1_len = amt;
+ *out_data_2 = NULL;
+ *current_offset = offset + amt;
+ }
+ break;
+ }
+
+ case CRYPTO_DATA_UIO: {
+ uio_t *uio = out->cd_uio;
+ iovec_t *iov;
+ offset_t offset;
+ uintptr_t vec_idx;
+ uint8_t *p;
+
+ offset = *current_offset;
+ vec_idx = (uintptr_t)(*iov_or_mp);
+ iov = (iovec_t *)&uio->uio_iov[vec_idx];
+ p = (uint8_t *)iov->iov_base + offset;
+ *out_data_1 = p;
+
+ if (offset + amt <= iov->iov_len) {
+ /* can fit one block into this iov */
+ *out_data_1_len = amt;
+ *out_data_2 = NULL;
+ *current_offset = offset + amt;
+ } else {
+ /* one block spans two iovecs */
+ *out_data_1_len = iov->iov_len - offset;
+ if (vec_idx == uio->uio_iovcnt)
+ return;
+ vec_idx++;
+ iov = (iovec_t *)&uio->uio_iov[vec_idx];
+ *out_data_2 = (uint8_t *)iov->iov_base;
+ *current_offset = amt - *out_data_1_len;
+ }
+ *iov_or_mp = (void *)vec_idx;
+ break;
+ }
+ } /* end switch */
+}
+
+void
+crypto_free_mode_ctx(void *ctx)
+{
+ common_ctx_t *common_ctx = (common_ctx_t *)ctx;
+
+ switch (common_ctx->cc_flags &
+ (ECB_MODE|CBC_MODE|CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) {
+ case ECB_MODE:
+ kmem_free(common_ctx, sizeof (ecb_ctx_t));
+ break;
+
+ case CBC_MODE:
+ kmem_free(common_ctx, sizeof (cbc_ctx_t));
+ break;
+
+ case CTR_MODE:
+ kmem_free(common_ctx, sizeof (ctr_ctx_t));
+ break;
+
+ case CCM_MODE:
+ if (((ccm_ctx_t *)ctx)->ccm_pt_buf != NULL)
+ vmem_free(((ccm_ctx_t *)ctx)->ccm_pt_buf,
+ ((ccm_ctx_t *)ctx)->ccm_data_len);
+
+ kmem_free(ctx, sizeof (ccm_ctx_t));
+ break;
+
+ case GCM_MODE:
+ case GMAC_MODE:
+ if (((gcm_ctx_t *)ctx)->gcm_pt_buf != NULL)
+ vmem_free(((gcm_ctx_t *)ctx)->gcm_pt_buf,
+ ((gcm_ctx_t *)ctx)->gcm_pt_buf_len);
+
+ kmem_free(ctx, sizeof (gcm_ctx_t));
+ }
+}
--- /dev/null
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * The basic framework for this code came from the reference
+ * implementation for MD5. That implementation is Copyright (C)
+ * 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved.
+ *
+ * License to copy and use this software is granted provided that it
+ * is identified as the "RSA Data Security, Inc. MD5 Message-Digest
+ * Algorithm" in all material mentioning or referencing this software
+ * or this function.
+ *
+ * License is also granted to make and use derivative works provided
+ * that such works are identified as "derived from the RSA Data
+ * Security, Inc. MD5 Message-Digest Algorithm" in all material
+ * mentioning or referencing the derived work.
+ *
+ * RSA Data Security, Inc. makes no representations concerning either
+ * the merchantability of this software or the suitability of this
+ * software for any particular purpose. It is provided "as is"
+ * without express or implied warranty of any kind.
+ *
+ * These notices must be retained in any copies of any part of this
+ * documentation and/or software.
+ *
+ * NOTE: Cleaned-up and optimized, version of SHA1, based on the FIPS 180-1
+ * standard, available at http://www.itl.nist.gov/fipspubs/fip180-1.htm
+ * Not as fast as one would like -- further optimizations are encouraged
+ * and appreciated.
+ */
+
+#include <sys/zfs_context.h>
+#include <sha1/sha1.h>
+#include <sha1/sha1_consts.h>
+
+#ifdef _LITTLE_ENDIAN
+#include <sys/byteorder.h>
+#define HAVE_HTONL
+#endif
+
+#define _RESTRICT_KYWD
+
+static void Encode(uint8_t *, const uint32_t *, size_t);
+
+#if defined(__amd64)
+
+#define SHA1_TRANSFORM(ctx, in) sha1_block_data_order((ctx), (in), 1)
+#define SHA1_TRANSFORM_BLOCKS(ctx, in, num) sha1_block_data_order((ctx), \
+ (in), (num))
+
+void sha1_block_data_order(SHA1_CTX *ctx, const void *inpp, size_t num_blocks);
+
+#else
+
+#define SHA1_TRANSFORM(ctx, in) SHA1Transform((ctx), (in))
+
+static void SHA1Transform(SHA1_CTX *, const uint8_t *);
+
+#endif
+
+
+static uint8_t PADDING[64] = { 0x80, /* all zeros */ };
+
+/*
+ * F, G, and H are the basic SHA1 functions.
+ */
+#define F(b, c, d) (((b) & (c)) | ((~b) & (d)))
+#define G(b, c, d) ((b) ^ (c) ^ (d))
+#define H(b, c, d) (((b) & (c)) | (((b)|(c)) & (d)))
+
+/*
+ * ROTATE_LEFT rotates x left n bits.
+ */
+
+#if defined(__GNUC__) && defined(_LP64)
+static __inline__ uint64_t
+ROTATE_LEFT(uint64_t value, uint32_t n)
+{
+ uint32_t t32;
+
+ t32 = (uint32_t)value;
+ return ((t32 << n) | (t32 >> (32 - n)));
+}
+
+#else
+
+#define ROTATE_LEFT(x, n) \
+ (((x) << (n)) | ((x) >> ((sizeof (x) * NBBY)-(n))))
+
+#endif
+
+
+/*
+ * SHA1Init()
+ *
+ * purpose: initializes the sha1 context and begins and sha1 digest operation
+ * input: SHA1_CTX * : the context to initializes.
+ * output: void
+ */
+
+void
+SHA1Init(SHA1_CTX *ctx)
+{
+ ctx->count[0] = ctx->count[1] = 0;
+
+ /*
+ * load magic initialization constants. Tell lint
+ * that these constants are unsigned by using U.
+ */
+
+ ctx->state[0] = 0x67452301U;
+ ctx->state[1] = 0xefcdab89U;
+ ctx->state[2] = 0x98badcfeU;
+ ctx->state[3] = 0x10325476U;
+ ctx->state[4] = 0xc3d2e1f0U;
+}
+
+void
+SHA1Update(SHA1_CTX *ctx, const void *inptr, size_t input_len)
+{
+ uint32_t i, buf_index, buf_len;
+ const uint8_t *input = inptr;
+#if defined(__amd64)
+ uint32_t block_count;
+#endif /* __amd64 */
+
+ /* check for noop */
+ if (input_len == 0)
+ return;
+
+ /* compute number of bytes mod 64 */
+ buf_index = (ctx->count[1] >> 3) & 0x3F;
+
+ /* update number of bits */
+ if ((ctx->count[1] += (input_len << 3)) < (input_len << 3))
+ ctx->count[0]++;
+
+ ctx->count[0] += (input_len >> 29);
+
+ buf_len = 64 - buf_index;
+
+ /* transform as many times as possible */
+ i = 0;
+ if (input_len >= buf_len) {
+
+ /*
+ * general optimization:
+ *
+ * only do initial bcopy() and SHA1Transform() if
+ * buf_index != 0. if buf_index == 0, we're just
+ * wasting our time doing the bcopy() since there
+ * wasn't any data left over from a previous call to
+ * SHA1Update().
+ */
+
+ if (buf_index) {
+ bcopy(input, &ctx->buf_un.buf8[buf_index], buf_len);
+ SHA1_TRANSFORM(ctx, ctx->buf_un.buf8);
+ i = buf_len;
+ }
+
+#if !defined(__amd64)
+ for (; i + 63 < input_len; i += 64)
+ SHA1_TRANSFORM(ctx, &input[i]);
+#else
+ block_count = (input_len - i) >> 6;
+ if (block_count > 0) {
+ SHA1_TRANSFORM_BLOCKS(ctx, &input[i], block_count);
+ i += block_count << 6;
+ }
+#endif /* !__amd64 */
+
+ /*
+ * general optimization:
+ *
+ * if i and input_len are the same, return now instead
+ * of calling bcopy(), since the bcopy() in this case
+ * will be an expensive nop.
+ */
+
+ if (input_len == i)
+ return;
+
+ buf_index = 0;
+ }
+
+ /* buffer remaining input */
+ bcopy(&input[i], &ctx->buf_un.buf8[buf_index], input_len - i);
+}
+
+/*
+ * SHA1Final()
+ *
+ * purpose: ends an sha1 digest operation, finalizing the message digest and
+ * zeroing the context.
+ * input: uchar_t * : A buffer to store the digest.
+ * : The function actually uses void* because many
+ * : callers pass things other than uchar_t here.
+ * SHA1_CTX * : the context to finalize, save, and zero
+ * output: void
+ */
+
+void
+SHA1Final(void *digest, SHA1_CTX *ctx)
+{
+ uint8_t bitcount_be[sizeof (ctx->count)];
+ uint32_t index = (ctx->count[1] >> 3) & 0x3f;
+
+ /* store bit count, big endian */
+ Encode(bitcount_be, ctx->count, sizeof (bitcount_be));
+
+ /* pad out to 56 mod 64 */
+ SHA1Update(ctx, PADDING, ((index < 56) ? 56 : 120) - index);
+
+ /* append length (before padding) */
+ SHA1Update(ctx, bitcount_be, sizeof (bitcount_be));
+
+ /* store state in digest */
+ Encode(digest, ctx->state, sizeof (ctx->state));
+
+ /* zeroize sensitive information */
+ bzero(ctx, sizeof (*ctx));
+}
+
+
+#if !defined(__amd64)
+
+typedef uint32_t sha1word;
+
+/*
+ * sparc optimization:
+ *
+ * on the sparc, we can load big endian 32-bit data easily. note that
+ * special care must be taken to ensure the address is 32-bit aligned.
+ * in the interest of speed, we don't check to make sure, since
+ * careful programming can guarantee this for us.
+ */
+
+#if defined(_BIG_ENDIAN)
+#define LOAD_BIG_32(addr) (*(uint32_t *)(addr))
+
+#elif defined(HAVE_HTONL)
+#define LOAD_BIG_32(addr) htonl(*((uint32_t *)(addr)))
+
+#else
+/* little endian -- will work on big endian, but slowly */
+#define LOAD_BIG_32(addr) \
+ (((addr)[0] << 24) | ((addr)[1] << 16) | ((addr)[2] << 8) | (addr)[3])
+#endif /* _BIG_ENDIAN */
+
+/*
+ * SHA1Transform()
+ */
+#if defined(W_ARRAY)
+#define W(n) w[n]
+#else /* !defined(W_ARRAY) */
+#define W(n) w_ ## n
+#endif /* !defined(W_ARRAY) */
+
+void /* CSTYLED */
+SHA1Transform(SHA1_CTX *ctx, const uint8_t blk[64])
+{
+ /* CSTYLED */
+ sha1word a = ctx->state[0];
+ sha1word b = ctx->state[1];
+ sha1word c = ctx->state[2];
+ sha1word d = ctx->state[3];
+ sha1word e = ctx->state[4];
+
+#if defined(W_ARRAY)
+ sha1word w[16];
+#else /* !defined(W_ARRAY) */
+ sha1word w_0, w_1, w_2, w_3, w_4, w_5, w_6, w_7;
+ sha1word w_8, w_9, w_10, w_11, w_12, w_13, w_14, w_15;
+#endif /* !defined(W_ARRAY) */
+
+ W(0) = LOAD_BIG_32((void *)(blk + 0));
+ W(1) = LOAD_BIG_32((void *)(blk + 4));
+ W(2) = LOAD_BIG_32((void *)(blk + 8));
+ W(3) = LOAD_BIG_32((void *)(blk + 12));
+ W(4) = LOAD_BIG_32((void *)(blk + 16));
+ W(5) = LOAD_BIG_32((void *)(blk + 20));
+ W(6) = LOAD_BIG_32((void *)(blk + 24));
+ W(7) = LOAD_BIG_32((void *)(blk + 28));
+ W(8) = LOAD_BIG_32((void *)(blk + 32));
+ W(9) = LOAD_BIG_32((void *)(blk + 36));
+ W(10) = LOAD_BIG_32((void *)(blk + 40));
+ W(11) = LOAD_BIG_32((void *)(blk + 44));
+ W(12) = LOAD_BIG_32((void *)(blk + 48));
+ W(13) = LOAD_BIG_32((void *)(blk + 52));
+ W(14) = LOAD_BIG_32((void *)(blk + 56));
+ W(15) = LOAD_BIG_32((void *)(blk + 60));
+
+ /*
+ * general optimization:
+ *
+ * even though this approach is described in the standard as
+ * being slower algorithmically, it is 30-40% faster than the
+ * "faster" version under SPARC, because this version has more
+ * of the constraints specified at compile-time and uses fewer
+ * variables (and therefore has better register utilization)
+ * than its "speedier" brother. (i've tried both, trust me)
+ *
+ * for either method given in the spec, there is an "assignment"
+ * phase where the following takes place:
+ *
+ * tmp = (main_computation);
+ * e = d; d = c; c = rotate_left(b, 30); b = a; a = tmp;
+ *
+ * we can make the algorithm go faster by not doing this work,
+ * but just pretending that `d' is now `e', etc. this works
+ * really well and obviates the need for a temporary variable.
+ * however, we still explicitly perform the rotate action,
+ * since it is cheaper on SPARC to do it once than to have to
+ * do it over and over again.
+ */
+
+ /* round 1 */
+ e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(0) + SHA1_CONST(0); /* 0 */
+ b = ROTATE_LEFT(b, 30);
+
+ d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(1) + SHA1_CONST(0); /* 1 */
+ a = ROTATE_LEFT(a, 30);
+
+ c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(2) + SHA1_CONST(0); /* 2 */
+ e = ROTATE_LEFT(e, 30);
+
+ b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(3) + SHA1_CONST(0); /* 3 */
+ d = ROTATE_LEFT(d, 30);
+
+ a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(4) + SHA1_CONST(0); /* 4 */
+ c = ROTATE_LEFT(c, 30);
+
+ e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(5) + SHA1_CONST(0); /* 5 */
+ b = ROTATE_LEFT(b, 30);
+
+ d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(6) + SHA1_CONST(0); /* 6 */
+ a = ROTATE_LEFT(a, 30);
+
+ c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(7) + SHA1_CONST(0); /* 7 */
+ e = ROTATE_LEFT(e, 30);
+
+ b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(8) + SHA1_CONST(0); /* 8 */
+ d = ROTATE_LEFT(d, 30);
+
+ a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(9) + SHA1_CONST(0); /* 9 */
+ c = ROTATE_LEFT(c, 30);
+
+ e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(10) + SHA1_CONST(0); /* 10 */
+ b = ROTATE_LEFT(b, 30);
+
+ d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(11) + SHA1_CONST(0); /* 11 */
+ a = ROTATE_LEFT(a, 30);
+
+ c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(12) + SHA1_CONST(0); /* 12 */
+ e = ROTATE_LEFT(e, 30);
+
+ b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(13) + SHA1_CONST(0); /* 13 */
+ d = ROTATE_LEFT(d, 30);
+
+ a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(14) + SHA1_CONST(0); /* 14 */
+ c = ROTATE_LEFT(c, 30);
+
+ e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(15) + SHA1_CONST(0); /* 15 */
+ b = ROTATE_LEFT(b, 30);
+
+ W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1); /* 16 */
+ d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(0) + SHA1_CONST(0);
+ a = ROTATE_LEFT(a, 30);
+
+ W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1); /* 17 */
+ c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(1) + SHA1_CONST(0);
+ e = ROTATE_LEFT(e, 30);
+
+ W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1); /* 18 */
+ b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(2) + SHA1_CONST(0);
+ d = ROTATE_LEFT(d, 30);
+
+ W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1); /* 19 */
+ a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(3) + SHA1_CONST(0);
+ c = ROTATE_LEFT(c, 30);
+
+ /* round 2 */
+ W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1); /* 20 */
+ e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(4) + SHA1_CONST(1);
+ b = ROTATE_LEFT(b, 30);
+
+ W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1); /* 21 */
+ d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(5) + SHA1_CONST(1);
+ a = ROTATE_LEFT(a, 30);
+
+ W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1); /* 22 */
+ c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(6) + SHA1_CONST(1);
+ e = ROTATE_LEFT(e, 30);
+
+ W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1); /* 23 */
+ b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(7) + SHA1_CONST(1);
+ d = ROTATE_LEFT(d, 30);
+
+ W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1); /* 24 */
+ a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(8) + SHA1_CONST(1);
+ c = ROTATE_LEFT(c, 30);
+
+ W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1); /* 25 */
+ e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(9) + SHA1_CONST(1);
+ b = ROTATE_LEFT(b, 30);
+
+ W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1); /* 26 */
+ d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(10) + SHA1_CONST(1);
+ a = ROTATE_LEFT(a, 30);
+
+ W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1); /* 27 */
+ c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(11) + SHA1_CONST(1);
+ e = ROTATE_LEFT(e, 30);
+
+ W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1); /* 28 */
+ b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(12) + SHA1_CONST(1);
+ d = ROTATE_LEFT(d, 30);
+
+ W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1); /* 29 */
+ a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(13) + SHA1_CONST(1);
+ c = ROTATE_LEFT(c, 30);
+
+ W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1); /* 30 */
+ e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(14) + SHA1_CONST(1);
+ b = ROTATE_LEFT(b, 30);
+
+ W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1); /* 31 */
+ d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(15) + SHA1_CONST(1);
+ a = ROTATE_LEFT(a, 30);
+
+ W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1); /* 32 */
+ c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(0) + SHA1_CONST(1);
+ e = ROTATE_LEFT(e, 30);
+
+ W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1); /* 33 */
+ b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(1) + SHA1_CONST(1);
+ d = ROTATE_LEFT(d, 30);
+
+ W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1); /* 34 */
+ a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(2) + SHA1_CONST(1);
+ c = ROTATE_LEFT(c, 30);
+
+ W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1); /* 35 */
+ e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(3) + SHA1_CONST(1);
+ b = ROTATE_LEFT(b, 30);
+
+ W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1); /* 36 */
+ d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(4) + SHA1_CONST(1);
+ a = ROTATE_LEFT(a, 30);
+
+ W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1); /* 37 */
+ c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(5) + SHA1_CONST(1);
+ e = ROTATE_LEFT(e, 30);
+
+ W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1); /* 38 */
+ b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(6) + SHA1_CONST(1);
+ d = ROTATE_LEFT(d, 30);
+
+ W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1); /* 39 */
+ a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(7) + SHA1_CONST(1);
+ c = ROTATE_LEFT(c, 30);
+
+ /* round 3 */
+ W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1); /* 40 */
+ e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(8) + SHA1_CONST(2);
+ b = ROTATE_LEFT(b, 30);
+
+ W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1); /* 41 */
+ d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(9) + SHA1_CONST(2);
+ a = ROTATE_LEFT(a, 30);
+
+ W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1); /* 42 */
+ c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(10) + SHA1_CONST(2);
+ e = ROTATE_LEFT(e, 30);
+
+ W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1); /* 43 */
+ b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(11) + SHA1_CONST(2);
+ d = ROTATE_LEFT(d, 30);
+
+ W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1); /* 44 */
+ a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(12) + SHA1_CONST(2);
+ c = ROTATE_LEFT(c, 30);
+
+ W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1); /* 45 */
+ e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(13) + SHA1_CONST(2);
+ b = ROTATE_LEFT(b, 30);
+
+ W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1); /* 46 */
+ d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(14) + SHA1_CONST(2);
+ a = ROTATE_LEFT(a, 30);
+
+ W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1); /* 47 */
+ c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(15) + SHA1_CONST(2);
+ e = ROTATE_LEFT(e, 30);
+
+ W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1); /* 48 */
+ b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(0) + SHA1_CONST(2);
+ d = ROTATE_LEFT(d, 30);
+
+ W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1); /* 49 */
+ a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(1) + SHA1_CONST(2);
+ c = ROTATE_LEFT(c, 30);
+
+ W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1); /* 50 */
+ e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(2) + SHA1_CONST(2);
+ b = ROTATE_LEFT(b, 30);
+
+ W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1); /* 51 */
+ d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(3) + SHA1_CONST(2);
+ a = ROTATE_LEFT(a, 30);
+
+ W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1); /* 52 */
+ c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(4) + SHA1_CONST(2);
+ e = ROTATE_LEFT(e, 30);
+
+ W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1); /* 53 */
+ b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(5) + SHA1_CONST(2);
+ d = ROTATE_LEFT(d, 30);
+
+ W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1); /* 54 */
+ a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(6) + SHA1_CONST(2);
+ c = ROTATE_LEFT(c, 30);
+
+ W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1); /* 55 */
+ e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(7) + SHA1_CONST(2);
+ b = ROTATE_LEFT(b, 30);
+
+ W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1); /* 56 */
+ d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(8) + SHA1_CONST(2);
+ a = ROTATE_LEFT(a, 30);
+
+ W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1); /* 57 */
+ c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(9) + SHA1_CONST(2);
+ e = ROTATE_LEFT(e, 30);
+
+ W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1); /* 58 */
+ b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(10) + SHA1_CONST(2);
+ d = ROTATE_LEFT(d, 30);
+
+ W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1); /* 59 */
+ a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(11) + SHA1_CONST(2);
+ c = ROTATE_LEFT(c, 30);
+
+ /* round 4 */
+ W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1); /* 60 */
+ e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(12) + SHA1_CONST(3);
+ b = ROTATE_LEFT(b, 30);
+
+ W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1); /* 61 */
+ d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(13) + SHA1_CONST(3);
+ a = ROTATE_LEFT(a, 30);
+
+ W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1); /* 62 */
+ c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(14) + SHA1_CONST(3);
+ e = ROTATE_LEFT(e, 30);
+
+ W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1); /* 63 */
+ b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(15) + SHA1_CONST(3);
+ d = ROTATE_LEFT(d, 30);
+
+ W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1); /* 64 */
+ a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(0) + SHA1_CONST(3);
+ c = ROTATE_LEFT(c, 30);
+
+ W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1); /* 65 */
+ e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(1) + SHA1_CONST(3);
+ b = ROTATE_LEFT(b, 30);
+
+ W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1); /* 66 */
+ d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(2) + SHA1_CONST(3);
+ a = ROTATE_LEFT(a, 30);
+
+ W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1); /* 67 */
+ c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(3) + SHA1_CONST(3);
+ e = ROTATE_LEFT(e, 30);
+
+ W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1); /* 68 */
+ b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(4) + SHA1_CONST(3);
+ d = ROTATE_LEFT(d, 30);
+
+ W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1); /* 69 */
+ a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(5) + SHA1_CONST(3);
+ c = ROTATE_LEFT(c, 30);
+
+ W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1); /* 70 */
+ e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(6) + SHA1_CONST(3);
+ b = ROTATE_LEFT(b, 30);
+
+ W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1); /* 71 */
+ d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(7) + SHA1_CONST(3);
+ a = ROTATE_LEFT(a, 30);
+
+ W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1); /* 72 */
+ c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(8) + SHA1_CONST(3);
+ e = ROTATE_LEFT(e, 30);
+
+ W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1); /* 73 */
+ b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(9) + SHA1_CONST(3);
+ d = ROTATE_LEFT(d, 30);
+
+ W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1); /* 74 */
+ a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(10) + SHA1_CONST(3);
+ c = ROTATE_LEFT(c, 30);
+
+ W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1); /* 75 */
+ e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(11) + SHA1_CONST(3);
+ b = ROTATE_LEFT(b, 30);
+
+ W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1); /* 76 */
+ d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(12) + SHA1_CONST(3);
+ a = ROTATE_LEFT(a, 30);
+
+ W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1); /* 77 */
+ c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(13) + SHA1_CONST(3);
+ e = ROTATE_LEFT(e, 30);
+
+ W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1); /* 78 */
+ b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(14) + SHA1_CONST(3);
+ d = ROTATE_LEFT(d, 30);
+
+ W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1); /* 79 */
+
+ ctx->state[0] += ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(15) +
+ SHA1_CONST(3);
+ ctx->state[1] += b;
+ ctx->state[2] += ROTATE_LEFT(c, 30);
+ ctx->state[3] += d;
+ ctx->state[4] += e;
+
+ /* zeroize sensitive information */
+ W(0) = W(1) = W(2) = W(3) = W(4) = W(5) = W(6) = W(7) = W(8) = 0;
+ W(9) = W(10) = W(11) = W(12) = W(13) = W(14) = W(15) = 0;
+}
+#endif /* !__amd64 */
+
+
+/*
+ * Encode()
+ *
+ * purpose: to convert a list of numbers from little endian to big endian
+ * input: uint8_t * : place to store the converted big endian numbers
+ * uint32_t * : place to get numbers to convert from
+ * size_t : the length of the input in bytes
+ * output: void
+ */
+
+static void
+Encode(uint8_t *_RESTRICT_KYWD output, const uint32_t *_RESTRICT_KYWD input,
+ size_t len)
+{
+ size_t i, j;
+
+ for (i = 0, j = 0; j < len; i++, j += 4) {
+ output[j] = (input[i] >> 24) & 0xff;
+ output[j + 1] = (input[i] >> 16) & 0xff;
+ output[j + 2] = (input[i] >> 8) & 0xff;
+ output[j + 3] = input[i] & 0xff;
+ }
+}
--- /dev/null
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+/*
+ * Copyright 2013 Saso Kiselkov. All rights reserved.
+ */
+
+/*
+ * The basic framework for this code came from the reference
+ * implementation for MD5. That implementation is Copyright (C)
+ * 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved.
+ *
+ * License to copy and use this software is granted provided that it
+ * is identified as the "RSA Data Security, Inc. MD5 Message-Digest
+ * Algorithm" in all material mentioning or referencing this software
+ * or this function.
+ *
+ * License is also granted to make and use derivative works provided
+ * that such works are identified as "derived from the RSA Data
+ * Security, Inc. MD5 Message-Digest Algorithm" in all material
+ * mentioning or referencing the derived work.
+ *
+ * RSA Data Security, Inc. makes no representations concerning either
+ * the merchantability of this software or the suitability of this
+ * software for any particular purpose. It is provided "as is"
+ * without express or implied warranty of any kind.
+ *
+ * These notices must be retained in any copies of any part of this
+ * documentation and/or software.
+ *
+ * NOTE: Cleaned-up and optimized, version of SHA2, based on the FIPS 180-2
+ * standard, available at
+ * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf
+ * Not as fast as one would like -- further optimizations are encouraged
+ * and appreciated.
+ */
+
+#include <sys/zfs_context.h>
+#define _SHA2_IMPL
+#include <sha2/sha2.h>
+#include <sha2/sha2_consts.h>
+
+#define _RESTRICT_KYWD
+
+#ifdef _LITTLE_ENDIAN
+#include <sys/byteorder.h>
+#define HAVE_HTONL
+#endif
+
+static void Encode(uint8_t *, uint32_t *, size_t);
+
+#if defined(__amd64)
+#define SHA256Transform(ctx, in) SHA256TransformBlocks((ctx), (in), 1)
+void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
+#else
+static void SHA256Transform(SHA2_CTX *, const uint8_t *);
+#endif /* __amd64 */
+
+static uint8_t PADDING[128] = { 0x80, /* all zeros */ };
+
+/* Ch and Maj are the basic SHA2 functions. */
+#define Ch(b, c, d) (((b) & (c)) ^ ((~b) & (d)))
+#define Maj(b, c, d) (((b) & (c)) ^ ((b) & (d)) ^ ((c) & (d)))
+
+/* Rotates x right n bits. */
+#define ROTR(x, n) \
+ (((x) >> (n)) | ((x) << ((sizeof (x) * NBBY)-(n))))
+
+/* Shift x right n bits */
+#define SHR(x, n) ((x) >> (n))
+
+/* SHA256 Functions */
+#define BIGSIGMA0_256(x) (ROTR((x), 2) ^ ROTR((x), 13) ^ ROTR((x), 22))
+#define BIGSIGMA1_256(x) (ROTR((x), 6) ^ ROTR((x), 11) ^ ROTR((x), 25))
+#define SIGMA0_256(x) (ROTR((x), 7) ^ ROTR((x), 18) ^ SHR((x), 3))
+#define SIGMA1_256(x) (ROTR((x), 17) ^ ROTR((x), 19) ^ SHR((x), 10))
+
+#define SHA256ROUND(a, b, c, d, e, f, g, h, i, w) \
+ T1 = h + BIGSIGMA1_256(e) + Ch(e, f, g) + SHA256_CONST(i) + w; \
+ d += T1; \
+ T2 = BIGSIGMA0_256(a) + Maj(a, b, c); \
+ h = T1 + T2
+
+/*
+ * sparc optimization:
+ *
+ * on the sparc, we can load big endian 32-bit data easily. note that
+ * special care must be taken to ensure the address is 32-bit aligned.
+ * in the interest of speed, we don't check to make sure, since
+ * careful programming can guarantee this for us.
+ */
+
+#if defined(_BIG_ENDIAN)
+#define LOAD_BIG_32(addr) (*(uint32_t *)(addr))
+#define LOAD_BIG_64(addr) (*(uint64_t *)(addr))
+
+#elif defined(HAVE_HTONL)
+#define LOAD_BIG_32(addr) htonl(*((uint32_t *)(addr)))
+#define LOAD_BIG_64(addr) htonll(*((uint64_t *)(addr)))
+
+#else
+/* little endian -- will work on big endian, but slowly */
+#define LOAD_BIG_32(addr) \
+ (((addr)[0] << 24) | ((addr)[1] << 16) | ((addr)[2] << 8) | (addr)[3])
+#define LOAD_BIG_64(addr) \
+ (((uint64_t)(addr)[0] << 56) | ((uint64_t)(addr)[1] << 48) | \
+ ((uint64_t)(addr)[2] << 40) | ((uint64_t)(addr)[3] << 32) | \
+ ((uint64_t)(addr)[4] << 24) | ((uint64_t)(addr)[5] << 16) | \
+ ((uint64_t)(addr)[6] << 8) | (uint64_t)(addr)[7])
+#endif /* _BIG_ENDIAN */
+
+
+#if !defined(__amd64)
+/* SHA256 Transform */
+
+static void
+SHA256Transform(SHA2_CTX *ctx, const uint8_t *blk)
+{
+ uint32_t a = ctx->state.s32[0];
+ uint32_t b = ctx->state.s32[1];
+ uint32_t c = ctx->state.s32[2];
+ uint32_t d = ctx->state.s32[3];
+ uint32_t e = ctx->state.s32[4];
+ uint32_t f = ctx->state.s32[5];
+ uint32_t g = ctx->state.s32[6];
+ uint32_t h = ctx->state.s32[7];
+
+ uint32_t w0, w1, w2, w3, w4, w5, w6, w7;
+ uint32_t w8, w9, w10, w11, w12, w13, w14, w15;
+ uint32_t T1, T2;
+
+ if ((uintptr_t)blk & 0x3) { /* not 4-byte aligned? */
+ bcopy(blk, ctx->buf_un.buf32, sizeof (ctx->buf_un.buf32));
+ blk = (uint8_t *)ctx->buf_un.buf32;
+ }
+
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w0 = LOAD_BIG_32(blk + 4 * 0);
+ SHA256ROUND(a, b, c, d, e, f, g, h, 0, w0);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w1 = LOAD_BIG_32(blk + 4 * 1);
+ SHA256ROUND(h, a, b, c, d, e, f, g, 1, w1);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w2 = LOAD_BIG_32(blk + 4 * 2);
+ SHA256ROUND(g, h, a, b, c, d, e, f, 2, w2);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w3 = LOAD_BIG_32(blk + 4 * 3);
+ SHA256ROUND(f, g, h, a, b, c, d, e, 3, w3);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w4 = LOAD_BIG_32(blk + 4 * 4);
+ SHA256ROUND(e, f, g, h, a, b, c, d, 4, w4);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w5 = LOAD_BIG_32(blk + 4 * 5);
+ SHA256ROUND(d, e, f, g, h, a, b, c, 5, w5);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w6 = LOAD_BIG_32(blk + 4 * 6);
+ SHA256ROUND(c, d, e, f, g, h, a, b, 6, w6);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w7 = LOAD_BIG_32(blk + 4 * 7);
+ SHA256ROUND(b, c, d, e, f, g, h, a, 7, w7);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w8 = LOAD_BIG_32(blk + 4 * 8);
+ SHA256ROUND(a, b, c, d, e, f, g, h, 8, w8);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w9 = LOAD_BIG_32(blk + 4 * 9);
+ SHA256ROUND(h, a, b, c, d, e, f, g, 9, w9);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w10 = LOAD_BIG_32(blk + 4 * 10);
+ SHA256ROUND(g, h, a, b, c, d, e, f, 10, w10);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w11 = LOAD_BIG_32(blk + 4 * 11);
+ SHA256ROUND(f, g, h, a, b, c, d, e, 11, w11);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w12 = LOAD_BIG_32(blk + 4 * 12);
+ SHA256ROUND(e, f, g, h, a, b, c, d, 12, w12);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w13 = LOAD_BIG_32(blk + 4 * 13);
+ SHA256ROUND(d, e, f, g, h, a, b, c, 13, w13);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w14 = LOAD_BIG_32(blk + 4 * 14);
+ SHA256ROUND(c, d, e, f, g, h, a, b, 14, w14);
+ /* LINTED E_BAD_PTR_CAST_ALIGN */
+ w15 = LOAD_BIG_32(blk + 4 * 15);
+ SHA256ROUND(b, c, d, e, f, g, h, a, 15, w15);
+
+ w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0;
+ SHA256ROUND(a, b, c, d, e, f, g, h, 16, w0);
+ w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1;
+ SHA256ROUND(h, a, b, c, d, e, f, g, 17, w1);
+ w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2;
+ SHA256ROUND(g, h, a, b, c, d, e, f, 18, w2);
+ w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3;
+ SHA256ROUND(f, g, h, a, b, c, d, e, 19, w3);
+ w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4;
+ SHA256ROUND(e, f, g, h, a, b, c, d, 20, w4);
+ w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5;
+ SHA256ROUND(d, e, f, g, h, a, b, c, 21, w5);
+ w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6;
+ SHA256ROUND(c, d, e, f, g, h, a, b, 22, w6);
+ w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7;
+ SHA256ROUND(b, c, d, e, f, g, h, a, 23, w7);
+ w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8;
+ SHA256ROUND(a, b, c, d, e, f, g, h, 24, w8);
+ w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9;
+ SHA256ROUND(h, a, b, c, d, e, f, g, 25, w9);
+ w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10;
+ SHA256ROUND(g, h, a, b, c, d, e, f, 26, w10);
+ w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11;
+ SHA256ROUND(f, g, h, a, b, c, d, e, 27, w11);
+ w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12;
+ SHA256ROUND(e, f, g, h, a, b, c, d, 28, w12);
+ w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13;
+ SHA256ROUND(d, e, f, g, h, a, b, c, 29, w13);
+ w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14;
+ SHA256ROUND(c, d, e, f, g, h, a, b, 30, w14);
+ w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15;
+ SHA256ROUND(b, c, d, e, f, g, h, a, 31, w15);
+
+ w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0;
+ SHA256ROUND(a, b, c, d, e, f, g, h, 32, w0);
+ w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1;
+ SHA256ROUND(h, a, b, c, d, e, f, g, 33, w1);
+ w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2;
+ SHA256ROUND(g, h, a, b, c, d, e, f, 34, w2);
+ w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3;
+ SHA256ROUND(f, g, h, a, b, c, d, e, 35, w3);
+ w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4;
+ SHA256ROUND(e, f, g, h, a, b, c, d, 36, w4);
+ w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5;
+ SHA256ROUND(d, e, f, g, h, a, b, c, 37, w5);
+ w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6;
+ SHA256ROUND(c, d, e, f, g, h, a, b, 38, w6);
+ w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7;
+ SHA256ROUND(b, c, d, e, f, g, h, a, 39, w7);
+ w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8;
+ SHA256ROUND(a, b, c, d, e, f, g, h, 40, w8);
+ w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9;
+ SHA256ROUND(h, a, b, c, d, e, f, g, 41, w9);
+ w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10;
+ SHA256ROUND(g, h, a, b, c, d, e, f, 42, w10);
+ w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11;
+ SHA256ROUND(f, g, h, a, b, c, d, e, 43, w11);
+ w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12;
+ SHA256ROUND(e, f, g, h, a, b, c, d, 44, w12);
+ w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13;
+ SHA256ROUND(d, e, f, g, h, a, b, c, 45, w13);
+ w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14;
+ SHA256ROUND(c, d, e, f, g, h, a, b, 46, w14);
+ w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15;
+ SHA256ROUND(b, c, d, e, f, g, h, a, 47, w15);
+
+ w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0;
+ SHA256ROUND(a, b, c, d, e, f, g, h, 48, w0);
+ w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1;
+ SHA256ROUND(h, a, b, c, d, e, f, g, 49, w1);
+ w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2;
+ SHA256ROUND(g, h, a, b, c, d, e, f, 50, w2);
+ w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3;
+ SHA256ROUND(f, g, h, a, b, c, d, e, 51, w3);
+ w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4;
+ SHA256ROUND(e, f, g, h, a, b, c, d, 52, w4);
+ w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5;
+ SHA256ROUND(d, e, f, g, h, a, b, c, 53, w5);
+ w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6;
+ SHA256ROUND(c, d, e, f, g, h, a, b, 54, w6);
+ w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7;
+ SHA256ROUND(b, c, d, e, f, g, h, a, 55, w7);
+ w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8;
+ SHA256ROUND(a, b, c, d, e, f, g, h, 56, w8);
+ w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9;
+ SHA256ROUND(h, a, b, c, d, e, f, g, 57, w9);
+ w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10;
+ SHA256ROUND(g, h, a, b, c, d, e, f, 58, w10);
+ w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11;
+ SHA256ROUND(f, g, h, a, b, c, d, e, 59, w11);
+ w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12;
+ SHA256ROUND(e, f, g, h, a, b, c, d, 60, w12);
+ w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13;
+ SHA256ROUND(d, e, f, g, h, a, b, c, 61, w13);
+ w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14;
+ SHA256ROUND(c, d, e, f, g, h, a, b, 62, w14);
+ w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15;
+ SHA256ROUND(b, c, d, e, f, g, h, a, 63, w15);
+
+ ctx->state.s32[0] += a;
+ ctx->state.s32[1] += b;
+ ctx->state.s32[2] += c;
+ ctx->state.s32[3] += d;
+ ctx->state.s32[4] += e;
+ ctx->state.s32[5] += f;
+ ctx->state.s32[6] += g;
+ ctx->state.s32[7] += h;
+}
+#endif /* !__amd64 */
+
+
+/*
+ * Encode()
+ *
+ * purpose: to convert a list of numbers from little endian to big endian
+ * input: uint8_t * : place to store the converted big endian numbers
+ * uint32_t * : place to get numbers to convert from
+ * size_t : the length of the input in bytes
+ * output: void
+ */
+
+static void
+Encode(uint8_t *_RESTRICT_KYWD output, uint32_t *_RESTRICT_KYWD input,
+ size_t len)
+{
+ size_t i, j;
+
+ for (i = 0, j = 0; j < len; i++, j += 4) {
+ output[j] = (input[i] >> 24) & 0xff;
+ output[j + 1] = (input[i] >> 16) & 0xff;
+ output[j + 2] = (input[i] >> 8) & 0xff;
+ output[j + 3] = input[i] & 0xff;
+ }
+}
+
+void
+SHA2Init(uint64_t mech, SHA2_CTX *ctx)
+{
+
+ switch (mech) {
+ case SHA256_MECH_INFO_TYPE:
+ case SHA256_HMAC_MECH_INFO_TYPE:
+ case SHA256_HMAC_GEN_MECH_INFO_TYPE:
+ ctx->state.s32[0] = 0x6a09e667U;
+ ctx->state.s32[1] = 0xbb67ae85U;
+ ctx->state.s32[2] = 0x3c6ef372U;
+ ctx->state.s32[3] = 0xa54ff53aU;
+ ctx->state.s32[4] = 0x510e527fU;
+ ctx->state.s32[5] = 0x9b05688cU;
+ ctx->state.s32[6] = 0x1f83d9abU;
+ ctx->state.s32[7] = 0x5be0cd19U;
+ break;
+ default:
+ cmn_err(CE_PANIC,
+ "sha2_init: failed to find a supported algorithm: 0x%x",
+ (uint32_t)mech);
+ }
+
+ ctx->algotype = (uint32_t)mech;
+ ctx->count.c64[0] = ctx->count.c64[1] = 0;
+}
+
+void
+SHA256Init(SHA256_CTX *ctx)
+{
+ SHA2Init(SHA256, ctx);
+}
+
+/*
+ * SHA2Update()
+ *
+ * purpose: continues an sha2 digest operation, using the message block
+ * to update the context.
+ * input: SHA2_CTX * : the context to update
+ * void * : the message block
+ * size_t : the length of the message block, in bytes
+ * output: void
+ */
+
+void
+SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len)
+{
+ uint32_t i, buf_index, buf_len, buf_limit;
+ const uint8_t *input = inptr;
+ uint32_t algotype = ctx->algotype;
+#if defined(__amd64)
+ uint32_t block_count;
+#endif /* !__amd64 */
+
+
+ /* check for noop */
+ if (input_len == 0)
+ return;
+
+ if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
+ buf_limit = 64;
+
+ /* compute number of bytes mod 64 */
+ buf_index = (ctx->count.c32[1] >> 3) & 0x3F;
+
+ /* update number of bits */
+ if ((ctx->count.c32[1] += (input_len << 3)) < (input_len << 3))
+ ctx->count.c32[0]++;
+
+ ctx->count.c32[0] += (input_len >> 29);
+
+ } else {
+ buf_limit = 128;
+
+ /* compute number of bytes mod 128 */
+ buf_index = (ctx->count.c64[1] >> 3) & 0x7F;
+
+ /* update number of bits */
+ if ((ctx->count.c64[1] += (input_len << 3)) < (input_len << 3))
+ ctx->count.c64[0]++;
+
+ ctx->count.c64[0] += (input_len >> 29);
+ }
+
+ buf_len = buf_limit - buf_index;
+
+ /* transform as many times as possible */
+ i = 0;
+ if (input_len >= buf_len) {
+
+ /*
+ * general optimization:
+ *
+ * only do initial bcopy() and SHA2Transform() if
+ * buf_index != 0. if buf_index == 0, we're just
+ * wasting our time doing the bcopy() since there
+ * wasn't any data left over from a previous call to
+ * SHA2Update().
+ */
+ if (buf_index) {
+ bcopy(input, &ctx->buf_un.buf8[buf_index], buf_len);
+ if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE)
+ SHA256Transform(ctx, ctx->buf_un.buf8);
+
+ i = buf_len;
+ }
+
+#if !defined(__amd64)
+ if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
+ for (; i + buf_limit - 1 < input_len; i += buf_limit) {
+ SHA256Transform(ctx, &input[i]);
+ }
+ }
+
+#else
+ if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
+ block_count = (input_len - i) >> 6;
+ if (block_count > 0) {
+ SHA256TransformBlocks(ctx, &input[i],
+ block_count);
+ i += block_count << 6;
+ }
+ }
+#endif /* !__amd64 */
+
+ /*
+ * general optimization:
+ *
+ * if i and input_len are the same, return now instead
+ * of calling bcopy(), since the bcopy() in this case
+ * will be an expensive noop.
+ */
+
+ if (input_len == i)
+ return;
+
+ buf_index = 0;
+ }
+
+ /* buffer remaining input */
+ bcopy(&input[i], &ctx->buf_un.buf8[buf_index], input_len - i);
+}
+
+
+/*
+ * SHA2Final()
+ *
+ * purpose: ends an sha2 digest operation, finalizing the message digest and
+ * zeroing the context.
+ * input: uchar_t * : a buffer to store the digest
+ * : The function actually uses void* because many
+ * : callers pass things other than uchar_t here.
+ * SHA2_CTX * : the context to finalize, save, and zero
+ * output: void
+ */
+
+void
+SHA2Final(void *digest, SHA2_CTX *ctx)
+{
+ uint8_t bitcount_be[sizeof (ctx->count.c32)];
+ uint32_t index;
+ uint32_t algotype = ctx->algotype;
+
+ if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
+ index = (ctx->count.c32[1] >> 3) & 0x3f;
+ Encode(bitcount_be, ctx->count.c32, sizeof (bitcount_be));
+ SHA2Update(ctx, PADDING, ((index < 56) ? 56 : 120) - index);
+ SHA2Update(ctx, bitcount_be, sizeof (bitcount_be));
+ Encode(digest, ctx->state.s32, sizeof (ctx->state.s32));
+ }
+
+ /* zeroize sensitive information */
+ bzero(ctx, sizeof (*ctx));
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/spi.h>
+#include <sys/crypto/sched_impl.h>
+
+#define CRYPTO_OPS_OFFSET(f) offsetof(crypto_ops_t, co_##f)
+#define CRYPTO_CIPHER_OFFSET(f) offsetof(crypto_cipher_ops_t, f)
+
+/*
+ * Encryption and decryption routines.
+ */
+
+/*
+ * The following are the possible returned values common to all the routines
+ * below. The applicability of some of these return values depends on the
+ * presence of the arguments.
+ *
+ * CRYPTO_SUCCESS: The operation completed successfully.
+ * CRYPTO_QUEUED: A request was submitted successfully. The callback
+ * routine will be called when the operation is done.
+ * CRYPTO_INVALID_MECH_NUMBER, CRYPTO_INVALID_MECH_PARAM, or
+ * CRYPTO_INVALID_MECH for problems with the 'mech'.
+ * CRYPTO_INVALID_DATA for bogus 'data'
+ * CRYPTO_HOST_MEMORY for failure to allocate memory to handle this work.
+ * CRYPTO_INVALID_CONTEXT: Not a valid context.
+ * CRYPTO_BUSY: Cannot process the request now. Schedule a
+ * crypto_bufcall(), or try later.
+ * CRYPTO_NOT_SUPPORTED and CRYPTO_MECH_NOT_SUPPORTED: No provider is
+ * capable of a function or a mechanism.
+ * CRYPTO_INVALID_KEY: bogus 'key' argument.
+ * CRYPTO_INVALID_PLAINTEXT: bogus 'plaintext' argument.
+ * CRYPTO_INVALID_CIPHERTEXT: bogus 'ciphertext' argument.
+ */
+
+/*
+ * crypto_cipher_init_prov()
+ *
+ * Arguments:
+ *
+ * pd: provider descriptor
+ * sid: session id
+ * mech: crypto_mechanism_t pointer.
+ * mech_type is a valid value previously returned by
+ * crypto_mech2id();
+ * When the mech's parameter is not NULL, its definition depends
+ * on the standard definition of the mechanism.
+ * key: pointer to a crypto_key_t structure.
+ * tmpl: a crypto_ctx_template_t, opaque template of a context of an
+ * encryption or decryption with the 'mech' using 'key'.
+ * 'tmpl' is created by a previous call to
+ * crypto_create_ctx_template().
+ * ctxp: Pointer to a crypto_context_t.
+ * func: CRYPTO_FG_ENCRYPT or CRYPTO_FG_DECRYPT.
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * This is a common function invoked internally by both
+ * crypto_encrypt_init() and crypto_decrypt_init().
+ * Asynchronously submits a request for, or synchronously performs the
+ * initialization of an encryption or a decryption operation.
+ * When possible and applicable, will internally use the pre-expanded key
+ * schedule from the context template, tmpl.
+ * When complete and successful, 'ctxp' will contain a crypto_context_t
+ * valid for later calls to encrypt_update() and encrypt_final(), or
+ * decrypt_update() and decrypt_final().
+ * The caller should hold a reference on the specified provider
+ * descriptor before calling this function.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+static int
+crypto_cipher_init_prov(crypto_provider_t provider, crypto_session_id_t sid,
+ crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_spi_ctx_template_t tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *crq, crypto_func_group_t func)
+{
+ int error;
+ crypto_ctx_t *ctx;
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd = provider;
+ kcf_provider_desc_t *real_provider = pd;
+
+ ASSERT(KCF_PROV_REFHELD(pd));
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ if (func == CRYPTO_FG_ENCRYPT) {
+ error = kcf_get_hardware_provider(mech->cm_type,
+ CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd,
+ &real_provider, CRYPTO_FG_ENCRYPT);
+ } else {
+ error = kcf_get_hardware_provider(mech->cm_type,
+ CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd,
+ &real_provider, CRYPTO_FG_DECRYPT);
+ }
+
+ if (error != CRYPTO_SUCCESS)
+ return (error);
+ }
+
+ /* Allocate and initialize the canonical context */
+ if ((ctx = kcf_new_ctx(crq, real_provider, sid)) == NULL) {
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+ return (CRYPTO_HOST_MEMORY);
+ }
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(crq, pd)) {
+ crypto_mechanism_t lmech;
+
+ lmech = *mech;
+ KCF_SET_PROVIDER_MECHNUM(mech->cm_type, real_provider, &lmech);
+
+ if (func == CRYPTO_FG_ENCRYPT)
+ error = KCF_PROV_ENCRYPT_INIT(real_provider, ctx,
+ &lmech, key, tmpl, KCF_SWFP_RHNDL(crq));
+ else {
+ ASSERT(func == CRYPTO_FG_DECRYPT);
+
+ error = KCF_PROV_DECRYPT_INIT(real_provider, ctx,
+ &lmech, key, tmpl, KCF_SWFP_RHNDL(crq));
+ }
+ KCF_PROV_INCRSTATS(pd, error);
+
+ goto done;
+ }
+
+ /* Check if context sharing is possible */
+ if (pd->pd_prov_type == CRYPTO_HW_PROVIDER &&
+ key->ck_format == CRYPTO_KEY_RAW &&
+ KCF_CAN_SHARE_OPSTATE(pd, mech->cm_type)) {
+ kcf_context_t *tctxp = (kcf_context_t *)ctx;
+ kcf_provider_desc_t *tpd = NULL;
+ crypto_mech_info_t *sinfo;
+
+ if ((kcf_get_sw_prov(mech->cm_type, &tpd, &tctxp->kc_mech,
+ B_FALSE) == CRYPTO_SUCCESS)) {
+ int tlen;
+
+ sinfo = &(KCF_TO_PROV_MECHINFO(tpd, mech->cm_type));
+ /*
+ * key->ck_length from the consumer is always in bits.
+ * We convert it to be in the same unit registered by
+ * the provider in order to do a comparison.
+ */
+ if (sinfo->cm_mech_flags & CRYPTO_KEYSIZE_UNIT_IN_BYTES)
+ tlen = key->ck_length >> 3;
+ else
+ tlen = key->ck_length;
+ /*
+ * Check if the software provider can support context
+ * sharing and support this key length.
+ */
+ if ((sinfo->cm_mech_flags & CRYPTO_CAN_SHARE_OPSTATE) &&
+ (tlen >= sinfo->cm_min_key_length) &&
+ (tlen <= sinfo->cm_max_key_length)) {
+ ctx->cc_flags = CRYPTO_INIT_OPSTATE;
+ tctxp->kc_sw_prov_desc = tpd;
+ } else
+ KCF_PROV_REFRELE(tpd);
+ }
+ }
+
+ if (func == CRYPTO_FG_ENCRYPT) {
+ KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_INIT, sid,
+ mech, key, NULL, NULL, tmpl);
+ } else {
+ ASSERT(func == CRYPTO_FG_DECRYPT);
+ KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_INIT, sid,
+ mech, key, NULL, NULL, tmpl);
+ }
+
+ error = kcf_submit_request(real_provider, ctx, crq, ¶ms,
+ B_FALSE);
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+
+done:
+ if ((error == CRYPTO_SUCCESS) || (error == CRYPTO_QUEUED))
+ *ctxp = (crypto_context_t)ctx;
+ else {
+ /* Release the hold done in kcf_new_ctx(). */
+ KCF_CONTEXT_REFRELE((kcf_context_t *)ctx->cc_framework_private);
+ }
+
+ return (error);
+}
+
+/*
+ * Same as crypto_cipher_init_prov(), but relies on the scheduler to pick
+ * an appropriate provider. See crypto_cipher_init_prov() comments for more
+ * details.
+ */
+static int
+crypto_cipher_init(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *crq, crypto_func_group_t func)
+{
+ int error;
+ kcf_mech_entry_t *me;
+ kcf_provider_desc_t *pd;
+ kcf_ctx_template_t *ctx_tmpl;
+ crypto_spi_ctx_template_t spi_ctx_tmpl = NULL;
+ kcf_prov_tried_t *list = NULL;
+
+retry:
+ /* pd is returned held */
+ if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error,
+ list, func, CHECK_RESTRICT(crq), 0)) == NULL) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ return (error);
+ }
+
+ /*
+ * For SW providers, check the validity of the context template
+ * It is very rare that the generation number mis-matches, so
+ * is acceptable to fail here, and let the consumer recover by
+ * freeing this tmpl and create a new one for the key and new SW
+ * provider
+ */
+ if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) &&
+ ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) {
+ if (ctx_tmpl->ct_generation != me->me_gen_swprov) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ KCF_PROV_REFRELE(pd);
+ return (CRYPTO_OLD_CTX_TEMPLATE);
+ } else {
+ spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl;
+ }
+ }
+
+ error = crypto_cipher_init_prov(pd, pd->pd_sid, mech, key,
+ spi_ctx_tmpl, ctxp, crq, func);
+ if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED &&
+ IS_RECOVERABLE(error)) {
+ /* Add pd to the linked list of providers tried. */
+ if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL)
+ goto retry;
+ }
+
+ if (list != NULL)
+ kcf_free_triedlist(list);
+
+ KCF_PROV_REFRELE(pd);
+ return (error);
+}
+
+/*
+ * crypto_encrypt_prov()
+ *
+ * Arguments:
+ * pd: provider descriptor
+ * sid: session id
+ * mech: crypto_mechanism_t pointer.
+ * mech_type is a valid value previously returned by
+ * crypto_mech2id();
+ * When the mech's parameter is not NULL, its definition depends
+ * on the standard definition of the mechanism.
+ * key: pointer to a crypto_key_t structure.
+ * plaintext: The message to be encrypted
+ * ciphertext: Storage for the encrypted message. The length needed
+ * depends on the mechanism, and the plaintext's size.
+ * tmpl: a crypto_ctx_template_t, opaque template of a context of an
+ * encryption with the 'mech' using 'key'. 'tmpl' is created by
+ * a previous call to crypto_create_ctx_template().
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs a
+ * single-part encryption of 'plaintext' with the mechanism 'mech', using
+ * the key 'key'.
+ * When complete and successful, 'ciphertext' will contain the encrypted
+ * message.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_encrypt_prov(crypto_provider_t provider, crypto_session_id_t sid,
+ crypto_mechanism_t *mech, crypto_data_t *plaintext, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_data_t *ciphertext,
+ crypto_call_req_t *crq)
+{
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd = provider;
+ kcf_provider_desc_t *real_provider = pd;
+ int error;
+
+ ASSERT(KCF_PROV_REFHELD(pd));
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ error = kcf_get_hardware_provider(mech->cm_type,
+ CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd,
+ &real_provider, CRYPTO_FG_ENCRYPT_ATOMIC);
+
+ if (error != CRYPTO_SUCCESS)
+ return (error);
+ }
+
+ KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, sid, mech, key,
+ plaintext, ciphertext, tmpl);
+
+ error = kcf_submit_request(real_provider, NULL, crq, ¶ms, B_FALSE);
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+
+ return (error);
+}
+
+/*
+ * Same as crypto_encrypt_prov(), but relies on the scheduler to pick
+ * a provider. See crypto_encrypt_prov() for more details.
+ */
+int
+crypto_encrypt(crypto_mechanism_t *mech, crypto_data_t *plaintext,
+ crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *ciphertext,
+ crypto_call_req_t *crq)
+{
+ int error;
+ kcf_mech_entry_t *me;
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd;
+ kcf_ctx_template_t *ctx_tmpl;
+ crypto_spi_ctx_template_t spi_ctx_tmpl = NULL;
+ kcf_prov_tried_t *list = NULL;
+
+retry:
+ /* pd is returned held */
+ if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error,
+ list, CRYPTO_FG_ENCRYPT_ATOMIC, CHECK_RESTRICT(crq),
+ plaintext->cd_length)) == NULL) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ return (error);
+ }
+
+ /*
+ * For SW providers, check the validity of the context template
+ * It is very rare that the generation number mis-matches, so
+ * is acceptable to fail here, and let the consumer recover by
+ * freeing this tmpl and create a new one for the key and new SW
+ * provider
+ */
+ if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) &&
+ ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) {
+ if (ctx_tmpl->ct_generation != me->me_gen_swprov) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ KCF_PROV_REFRELE(pd);
+ return (CRYPTO_OLD_CTX_TEMPLATE);
+ } else {
+ spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl;
+ }
+ }
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(crq, pd)) {
+ crypto_mechanism_t lmech;
+
+ lmech = *mech;
+ KCF_SET_PROVIDER_MECHNUM(mech->cm_type, pd, &lmech);
+
+ error = KCF_PROV_ENCRYPT_ATOMIC(pd, pd->pd_sid, &lmech, key,
+ plaintext, ciphertext, spi_ctx_tmpl, KCF_SWFP_RHNDL(crq));
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, pd->pd_sid,
+ mech, key, plaintext, ciphertext, spi_ctx_tmpl);
+ error = kcf_submit_request(pd, NULL, crq, ¶ms, B_FALSE);
+ }
+
+ if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED &&
+ IS_RECOVERABLE(error)) {
+ /* Add pd to the linked list of providers tried. */
+ if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL)
+ goto retry;
+ }
+
+ if (list != NULL)
+ kcf_free_triedlist(list);
+
+ KCF_PROV_REFRELE(pd);
+ return (error);
+}
+
+/*
+ * crypto_encrypt_init_prov()
+ *
+ * Calls crypto_cipher_init_prov() to initialize an encryption operation.
+ */
+int
+crypto_encrypt_init_prov(crypto_provider_t pd, crypto_session_id_t sid,
+ crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *crq)
+{
+ return (crypto_cipher_init_prov(pd, sid, mech, key, tmpl, ctxp, crq,
+ CRYPTO_FG_ENCRYPT));
+}
+
+/*
+ * crypto_encrypt_init()
+ *
+ * Calls crypto_cipher_init() to initialize an encryption operation
+ */
+int
+crypto_encrypt_init(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *crq)
+{
+ return (crypto_cipher_init(mech, key, tmpl, ctxp, crq,
+ CRYPTO_FG_ENCRYPT));
+}
+
+/*
+ * crypto_encrypt_update()
+ *
+ * Arguments:
+ * context: A crypto_context_t initialized by encrypt_init().
+ * plaintext: The message part to be encrypted
+ * ciphertext: Storage for the encrypted message part.
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs a
+ * part of an encryption operation.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_encrypt_update(crypto_context_t context, crypto_data_t *plaintext,
+ crypto_data_t *ciphertext, crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_ENCRYPT_UPDATE(pd, ctx, plaintext,
+ ciphertext, NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ return (error);
+ }
+
+ /* Check if we should use a software provider for small jobs */
+ if ((ctx->cc_flags & CRYPTO_USE_OPSTATE) && cr == NULL) {
+ if (plaintext->cd_length < kcf_ctx->kc_mech->me_threshold &&
+ kcf_ctx->kc_sw_prov_desc != NULL &&
+ KCF_IS_PROV_USABLE(kcf_ctx->kc_sw_prov_desc)) {
+ pd = kcf_ctx->kc_sw_prov_desc;
+ }
+ }
+
+ KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_UPDATE,
+ ctx->cc_session, NULL, NULL, plaintext, ciphertext, NULL);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+
+ return (error);
+}
+
+/*
+ * crypto_encrypt_final()
+ *
+ * Arguments:
+ * context: A crypto_context_t initialized by encrypt_init().
+ * ciphertext: Storage for the last part of encrypted message
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs the
+ * final part of an encryption operation.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_encrypt_final(crypto_context_t context, crypto_data_t *ciphertext,
+ crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_ENCRYPT_FINAL(pd, ctx, ciphertext, NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_FINAL,
+ ctx->cc_session, NULL, NULL, NULL, ciphertext, NULL);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ /* Release the hold done in kcf_new_ctx() during init step. */
+ KCF_CONTEXT_COND_RELEASE(error, kcf_ctx);
+ return (error);
+}
+
+/*
+ * crypto_decrypt_prov()
+ *
+ * Arguments:
+ * pd: provider descriptor
+ * sid: session id
+ * mech: crypto_mechanism_t pointer.
+ * mech_type is a valid value previously returned by
+ * crypto_mech2id();
+ * When the mech's parameter is not NULL, its definition depends
+ * on the standard definition of the mechanism.
+ * key: pointer to a crypto_key_t structure.
+ * ciphertext: The message to be encrypted
+ * plaintext: Storage for the encrypted message. The length needed
+ * depends on the mechanism, and the plaintext's size.
+ * tmpl: a crypto_ctx_template_t, opaque template of a context of an
+ * encryption with the 'mech' using 'key'. 'tmpl' is created by
+ * a previous call to crypto_create_ctx_template().
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs a
+ * single-part decryption of 'ciphertext' with the mechanism 'mech', using
+ * the key 'key'.
+ * When complete and successful, 'plaintext' will contain the decrypted
+ * message.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_decrypt_prov(crypto_provider_t provider, crypto_session_id_t sid,
+ crypto_mechanism_t *mech, crypto_data_t *ciphertext, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_data_t *plaintext,
+ crypto_call_req_t *crq)
+{
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd = provider;
+ kcf_provider_desc_t *real_provider = pd;
+ int rv;
+
+ ASSERT(KCF_PROV_REFHELD(pd));
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ rv = kcf_get_hardware_provider(mech->cm_type,
+ CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd,
+ &real_provider, CRYPTO_FG_DECRYPT_ATOMIC);
+
+ if (rv != CRYPTO_SUCCESS)
+ return (rv);
+ }
+
+ KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, sid, mech, key,
+ ciphertext, plaintext, tmpl);
+
+ rv = kcf_submit_request(real_provider, NULL, crq, ¶ms, B_FALSE);
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+
+ return (rv);
+}
+
+/*
+ * Same as crypto_decrypt_prov(), but relies on the KCF scheduler to
+ * choose a provider. See crypto_decrypt_prov() comments for more
+ * information.
+ */
+int
+crypto_decrypt(crypto_mechanism_t *mech, crypto_data_t *ciphertext,
+ crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *plaintext,
+ crypto_call_req_t *crq)
+{
+ int error;
+ kcf_mech_entry_t *me;
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd;
+ kcf_ctx_template_t *ctx_tmpl;
+ crypto_spi_ctx_template_t spi_ctx_tmpl = NULL;
+ kcf_prov_tried_t *list = NULL;
+
+retry:
+ /* pd is returned held */
+ if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error,
+ list, CRYPTO_FG_DECRYPT_ATOMIC, CHECK_RESTRICT(crq),
+ ciphertext->cd_length)) == NULL) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ return (error);
+ }
+
+ /*
+ * For SW providers, check the validity of the context template
+ * It is very rare that the generation number mis-matches, so
+ * is acceptable to fail here, and let the consumer recover by
+ * freeing this tmpl and create a new one for the key and new SW
+ * provider
+ */
+ if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) &&
+ ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) {
+ if (ctx_tmpl->ct_generation != me->me_gen_swprov) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ KCF_PROV_REFRELE(pd);
+ return (CRYPTO_OLD_CTX_TEMPLATE);
+ } else {
+ spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl;
+ }
+ }
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(crq, pd)) {
+ crypto_mechanism_t lmech;
+
+ lmech = *mech;
+ KCF_SET_PROVIDER_MECHNUM(mech->cm_type, pd, &lmech);
+
+ error = KCF_PROV_DECRYPT_ATOMIC(pd, pd->pd_sid, &lmech, key,
+ ciphertext, plaintext, spi_ctx_tmpl, KCF_SWFP_RHNDL(crq));
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, pd->pd_sid,
+ mech, key, ciphertext, plaintext, spi_ctx_tmpl);
+ error = kcf_submit_request(pd, NULL, crq, ¶ms, B_FALSE);
+ }
+
+ if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED &&
+ IS_RECOVERABLE(error)) {
+ /* Add pd to the linked list of providers tried. */
+ if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL)
+ goto retry;
+ }
+
+ if (list != NULL)
+ kcf_free_triedlist(list);
+
+ KCF_PROV_REFRELE(pd);
+ return (error);
+}
+
+/*
+ * crypto_decrypt_init_prov()
+ *
+ * Calls crypto_cipher_init_prov() to initialize a decryption operation
+ */
+int
+crypto_decrypt_init_prov(crypto_provider_t pd, crypto_session_id_t sid,
+ crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *crq)
+{
+ return (crypto_cipher_init_prov(pd, sid, mech, key, tmpl, ctxp, crq,
+ CRYPTO_FG_DECRYPT));
+}
+
+/*
+ * crypto_decrypt_init()
+ *
+ * Calls crypto_cipher_init() to initialize a decryption operation
+ */
+int
+crypto_decrypt_init(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *crq)
+{
+ return (crypto_cipher_init(mech, key, tmpl, ctxp, crq,
+ CRYPTO_FG_DECRYPT));
+}
+
+/*
+ * crypto_decrypt_update()
+ *
+ * Arguments:
+ * context: A crypto_context_t initialized by decrypt_init().
+ * ciphertext: The message part to be decrypted
+ * plaintext: Storage for the decrypted message part.
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs a
+ * part of an decryption operation.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_decrypt_update(crypto_context_t context, crypto_data_t *ciphertext,
+ crypto_data_t *plaintext, crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_DECRYPT_UPDATE(pd, ctx, ciphertext,
+ plaintext, NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ return (error);
+ }
+
+ /* Check if we should use a software provider for small jobs */
+ if ((ctx->cc_flags & CRYPTO_USE_OPSTATE) && cr == NULL) {
+ if (ciphertext->cd_length < kcf_ctx->kc_mech->me_threshold &&
+ kcf_ctx->kc_sw_prov_desc != NULL &&
+ KCF_IS_PROV_USABLE(kcf_ctx->kc_sw_prov_desc)) {
+ pd = kcf_ctx->kc_sw_prov_desc;
+ }
+ }
+
+ KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_UPDATE,
+ ctx->cc_session, NULL, NULL, ciphertext, plaintext, NULL);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+
+ return (error);
+}
+
+/*
+ * crypto_decrypt_final()
+ *
+ * Arguments:
+ * context: A crypto_context_t initialized by decrypt_init().
+ * plaintext: Storage for the last part of the decrypted message
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs the
+ * final part of a decryption operation.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_decrypt_final(crypto_context_t context, crypto_data_t *plaintext,
+ crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_DECRYPT_FINAL(pd, ctx, plaintext,
+ NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_FINAL,
+ ctx->cc_session, NULL, NULL, NULL, plaintext, NULL);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ /* Release the hold done in kcf_new_ctx() during init step. */
+ KCF_CONTEXT_COND_RELEASE(error, kcf_ctx);
+ return (error);
+}
+
+/*
+ * See comments for crypto_encrypt_update().
+ */
+int
+crypto_encrypt_single(crypto_context_t context, crypto_data_t *plaintext,
+ crypto_data_t *ciphertext, crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_ENCRYPT(pd, ctx, plaintext,
+ ciphertext, NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_SINGLE, pd->pd_sid,
+ NULL, NULL, plaintext, ciphertext, NULL);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ /* Release the hold done in kcf_new_ctx() during init step. */
+ KCF_CONTEXT_COND_RELEASE(error, kcf_ctx);
+ return (error);
+}
+
+/*
+ * See comments for crypto_decrypt_update().
+ */
+int
+crypto_decrypt_single(crypto_context_t context, crypto_data_t *ciphertext,
+ crypto_data_t *plaintext, crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_DECRYPT(pd, ctx, ciphertext,
+ plaintext, NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_SINGLE, pd->pd_sid,
+ NULL, NULL, ciphertext, plaintext, NULL);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ /* Release the hold done in kcf_new_ctx() during init step. */
+ KCF_CONTEXT_COND_RELEASE(error, kcf_ctx);
+ return (error);
+}
+
+#if defined(_KERNEL) && defined(HAVE_SPL)
+EXPORT_SYMBOL(crypto_cipher_init_prov);
+EXPORT_SYMBOL(crypto_cipher_init);
+EXPORT_SYMBOL(crypto_encrypt_prov);
+EXPORT_SYMBOL(crypto_encrypt);
+EXPORT_SYMBOL(crypto_encrypt_init_prov);
+EXPORT_SYMBOL(crypto_encrypt_init);
+EXPORT_SYMBOL(crypto_encrypt_update);
+EXPORT_SYMBOL(crypto_encrypt_final);
+EXPORT_SYMBOL(crypto_decrypt_prov);
+EXPORT_SYMBOL(crypto_decrypt);
+EXPORT_SYMBOL(crypto_decrypt_init_prov);
+EXPORT_SYMBOL(crypto_decrypt_init);
+EXPORT_SYMBOL(crypto_decrypt_update);
+EXPORT_SYMBOL(crypto_decrypt_final);
+EXPORT_SYMBOL(crypto_encrypt_single);
+EXPORT_SYMBOL(crypto_decrypt_single);
+#endif
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/spi.h>
+#include <sys/crypto/sched_impl.h>
+
+/*
+ * Crypto contexts manipulation routines
+ */
+
+/*
+ * crypto_create_ctx_template()
+ *
+ * Arguments:
+ *
+ * mech: crypto_mechanism_t pointer.
+ * mech_type is a valid value previously returned by
+ * crypto_mech2id();
+ * When the mech's parameter is not NULL, its definition depends
+ * on the standard definition of the mechanism.
+ * key: pointer to a crypto_key_t structure.
+ * ptmpl: a storage for the opaque crypto_ctx_template_t, allocated and
+ * initialized by the software provider this routine is
+ * dispatched to.
+ * kmflag: KM_SLEEP/KM_NOSLEEP mem. alloc. flag.
+ *
+ * Description:
+ * Redirects the call to the software provider of the specified
+ * mechanism. That provider will allocate and pre-compute/pre-expand
+ * the context template, reusable by later calls to crypto_xxx_init().
+ * The size and address of that provider context template are stored
+ * in an internal structure, kcf_ctx_template_t. The address of that
+ * structure is given back to the caller in *ptmpl.
+ *
+ * Context:
+ * Process or interrupt.
+ *
+ * Returns:
+ * CRYPTO_SUCCESS when the context template is successfully created.
+ * CRYPTO_HOST_MEMEORY: mem alloc failure
+ * CRYPTO_ARGUMENTS_BAD: NULL storage for the ctx template.
+ * RYPTO_MECHANISM_INVALID: invalid mechanism 'mech'.
+ */
+int
+crypto_create_ctx_template(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t *ptmpl, int kmflag)
+{
+ int error;
+ kcf_mech_entry_t *me;
+ kcf_provider_desc_t *pd;
+ kcf_ctx_template_t *ctx_tmpl;
+ crypto_mechanism_t prov_mech;
+
+ /* A few args validation */
+
+ if (ptmpl == NULL)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ if (mech == NULL)
+ return (CRYPTO_MECHANISM_INVALID);
+
+ error = kcf_get_sw_prov(mech->cm_type, &pd, &me, B_TRUE);
+ if (error != CRYPTO_SUCCESS)
+ return (error);
+
+ if ((ctx_tmpl = (kcf_ctx_template_t *)kmem_alloc(
+ sizeof (kcf_ctx_template_t), kmflag)) == NULL) {
+ KCF_PROV_REFRELE(pd);
+ return (CRYPTO_HOST_MEMORY);
+ }
+
+ /* Pass a mechtype that the provider understands */
+ prov_mech.cm_type = KCF_TO_PROV_MECHNUM(pd, mech->cm_type);
+ prov_mech.cm_param = mech->cm_param;
+ prov_mech.cm_param_len = mech->cm_param_len;
+
+ error = KCF_PROV_CREATE_CTX_TEMPLATE(pd, &prov_mech, key,
+ &(ctx_tmpl->ct_prov_tmpl), &(ctx_tmpl->ct_size), KCF_RHNDL(kmflag));
+
+ if (error == CRYPTO_SUCCESS) {
+ ctx_tmpl->ct_generation = me->me_gen_swprov;
+ *ptmpl = ctx_tmpl;
+ } else {
+ kmem_free(ctx_tmpl, sizeof (kcf_ctx_template_t));
+ }
+ KCF_PROV_REFRELE(pd);
+
+ return (error);
+}
+
+/*
+ * crypto_destroy_ctx_template()
+ *
+ * Arguments:
+ *
+ * tmpl: an opaque crypto_ctx_template_t previously created by
+ * crypto_create_ctx_template()
+ *
+ * Description:
+ * Frees the inbedded crypto_spi_ctx_template_t, then the
+ * kcf_ctx_template_t.
+ *
+ * Context:
+ * Process or interrupt.
+ *
+ */
+void
+crypto_destroy_ctx_template(crypto_ctx_template_t tmpl)
+{
+ kcf_ctx_template_t *ctx_tmpl = (kcf_ctx_template_t *)tmpl;
+
+ if (ctx_tmpl == NULL)
+ return;
+
+ ASSERT(ctx_tmpl->ct_prov_tmpl != NULL);
+
+ bzero(ctx_tmpl->ct_prov_tmpl, ctx_tmpl->ct_size);
+ kmem_free(ctx_tmpl->ct_prov_tmpl, ctx_tmpl->ct_size);
+ kmem_free(ctx_tmpl, sizeof (kcf_ctx_template_t));
+}
+
+#if defined(_KERNEL) && defined(HAVE_SPL)
+EXPORT_SYMBOL(crypto_create_ctx_template);
+EXPORT_SYMBOL(crypto_destroy_ctx_template);
+#endif
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/spi.h>
+#include <sys/crypto/sched_impl.h>
+
+#define CRYPTO_OPS_OFFSET(f) offsetof(crypto_ops_t, co_##f)
+#define CRYPTO_DIGEST_OFFSET(f) offsetof(crypto_digest_ops_t, f)
+
+/*
+ * Message digest routines
+ */
+
+/*
+ * The following are the possible returned values common to all the routines
+ * below. The applicability of some of these return values depends on the
+ * presence of the arguments.
+ *
+ * CRYPTO_SUCCESS: The operation completed successfully.
+ * CRYPTO_QUEUED: A request was submitted successfully. The callback
+ * routine will be called when the operation is done.
+ * CRYPTO_MECHANISM_INVALID or CRYPTO_INVALID_MECH_PARAM
+ * for problems with the 'mech'.
+ * CRYPTO_INVALID_DATA for bogus 'data'
+ * CRYPTO_HOST_MEMORY for failure to allocate memory to handle this work.
+ * CRYPTO_INVALID_CONTEXT: Not a valid context.
+ * CRYPTO_BUSY: Cannot process the request now. Schedule a
+ * crypto_bufcall(), or try later.
+ * CRYPTO_NOT_SUPPORTED and CRYPTO_MECH_NOT_SUPPORTED:
+ * No provider is capable of a function or a mechanism.
+ */
+
+
+/*
+ * crypto_digest_prov()
+ *
+ * Arguments:
+ * pd: pointer to the descriptor of the provider to use for this
+ * operation.
+ * sid: provider session id.
+ * mech: crypto_mechanism_t pointer.
+ * mech_type is a valid value previously returned by
+ * crypto_mech2id();
+ * When the mech's parameter is not NULL, its definition depends
+ * on the standard definition of the mechanism.
+ * data: The message to be digested.
+ * digest: Storage for the digest. The length needed depends on the
+ * mechanism.
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs the
+ * digesting operation of 'data' on the specified
+ * provider with the specified session.
+ * When complete and successful, 'digest' will contain the digest value.
+ * The caller should hold a reference on the specified provider
+ * descriptor before calling this function.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_digest_prov(crypto_provider_t provider, crypto_session_id_t sid,
+ crypto_mechanism_t *mech, crypto_data_t *data, crypto_data_t *digest,
+ crypto_call_req_t *crq)
+{
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd = provider;
+ kcf_provider_desc_t *real_provider = pd;
+ int rv;
+
+ ASSERT(KCF_PROV_REFHELD(pd));
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ rv = kcf_get_hardware_provider(mech->cm_type,
+ CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq),
+ pd, &real_provider, CRYPTO_FG_DIGEST_ATOMIC);
+
+ if (rv != CRYPTO_SUCCESS)
+ return (rv);
+ }
+ KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, sid, mech, NULL,
+ data, digest);
+
+ /* no crypto context to carry between multiple parts. */
+ rv = kcf_submit_request(real_provider, NULL, crq, ¶ms, B_FALSE);
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+
+ return (rv);
+}
+
+
+/*
+ * Same as crypto_digest_prov(), but relies on the KCF scheduler to
+ * choose a provider. See crypto_digest_prov() comments for more information.
+ */
+int
+crypto_digest(crypto_mechanism_t *mech, crypto_data_t *data,
+ crypto_data_t *digest, crypto_call_req_t *crq)
+{
+ int error;
+ kcf_provider_desc_t *pd;
+ kcf_req_params_t params;
+ kcf_prov_tried_t *list = NULL;
+
+retry:
+ /* The pd is returned held */
+ if ((pd = kcf_get_mech_provider(mech->cm_type, NULL, &error, list,
+ CRYPTO_FG_DIGEST_ATOMIC, CHECK_RESTRICT(crq),
+ data->cd_length)) == NULL) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ return (error);
+ }
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(crq, pd)) {
+ crypto_mechanism_t lmech;
+
+ lmech = *mech;
+ KCF_SET_PROVIDER_MECHNUM(mech->cm_type, pd, &lmech);
+ error = KCF_PROV_DIGEST_ATOMIC(pd, pd->pd_sid, &lmech, data,
+ digest, KCF_SWFP_RHNDL(crq));
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ if (pd->pd_prov_type == CRYPTO_HW_PROVIDER &&
+ (pd->pd_flags & CRYPTO_HASH_NO_UPDATE) &&
+ (data->cd_length > pd->pd_hash_limit)) {
+ error = CRYPTO_BUFFER_TOO_BIG;
+ } else {
+ KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_ATOMIC,
+ pd->pd_sid, mech, NULL, data, digest);
+
+ /* no crypto context to carry between multiple parts. */
+ error = kcf_submit_request(pd, NULL, crq, ¶ms,
+ B_FALSE);
+ }
+ }
+
+ if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED &&
+ IS_RECOVERABLE(error)) {
+ /* Add pd to the linked list of providers tried. */
+ if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL)
+ goto retry;
+ }
+
+ if (list != NULL)
+ kcf_free_triedlist(list);
+
+ KCF_PROV_REFRELE(pd);
+ return (error);
+}
+
+/*
+ * crypto_digest_init_prov()
+ *
+ * pd: pointer to the descriptor of the provider to use for this
+ * operation.
+ * sid: provider session id.
+ * mech: crypto_mechanism_t pointer.
+ * mech_type is a valid value previously returned by
+ * crypto_mech2id();
+ * When the mech's parameter is not NULL, its definition depends
+ * on the standard definition of the mechanism.
+ * ctxp: Pointer to a crypto_context_t.
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs the
+ * initialization of a message digest operation on the specified
+ * provider with the specified session.
+ * When complete and successful, 'ctxp' will contain a crypto_context_t
+ * valid for later calls to digest_update() and digest_final().
+ * The caller should hold a reference on the specified provider
+ * descriptor before calling this function.
+ */
+int
+crypto_digest_init_prov(crypto_provider_t provider, crypto_session_id_t sid,
+ crypto_mechanism_t *mech, crypto_context_t *ctxp, crypto_call_req_t *crq)
+{
+ int error;
+ crypto_ctx_t *ctx;
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd = provider;
+ kcf_provider_desc_t *real_provider = pd;
+
+ ASSERT(KCF_PROV_REFHELD(pd));
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ error = kcf_get_hardware_provider(mech->cm_type,
+ CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd,
+ &real_provider, CRYPTO_FG_DIGEST);
+
+ if (error != CRYPTO_SUCCESS)
+ return (error);
+ }
+
+ /* Allocate and initialize the canonical context */
+ if ((ctx = kcf_new_ctx(crq, real_provider, sid)) == NULL) {
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+ return (CRYPTO_HOST_MEMORY);
+ }
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(crq, pd)) {
+ crypto_mechanism_t lmech;
+
+ lmech = *mech;
+ KCF_SET_PROVIDER_MECHNUM(mech->cm_type, real_provider, &lmech);
+ error = KCF_PROV_DIGEST_INIT(real_provider, ctx, &lmech,
+ KCF_SWFP_RHNDL(crq));
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_INIT, sid,
+ mech, NULL, NULL, NULL);
+ error = kcf_submit_request(real_provider, ctx, crq, ¶ms,
+ B_FALSE);
+ }
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+
+ if ((error == CRYPTO_SUCCESS) || (error == CRYPTO_QUEUED))
+ *ctxp = (crypto_context_t)ctx;
+ else {
+ /* Release the hold done in kcf_new_ctx(). */
+ KCF_CONTEXT_REFRELE((kcf_context_t *)ctx->cc_framework_private);
+ }
+
+ return (error);
+}
+
+/*
+ * Same as crypto_digest_init_prov(), but relies on the KCF scheduler
+ * to choose a provider. See crypto_digest_init_prov() comments for
+ * more information.
+ */
+int
+crypto_digest_init(crypto_mechanism_t *mech, crypto_context_t *ctxp,
+ crypto_call_req_t *crq)
+{
+ int error;
+ kcf_provider_desc_t *pd;
+ kcf_prov_tried_t *list = NULL;
+
+retry:
+ /* The pd is returned held */
+ if ((pd = kcf_get_mech_provider(mech->cm_type, NULL, &error,
+ list, CRYPTO_FG_DIGEST, CHECK_RESTRICT(crq), 0)) == NULL) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ return (error);
+ }
+
+ if (pd->pd_prov_type == CRYPTO_HW_PROVIDER &&
+ (pd->pd_flags & CRYPTO_HASH_NO_UPDATE)) {
+ /*
+ * The hardware provider has limited digest support.
+ * So, we fallback early here to using a software provider.
+ *
+ * XXX - need to enhance to do the fallback later in
+ * crypto_digest_update() if the size of accumulated input data
+ * exceeds the maximum size digestable by hardware provider.
+ */
+ error = CRYPTO_BUFFER_TOO_BIG;
+ } else {
+ error = crypto_digest_init_prov(pd, pd->pd_sid,
+ mech, ctxp, crq);
+ }
+
+ if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED &&
+ IS_RECOVERABLE(error)) {
+ /* Add pd to the linked list of providers tried. */
+ if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL)
+ goto retry;
+ }
+
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ KCF_PROV_REFRELE(pd);
+ return (error);
+}
+
+/*
+ * crypto_digest_update()
+ *
+ * Arguments:
+ * context: A crypto_context_t initialized by digest_init().
+ * data: The part of message to be digested.
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs a
+ * part of a message digest operation.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_digest_update(crypto_context_t context, crypto_data_t *data,
+ crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_DIGEST_UPDATE(pd, ctx, data, NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_UPDATE,
+ ctx->cc_session, NULL, NULL, data, NULL);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ return (error);
+}
+
+/*
+ * crypto_digest_final()
+ *
+ * Arguments:
+ * context: A crypto_context_t initialized by digest_init().
+ * digest: The storage for the digest.
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs the
+ * final part of a message digest operation.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_digest_final(crypto_context_t context, crypto_data_t *digest,
+ crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_DIGEST_FINAL(pd, ctx, digest, NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_FINAL,
+ ctx->cc_session, NULL, NULL, NULL, digest);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ /* Release the hold done in kcf_new_ctx() during init step. */
+ KCF_CONTEXT_COND_RELEASE(error, kcf_ctx);
+ return (error);
+}
+
+/*
+ * Performs a digest update on the specified key. Note that there is
+ * no k-API crypto_digest_key() equivalent of this function.
+ */
+int
+crypto_digest_key_prov(crypto_context_t context, crypto_key_t *key,
+ crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_DIGEST_KEY(pd, ctx, key, NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_DIGEST_KEY,
+ ctx->cc_session, NULL, key, NULL, NULL);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ return (error);
+}
+
+/*
+ * See comments for crypto_digest_update() and crypto_digest_final().
+ */
+int
+crypto_digest_single(crypto_context_t context, crypto_data_t *data,
+ crypto_data_t *digest, crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_DIGEST(pd, ctx, data, digest, NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_SINGLE, pd->pd_sid,
+ NULL, NULL, data, digest);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ /* Release the hold done in kcf_new_ctx() during init step. */
+ KCF_CONTEXT_COND_RELEASE(error, kcf_ctx);
+ return (error);
+}
+
+#if defined(_KERNEL) && defined(HAVE_SPL)
+EXPORT_SYMBOL(crypto_digest_prov);
+EXPORT_SYMBOL(crypto_digest);
+EXPORT_SYMBOL(crypto_digest_init_prov);
+EXPORT_SYMBOL(crypto_digest_init);
+EXPORT_SYMBOL(crypto_digest_update);
+EXPORT_SYMBOL(crypto_digest_final);
+EXPORT_SYMBOL(crypto_digest_key_prov);
+EXPORT_SYMBOL(crypto_digest_single);
+#endif
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/spi.h>
+#include <sys/crypto/sched_impl.h>
+
+#define CRYPTO_OPS_OFFSET(f) offsetof(crypto_ops_t, co_##f)
+#define CRYPTO_MAC_OFFSET(f) offsetof(crypto_mac_ops_t, f)
+
+/*
+ * Message authentication codes routines.
+ */
+
+/*
+ * The following are the possible returned values common to all the routines
+ * below. The applicability of some of these return values depends on the
+ * presence of the arguments.
+ *
+ * CRYPTO_SUCCESS: The operation completed successfully.
+ * CRYPTO_QUEUED: A request was submitted successfully. The callback
+ * routine will be called when the operation is done.
+ * CRYPTO_INVALID_MECH_NUMBER, CRYPTO_INVALID_MECH_PARAM, or
+ * CRYPTO_INVALID_MECH for problems with the 'mech'.
+ * CRYPTO_INVALID_DATA for bogus 'data'
+ * CRYPTO_HOST_MEMORY for failure to allocate memory to handle this work.
+ * CRYPTO_INVALID_CONTEXT: Not a valid context.
+ * CRYPTO_BUSY: Cannot process the request now. Schedule a
+ * crypto_bufcall(), or try later.
+ * CRYPTO_NOT_SUPPORTED and CRYPTO_MECH_NOT_SUPPORTED: No provider is
+ * capable of a function or a mechanism.
+ * CRYPTO_INVALID_KEY: bogus 'key' argument.
+ * CRYPTO_INVALID_MAC: bogus 'mac' argument.
+ */
+
+/*
+ * crypto_mac_prov()
+ *
+ * Arguments:
+ * mech: crypto_mechanism_t pointer.
+ * mech_type is a valid value previously returned by
+ * crypto_mech2id();
+ * When the mech's parameter is not NULL, its definition depends
+ * on the standard definition of the mechanism.
+ * key: pointer to a crypto_key_t structure.
+ * data: The message to compute the MAC for.
+ * mac: Storage for the MAC. The length needed depends on the mechanism.
+ * tmpl: a crypto_ctx_template_t, opaque template of a context of a
+ * MAC with the 'mech' using 'key'. 'tmpl' is created by
+ * a previous call to crypto_create_ctx_template().
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs a
+ * single-part message authentication of 'data' with the mechanism
+ * 'mech', using * the key 'key', on the specified provider with
+ * the specified session id.
+ * When complete and successful, 'mac' will contain the message
+ * authentication code.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'crq'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_mac_prov(crypto_provider_t provider, crypto_session_id_t sid,
+ crypto_mechanism_t *mech, crypto_data_t *data, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_data_t *mac, crypto_call_req_t *crq)
+{
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd = provider;
+ kcf_provider_desc_t *real_provider = pd;
+ int rv;
+
+ ASSERT(KCF_PROV_REFHELD(pd));
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ rv = kcf_get_hardware_provider(mech->cm_type,
+ CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd,
+ &real_provider, CRYPTO_FG_MAC_ATOMIC);
+
+ if (rv != CRYPTO_SUCCESS)
+ return (rv);
+ }
+
+ KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, sid, mech, key,
+ data, mac, tmpl);
+ rv = kcf_submit_request(real_provider, NULL, crq, ¶ms, B_FALSE);
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+
+ return (rv);
+}
+
+/*
+ * Same as crypto_mac_prov(), but relies on the KCF scheduler to choose
+ * a provider. See crypto_mac() comments for more information.
+ */
+int
+crypto_mac(crypto_mechanism_t *mech, crypto_data_t *data,
+ crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *mac,
+ crypto_call_req_t *crq)
+{
+ int error;
+ kcf_mech_entry_t *me;
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd;
+ kcf_ctx_template_t *ctx_tmpl;
+ crypto_spi_ctx_template_t spi_ctx_tmpl = NULL;
+ kcf_prov_tried_t *list = NULL;
+
+retry:
+ /* The pd is returned held */
+ if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error,
+ list, CRYPTO_FG_MAC_ATOMIC, CHECK_RESTRICT(crq),
+ data->cd_length)) == NULL) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ return (error);
+ }
+
+ /*
+ * For SW providers, check the validity of the context template
+ * It is very rare that the generation number mis-matches, so
+ * is acceptable to fail here, and let the consumer recover by
+ * freeing this tmpl and create a new one for the key and new SW
+ * provider
+ */
+ if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) &&
+ ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) {
+ if (ctx_tmpl->ct_generation != me->me_gen_swprov) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ KCF_PROV_REFRELE(pd);
+ return (CRYPTO_OLD_CTX_TEMPLATE);
+ } else {
+ spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl;
+ }
+ }
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(crq, pd)) {
+ crypto_mechanism_t lmech;
+
+ lmech = *mech;
+ KCF_SET_PROVIDER_MECHNUM(mech->cm_type, pd, &lmech);
+
+ error = KCF_PROV_MAC_ATOMIC(pd, pd->pd_sid, &lmech, key, data,
+ mac, spi_ctx_tmpl, KCF_SWFP_RHNDL(crq));
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ if (pd->pd_prov_type == CRYPTO_HW_PROVIDER &&
+ (pd->pd_flags & CRYPTO_HASH_NO_UPDATE) &&
+ (data->cd_length > pd->pd_hash_limit)) {
+ /*
+ * XXX - We need a check to see if this is indeed
+ * a HMAC. So far, all kernel clients use
+ * this interface only for HMAC. So, this is fine
+ * for now.
+ */
+ error = CRYPTO_BUFFER_TOO_BIG;
+ } else {
+ KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_ATOMIC,
+ pd->pd_sid, mech, key, data, mac, spi_ctx_tmpl);
+
+ error = kcf_submit_request(pd, NULL, crq, ¶ms,
+ KCF_ISDUALREQ(crq));
+ }
+ }
+
+ if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED &&
+ IS_RECOVERABLE(error)) {
+ /* Add pd to the linked list of providers tried. */
+ if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL)
+ goto retry;
+ }
+
+ if (list != NULL)
+ kcf_free_triedlist(list);
+
+ KCF_PROV_REFRELE(pd);
+ return (error);
+}
+
+/*
+ * Single part operation to compute the MAC corresponding to the specified
+ * 'data' and to verify that it matches the MAC specified by 'mac'.
+ * The other arguments are the same as the function crypto_mac_prov().
+ */
+int
+crypto_mac_verify_prov(crypto_provider_t provider, crypto_session_id_t sid,
+ crypto_mechanism_t *mech, crypto_data_t *data, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_data_t *mac, crypto_call_req_t *crq)
+{
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd = provider;
+ kcf_provider_desc_t *real_provider = pd;
+ int rv;
+
+ ASSERT(KCF_PROV_REFHELD(pd));
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ rv = kcf_get_hardware_provider(mech->cm_type,
+ CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd,
+ &real_provider, CRYPTO_FG_MAC_ATOMIC);
+
+ if (rv != CRYPTO_SUCCESS)
+ return (rv);
+ }
+
+ KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_MAC_VERIFY_ATOMIC, sid, mech,
+ key, data, mac, tmpl);
+ rv = kcf_submit_request(real_provider, NULL, crq, ¶ms, B_FALSE);
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+
+ return (rv);
+}
+
+/*
+ * Same as crypto_mac_verify_prov(), but relies on the KCF scheduler to choose
+ * a provider. See crypto_mac_verify_prov() comments for more information.
+ */
+int
+crypto_mac_verify(crypto_mechanism_t *mech, crypto_data_t *data,
+ crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *mac,
+ crypto_call_req_t *crq)
+{
+ int error;
+ kcf_mech_entry_t *me;
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd;
+ kcf_ctx_template_t *ctx_tmpl;
+ crypto_spi_ctx_template_t spi_ctx_tmpl = NULL;
+ kcf_prov_tried_t *list = NULL;
+
+retry:
+ /* The pd is returned held */
+ if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error,
+ list, CRYPTO_FG_MAC_ATOMIC, CHECK_RESTRICT(crq),
+ data->cd_length)) == NULL) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ return (error);
+ }
+
+ /*
+ * For SW providers, check the validity of the context template
+ * It is very rare that the generation number mis-matches, so
+ * is acceptable to fail here, and let the consumer recover by
+ * freeing this tmpl and create a new one for the key and new SW
+ * provider
+ */
+ if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) &&
+ ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) {
+ if (ctx_tmpl->ct_generation != me->me_gen_swprov) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ KCF_PROV_REFRELE(pd);
+ return (CRYPTO_OLD_CTX_TEMPLATE);
+ } else {
+ spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl;
+ }
+ }
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(crq, pd)) {
+ crypto_mechanism_t lmech;
+
+ lmech = *mech;
+ KCF_SET_PROVIDER_MECHNUM(mech->cm_type, pd, &lmech);
+
+ error = KCF_PROV_MAC_VERIFY_ATOMIC(pd, pd->pd_sid, &lmech, key,
+ data, mac, spi_ctx_tmpl, KCF_SWFP_RHNDL(crq));
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ if (pd->pd_prov_type == CRYPTO_HW_PROVIDER &&
+ (pd->pd_flags & CRYPTO_HASH_NO_UPDATE) &&
+ (data->cd_length > pd->pd_hash_limit)) {
+ /* see comments in crypto_mac() */
+ error = CRYPTO_BUFFER_TOO_BIG;
+ } else {
+ KCF_WRAP_MAC_OPS_PARAMS(¶ms,
+ KCF_OP_MAC_VERIFY_ATOMIC, pd->pd_sid, mech,
+ key, data, mac, spi_ctx_tmpl);
+
+ error = kcf_submit_request(pd, NULL, crq, ¶ms,
+ KCF_ISDUALREQ(crq));
+ }
+ }
+
+ if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED &&
+ IS_RECOVERABLE(error)) {
+ /* Add pd to the linked list of providers tried. */
+ if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL)
+ goto retry;
+ }
+
+ if (list != NULL)
+ kcf_free_triedlist(list);
+
+ KCF_PROV_REFRELE(pd);
+ return (error);
+}
+
+/*
+ * crypto_mac_init_prov()
+ *
+ * Arguments:
+ * pd: pointer to the descriptor of the provider to use for this
+ * operation.
+ * sid: provider session id.
+ * mech: crypto_mechanism_t pointer.
+ * mech_type is a valid value previously returned by
+ * crypto_mech2id();
+ * When the mech's parameter is not NULL, its definition depends
+ * on the standard definition of the mechanism.
+ * key: pointer to a crypto_key_t structure.
+ * tmpl: a crypto_ctx_template_t, opaque template of a context of a
+ * MAC with the 'mech' using 'key'. 'tmpl' is created by
+ * a previous call to crypto_create_ctx_template().
+ * ctxp: Pointer to a crypto_context_t.
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs the
+ * initialization of a MAC operation on the specified provider with
+ * the specified session.
+ * When possible and applicable, will internally use the pre-computed MAC
+ * context from the context template, tmpl.
+ * When complete and successful, 'ctxp' will contain a crypto_context_t
+ * valid for later calls to mac_update() and mac_final().
+ * The caller should hold a reference on the specified provider
+ * descriptor before calling this function.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_mac_init_prov(crypto_provider_t provider, crypto_session_id_t sid,
+ crypto_mechanism_t *mech, crypto_key_t *key, crypto_spi_ctx_template_t tmpl,
+ crypto_context_t *ctxp, crypto_call_req_t *crq)
+{
+ int rv;
+ crypto_ctx_t *ctx;
+ kcf_req_params_t params;
+ kcf_provider_desc_t *pd = provider;
+ kcf_provider_desc_t *real_provider = pd;
+
+ ASSERT(KCF_PROV_REFHELD(pd));
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ rv = kcf_get_hardware_provider(mech->cm_type,
+ CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd,
+ &real_provider, CRYPTO_FG_MAC);
+
+ if (rv != CRYPTO_SUCCESS)
+ return (rv);
+ }
+
+ /* Allocate and initialize the canonical context */
+ if ((ctx = kcf_new_ctx(crq, real_provider, sid)) == NULL) {
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+ return (CRYPTO_HOST_MEMORY);
+ }
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(crq, pd)) {
+ crypto_mechanism_t lmech;
+
+ lmech = *mech;
+ KCF_SET_PROVIDER_MECHNUM(mech->cm_type, real_provider, &lmech);
+ rv = KCF_PROV_MAC_INIT(real_provider, ctx, &lmech, key, tmpl,
+ KCF_SWFP_RHNDL(crq));
+ KCF_PROV_INCRSTATS(pd, rv);
+ } else {
+ KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_INIT, sid, mech, key,
+ NULL, NULL, tmpl);
+ rv = kcf_submit_request(real_provider, ctx, crq, ¶ms,
+ B_FALSE);
+ }
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)
+ KCF_PROV_REFRELE(real_provider);
+
+ if ((rv == CRYPTO_SUCCESS) || (rv == CRYPTO_QUEUED))
+ *ctxp = (crypto_context_t)ctx;
+ else {
+ /* Release the hold done in kcf_new_ctx(). */
+ KCF_CONTEXT_REFRELE((kcf_context_t *)ctx->cc_framework_private);
+ }
+
+ return (rv);
+}
+
+/*
+ * Same as crypto_mac_init_prov(), but relies on the KCF scheduler to
+ * choose a provider. See crypto_mac_init_prov() comments for more
+ * information.
+ */
+int
+crypto_mac_init(crypto_mechanism_t *mech, crypto_key_t *key,
+ crypto_ctx_template_t tmpl, crypto_context_t *ctxp,
+ crypto_call_req_t *crq)
+{
+ int error;
+ kcf_mech_entry_t *me;
+ kcf_provider_desc_t *pd;
+ kcf_ctx_template_t *ctx_tmpl;
+ crypto_spi_ctx_template_t spi_ctx_tmpl = NULL;
+ kcf_prov_tried_t *list = NULL;
+
+retry:
+ /* The pd is returned held */
+ if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error,
+ list, CRYPTO_FG_MAC, CHECK_RESTRICT(crq), 0)) == NULL) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ return (error);
+ }
+
+ /*
+ * For SW providers, check the validity of the context template
+ * It is very rare that the generation number mis-matches, so
+ * is acceptable to fail here, and let the consumer recover by
+ * freeing this tmpl and create a new one for the key and new SW
+ * provider
+ */
+
+ if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) &&
+ ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) {
+ if (ctx_tmpl->ct_generation != me->me_gen_swprov) {
+ if (list != NULL)
+ kcf_free_triedlist(list);
+ KCF_PROV_REFRELE(pd);
+ return (CRYPTO_OLD_CTX_TEMPLATE);
+ } else {
+ spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl;
+ }
+ }
+
+ if (pd->pd_prov_type == CRYPTO_HW_PROVIDER &&
+ (pd->pd_flags & CRYPTO_HASH_NO_UPDATE)) {
+ /*
+ * The hardware provider has limited HMAC support.
+ * So, we fallback early here to using a software provider.
+ *
+ * XXX - need to enhance to do the fallback later in
+ * crypto_mac_update() if the size of accumulated input data
+ * exceeds the maximum size digestable by hardware provider.
+ */
+ error = CRYPTO_BUFFER_TOO_BIG;
+ } else {
+ error = crypto_mac_init_prov(pd, pd->pd_sid, mech, key,
+ spi_ctx_tmpl, ctxp, crq);
+ }
+ if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED &&
+ IS_RECOVERABLE(error)) {
+ /* Add pd to the linked list of providers tried. */
+ if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL)
+ goto retry;
+ }
+
+ if (list != NULL)
+ kcf_free_triedlist(list);
+
+ KCF_PROV_REFRELE(pd);
+ return (error);
+}
+
+/*
+ * crypto_mac_update()
+ *
+ * Arguments:
+ * context: A crypto_context_t initialized by mac_init().
+ * data: The message part to be MAC'ed
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs a
+ * part of a MAC operation.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_mac_update(crypto_context_t context, crypto_data_t *data,
+ crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ kcf_req_params_t params;
+ int rv;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ rv = KCF_PROV_MAC_UPDATE(pd, ctx, data, NULL);
+ KCF_PROV_INCRSTATS(pd, rv);
+ } else {
+ KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_UPDATE,
+ ctx->cc_session, NULL, NULL, data, NULL, NULL);
+ rv = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ return (rv);
+}
+
+/*
+ * crypto_mac_final()
+ *
+ * Arguments:
+ * context: A crypto_context_t initialized by mac_init().
+ * mac: Storage for the message authentication code.
+ * cr: crypto_call_req_t calling conditions and call back info.
+ *
+ * Description:
+ * Asynchronously submits a request for, or synchronously performs a
+ * part of a message authentication operation.
+ *
+ * Context:
+ * Process or interrupt, according to the semantics dictated by the 'cr'.
+ *
+ * Returns:
+ * See comment in the beginning of the file.
+ */
+int
+crypto_mac_final(crypto_context_t context, crypto_data_t *mac,
+ crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ kcf_req_params_t params;
+ int rv;
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+ ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ rv = KCF_PROV_MAC_FINAL(pd, ctx, mac, NULL);
+ KCF_PROV_INCRSTATS(pd, rv);
+ } else {
+ KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_FINAL,
+ ctx->cc_session, NULL, NULL, NULL, mac, NULL);
+ rv = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ /* Release the hold done in kcf_new_ctx() during init step. */
+ KCF_CONTEXT_COND_RELEASE(rv, kcf_ctx);
+ return (rv);
+}
+
+/*
+ * See comments for crypto_mac_update() and crypto_mac_final().
+ */
+int
+crypto_mac_single(crypto_context_t context, crypto_data_t *data,
+ crypto_data_t *mac, crypto_call_req_t *cr)
+{
+ crypto_ctx_t *ctx = (crypto_ctx_t *)context;
+ kcf_context_t *kcf_ctx;
+ kcf_provider_desc_t *pd;
+ int error;
+ kcf_req_params_t params;
+
+
+ if ((ctx == NULL) ||
+ ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) ||
+ ((pd = kcf_ctx->kc_prov_desc) == NULL)) {
+ return (CRYPTO_INVALID_CONTEXT);
+ }
+
+
+ /* The fast path for SW providers. */
+ if (CHECK_FASTPATH(cr, pd)) {
+ error = KCF_PROV_MAC(pd, ctx, data, mac, NULL);
+ KCF_PROV_INCRSTATS(pd, error);
+ } else {
+ KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_SINGLE, pd->pd_sid,
+ NULL, NULL, data, mac, NULL);
+ error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE);
+ }
+
+ /* Release the hold done in kcf_new_ctx() during init step. */
+ KCF_CONTEXT_COND_RELEASE(error, kcf_ctx);
+ return (error);
+}
+
+#if defined(_KERNEL) && defined(HAVE_SPL)
+EXPORT_SYMBOL(crypto_mac_prov);
+EXPORT_SYMBOL(crypto_mac);
+EXPORT_SYMBOL(crypto_mac_verify_prov);
+EXPORT_SYMBOL(crypto_mac_verify);
+EXPORT_SYMBOL(crypto_mac_init_prov);
+EXPORT_SYMBOL(crypto_mac_init);
+EXPORT_SYMBOL(crypto_mac_update);
+EXPORT_SYMBOL(crypto_mac_final);
+EXPORT_SYMBOL(crypto_mac_single);
+#endif
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/sched_impl.h>
+
+/*
+ * All event subscribers are put on a list. kcf_notify_list_lock
+ * protects changes to this list.
+ *
+ * The following locking order is maintained in the code - The
+ * global kcf_notify_list_lock followed by the individual lock
+ * in a kcf_ntfy_elem structure (kn_lock).
+ */
+kmutex_t ntfy_list_lock;
+kcondvar_t ntfy_list_cv; /* cv the service thread waits on */
+static kcf_ntfy_elem_t *ntfy_list_head;
+
+/*
+ * crypto_mech2id()
+ *
+ * Arguments:
+ * . mechname: A null-terminated string identifying the mechanism name.
+ *
+ * Description:
+ * Walks the mechanisms tables, looking for an entry that matches the
+ * mechname. Once it find it, it builds the 64-bit mech_type and returns
+ * it. If there are no hardware or software providers for the mechanism,
+ * but there is an unloaded software provider, this routine will attempt
+ * to load it.
+ *
+ * Context:
+ * Process and interruption.
+ *
+ * Returns:
+ * The unique mechanism identified by 'mechname', if found.
+ * CRYPTO_MECH_INVALID otherwise.
+ */
+crypto_mech_type_t
+crypto_mech2id(char *mechname)
+{
+ return (crypto_mech2id_common(mechname, B_TRUE));
+}
+
+/*
+ * We walk the notification list and do the callbacks.
+ */
+void
+kcf_walk_ntfylist(uint32_t event, void *event_arg)
+{
+ kcf_ntfy_elem_t *nep;
+ int nelem = 0;
+
+ mutex_enter(&ntfy_list_lock);
+
+ /*
+ * Count how many clients are on the notification list. We need
+ * this count to ensure that clients which joined the list after we
+ * have started this walk, are not wrongly notified.
+ */
+ for (nep = ntfy_list_head; nep != NULL; nep = nep->kn_next)
+ nelem++;
+
+ for (nep = ntfy_list_head; (nep != NULL && nelem); nep = nep->kn_next) {
+ nelem--;
+
+ /*
+ * Check if this client is interested in the
+ * event.
+ */
+ if (!(nep->kn_event_mask & event))
+ continue;
+
+ mutex_enter(&nep->kn_lock);
+ nep->kn_state = NTFY_RUNNING;
+ mutex_exit(&nep->kn_lock);
+ mutex_exit(&ntfy_list_lock);
+
+ /*
+ * We invoke the callback routine with no locks held. Another
+ * client could have joined the list meanwhile. This is fine
+ * as we maintain nelem as stated above. The NULL check in the
+ * for loop guards against shrinkage. Also, any callers of
+ * crypto_unnotify_events() at this point cv_wait till kn_state
+ * changes to NTFY_WAITING. Hence, nep is assured to be valid.
+ */
+ (*nep->kn_func)(event, event_arg);
+
+ mutex_enter(&nep->kn_lock);
+ nep->kn_state = NTFY_WAITING;
+ cv_broadcast(&nep->kn_cv);
+ mutex_exit(&nep->kn_lock);
+
+ mutex_enter(&ntfy_list_lock);
+ }
+
+ mutex_exit(&ntfy_list_lock);
+}
+
+#if defined(_KERNEL) && defined(HAVE_SPL)
+EXPORT_SYMBOL(crypto_mech2id);
+#endif
--- /dev/null
+ ---------------------------------------------------------------------------
+ Copyright (c) 1998-2007, Brian Gladman, Worcester, UK. All rights reserved.
+
+ LICENSE TERMS
+
+ The free distribution and use of this software is allowed (with or without
+ changes) provided that:
+
+ 1. source code distributions include the above copyright notice, this
+ list of conditions and the following disclaimer;
+
+ 2. binary distributions include the above copyright notice, this list
+ of conditions and the following disclaimer in their documentation;
+
+ 3. the name of the copyright holder is not used to endorse products
+ built using this software without specific written permission.
+
+ DISCLAIMER
+
+ This software is provided 'as is' with no explicit or implied warranties
+ in respect of its properties, including, but not limited to, correctness
+ and/or fitness for purpose.
+ ---------------------------------------------------------------------------
--- /dev/null
+PORTIONS OF AES FUNCTIONALITY
--- /dev/null
+
+ LICENSE ISSUES
+ ==============
+
+ The OpenSSL toolkit stays under a dual license, i.e. both the conditions of
+ the OpenSSL License and the original SSLeay license apply to the toolkit.
+ See below for the actual license texts. Actually both licenses are BSD-style
+ Open Source licenses. In case of any license issues related to OpenSSL
+ please contact openssl-core@openssl.org.
+
+ OpenSSL License
+ ---------------
+
+/* ====================================================================
+ * Copyright (c) 1998-2008 The OpenSSL Project. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ * software must display the following acknowledgment:
+ * "This product includes software developed by the OpenSSL Project
+ * for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ * endorse or promote products derived from this software without
+ * prior written permission. For written permission, please contact
+ * openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ * nor may "OpenSSL" appear in their names without prior written
+ * permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ * acknowledgment:
+ * "This product includes software developed by the OpenSSL Project
+ * for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * (eay@cryptsoft.com). This product includes software written by Tim
+ * Hudson (tjh@cryptsoft.com).
+ *
+ */
+
+ Original SSLeay License
+ -----------------------
+
+/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
+ * All rights reserved.
+ *
+ * This package is an SSL implementation written
+ * by Eric Young (eay@cryptsoft.com).
+ * The implementation was written so as to conform with Netscapes SSL.
+ *
+ * This library is free for commercial and non-commercial use as long as
+ * the following conditions are aheared to. The following conditions
+ * apply to all code found in this distribution, be it the RC4, RSA,
+ * lhash, DES, etc., code; not just the SSL code. The SSL documentation
+ * included with this distribution is covered by the same copyright terms
+ * except that the holder is Tim Hudson (tjh@cryptsoft.com).
+ *
+ * Copyright remains Eric Young's, and as such any Copyright notices in
+ * the code are not to be removed.
+ * If this package is used in a product, Eric Young should be given attribution
+ * as the author of the parts of the library used.
+ * This can be in the form of a textual message at program startup or
+ * in documentation (online or textual) provided with the package.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * "This product includes cryptographic software written by
+ * Eric Young (eay@cryptsoft.com)"
+ * The word 'cryptographic' can be left out if the rouines from the library
+ * being used are not cryptographic related :-).
+ * 4. If you include any Windows specific code (or a derivative thereof) from
+ * the apps directory (application code) you must include an acknowledgement:
+ * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The licence and distribution terms for any publically available version or
+ * derivative of this code cannot be changed. i.e. this code cannot simply be
+ * copied and put under another distribution licence
+ * [including the GNU Public Licence.]
+ */
+
--- /dev/null
+PORTIONS OF AES FUNCTIONALITY
--- /dev/null
+/*
+ * ---------------------------------------------------------------------------
+ * Copyright (c) 1998-2007, Brian Gladman, Worcester, UK. All rights reserved.
+ *
+ * LICENSE TERMS
+ *
+ * The free distribution and use of this software is allowed (with or without
+ * changes) provided that:
+ *
+ * 1. source code distributions include the above copyright notice, this
+ * list of conditions and the following disclaimer;
+ *
+ * 2. binary distributions include the above copyright notice, this list
+ * of conditions and the following disclaimer in their documentation;
+ *
+ * 3. the name of the copyright holder is not used to endorse products
+ * built using this software without specific written permission.
+ *
+ * DISCLAIMER
+ *
+ * This software is provided 'as is' with no explicit or implied warranties
+ * in respect of its properties, including, but not limited to, correctness
+ * and/or fitness for purpose.
+ * ---------------------------------------------------------------------------
+ * Issue 20/12/2007
+ *
+ * I am grateful to Dag Arne Osvik for many discussions of the techniques that
+ * can be used to optimise AES assembler code on AMD64/EM64T architectures.
+ * Some of the techniques used in this implementation are the result of
+ * suggestions made by him for which I am most grateful.
+ *
+ * An AES implementation for AMD64 processors using the YASM assembler. This
+ * implementation provides only encryption, decryption and hence requires key
+ * scheduling support in C. It uses 8k bytes of tables but its encryption and
+ * decryption performance is very close to that obtained using large tables.
+ * It can use either MS Windows or Gnu/Linux/OpenSolaris OS calling conventions,
+ * which are as follows:
+ * ms windows gnu/linux/opensolaris os
+ *
+ * in_blk rcx rdi
+ * out_blk rdx rsi
+ * context (cx) r8 rdx
+ *
+ * preserved rsi - + rbx, rbp, rsp, r12, r13, r14 & r15
+ * registers rdi - on both
+ *
+ * destroyed - rsi + rax, rcx, rdx, r8, r9, r10 & r11
+ * registers - rdi on both
+ *
+ * The convention used here is that for gnu/linux/opensolaris os.
+ *
+ * This code provides the standard AES block size (128 bits, 16 bytes) and the
+ * three standard AES key sizes (128, 192 and 256 bits). It has the same call
+ * interface as my C implementation. It uses the Microsoft C AMD64 calling
+ * conventions in which the three parameters are placed in rcx, rdx and r8
+ * respectively. The rbx, rsi, rdi, rbp and r12..r15 registers are preserved.
+ *
+ * OpenSolaris Note:
+ * Modified to use GNU/Linux/Solaris calling conventions.
+ * That is parameters are placed in rdi, rsi, rdx, and rcx, respectively.
+ *
+ * AES_RETURN aes_encrypt(const unsigned char in_blk[],
+ * unsigned char out_blk[], const aes_encrypt_ctx cx[1])/
+ *
+ * AES_RETURN aes_decrypt(const unsigned char in_blk[],
+ * unsigned char out_blk[], const aes_decrypt_ctx cx[1])/
+ *
+ * AES_RETURN aes_encrypt_key<NNN>(const unsigned char key[],
+ * const aes_encrypt_ctx cx[1])/
+ *
+ * AES_RETURN aes_decrypt_key<NNN>(const unsigned char key[],
+ * const aes_decrypt_ctx cx[1])/
+ *
+ * AES_RETURN aes_encrypt_key(const unsigned char key[],
+ * unsigned int len, const aes_decrypt_ctx cx[1])/
+ *
+ * AES_RETURN aes_decrypt_key(const unsigned char key[],
+ * unsigned int len, const aes_decrypt_ctx cx[1])/
+ *
+ * where <NNN> is 128, 102 or 256. In the last two calls the length can be in
+ * either bits or bytes.
+ *
+ * Comment in/out the following lines to obtain the desired subroutines. These
+ * selections MUST match those in the C header file aesopt.h
+ */
+#define AES_REV_DKS /* define if key decryption schedule is reversed */
+
+#define LAST_ROUND_TABLES /* define for the faster version using extra tables */
+
+/*
+ * The encryption key schedule has the following in memory layout where N is the
+ * number of rounds (10, 12 or 14):
+ *
+ * lo: | input key (round 0) | / each round is four 32-bit words
+ * | encryption round 1 |
+ * | encryption round 2 |
+ * ....
+ * | encryption round N-1 |
+ * hi: | encryption round N |
+ *
+ * The decryption key schedule is normally set up so that it has the same
+ * layout as above by actually reversing the order of the encryption key
+ * schedule in memory (this happens when AES_REV_DKS is set):
+ *
+ * lo: | decryption round 0 | = | encryption round N |
+ * | decryption round 1 | = INV_MIX_COL[ | encryption round N-1 | ]
+ * | decryption round 2 | = INV_MIX_COL[ | encryption round N-2 | ]
+ * .... ....
+ * | decryption round N-1 | = INV_MIX_COL[ | encryption round 1 | ]
+ * hi: | decryption round N | = | input key (round 0) |
+ *
+ * with rounds except the first and last modified using inv_mix_column()
+ * But if AES_REV_DKS is NOT set the order of keys is left as it is for
+ * encryption so that it has to be accessed in reverse when used for
+ * decryption (although the inverse mix column modifications are done)
+ *
+ * lo: | decryption round 0 | = | input key (round 0) |
+ * | decryption round 1 | = INV_MIX_COL[ | encryption round 1 | ]
+ * | decryption round 2 | = INV_MIX_COL[ | encryption round 2 | ]
+ * .... ....
+ * | decryption round N-1 | = INV_MIX_COL[ | encryption round N-1 | ]
+ * hi: | decryption round N | = | encryption round N |
+ *
+ * This layout is faster when the assembler key scheduling provided here
+ * is used.
+ *
+ * End of user defines
+ */
+
+/*
+ * ---------------------------------------------------------------------------
+ * OpenSolaris OS modifications
+ *
+ * This source originates from Brian Gladman file aes_amd64.asm
+ * in http://fp.gladman.plus.com/AES/aes-src-04-03-08.zip
+ * with these changes:
+ *
+ * 1. Removed MS Windows-specific code within DLL_EXPORT, _SEH_, and
+ * !__GNUC__ ifdefs. Also removed ENCRYPTION, DECRYPTION,
+ * AES_128, AES_192, AES_256, AES_VAR ifdefs.
+ *
+ * 2. Translate yasm/nasm %define and .macro definitions to cpp(1) #define
+ *
+ * 3. Translate yasm/nasm %ifdef/%ifndef to cpp(1) #ifdef
+ *
+ * 4. Translate Intel/yasm/nasm syntax to ATT/OpenSolaris as(1) syntax
+ * (operands reversed, literals prefixed with "$", registers prefixed with "%",
+ * and "[register+offset]", addressing changed to "offset(register)",
+ * parenthesis in constant expressions "()" changed to square brackets "[]",
+ * "." removed from local (numeric) labels, and other changes.
+ * Examples:
+ * Intel/yasm/nasm Syntax ATT/OpenSolaris Syntax
+ * mov rax,(4*20h) mov $[4*0x20],%rax
+ * mov rax,[ebx+20h] mov 0x20(%ebx),%rax
+ * lea rax,[ebx+ecx] lea (%ebx,%ecx),%rax
+ * sub rax,[ebx+ecx*4-20h] sub -0x20(%ebx,%ecx,4),%rax
+ *
+ * 5. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
+ * /usr/include/sys/asm_linkage.h, lint(1B) guards, and dummy C function
+ * definitions for lint.
+ *
+ * 6. Renamed functions and reordered parameters to match OpenSolaris:
+ * Original Gladman interface:
+ * int aes_encrypt(const unsigned char *in,
+ * unsigned char *out, const aes_encrypt_ctx cx[1])/
+ * int aes_decrypt(const unsigned char *in,
+ * unsigned char *out, const aes_encrypt_ctx cx[1])/
+ * Note: aes_encrypt_ctx contains ks, a 60 element array of uint32_t,
+ * and a union type, inf., containing inf.l, a uint32_t and
+ * inf.b, a 4-element array of uint32_t. Only b[0] in the array (aka "l") is
+ * used and contains the key schedule length * 16 where key schedule length is
+ * 10, 12, or 14 bytes.
+ *
+ * OpenSolaris OS interface:
+ * void aes_encrypt_amd64(const aes_ks_t *ks, int Nr,
+ * const uint32_t pt[4], uint32_t ct[4])/
+ * void aes_decrypt_amd64(const aes_ks_t *ks, int Nr,
+ * const uint32_t pt[4], uint32_t ct[4])/
+ * typedef union {uint64_t ks64[(MAX_AES_NR + 1) * 4]/
+ * uint32_t ks32[(MAX_AES_NR + 1) * 4]/ } aes_ks_t/
+ * Note: ks is the AES key schedule, Nr is number of rounds, pt is plain text,
+ * ct is crypto text, and MAX_AES_NR is 14.
+ * For the x86 64-bit architecture, OpenSolaris OS uses ks32 instead of ks64.
+ */
+
+#if defined(lint) || defined(__lint)
+
+#include <sys/types.h>
+/* ARGSUSED */
+void
+aes_encrypt_amd64(const uint32_t rk[], int Nr, const uint32_t pt[4],
+ uint32_t ct[4]) {
+}
+/* ARGSUSED */
+void
+aes_decrypt_amd64(const uint32_t rk[], int Nr, const uint32_t ct[4],
+ uint32_t pt[4]) {
+}
+
+
+#else
+
+#define _ASM
+#include <sys/asm_linkage.h>
+
+#define KS_LENGTH 60
+
+#define raxd eax
+#define rdxd edx
+#define rcxd ecx
+#define rbxd ebx
+#define rsid esi
+#define rdid edi
+
+#define raxb al
+#define rdxb dl
+#define rcxb cl
+#define rbxb bl
+#define rsib sil
+#define rdib dil
+
+// finite field multiplies by {02}, {04} and {08}
+
+#define f2(x) [[x<<1]^[[[x>>7]&1]*0x11b]]
+#define f4(x) [[x<<2]^[[[x>>6]&1]*0x11b]^[[[x>>6]&2]*0x11b]]
+#define f8(x) [[x<<3]^[[[x>>5]&1]*0x11b]^[[[x>>5]&2]*0x11b]^[[[x>>5]&4]*0x11b]]
+
+// finite field multiplies required in table generation
+
+#define f3(x) [[f2(x)] ^ [x]]
+#define f9(x) [[f8(x)] ^ [x]]
+#define fb(x) [[f8(x)] ^ [f2(x)] ^ [x]]
+#define fd(x) [[f8(x)] ^ [f4(x)] ^ [x]]
+#define fe(x) [[f8(x)] ^ [f4(x)] ^ [f2(x)]]
+
+// macros for expanding S-box data
+
+#define u8(x) [f2(x)], [x], [x], [f3(x)], [f2(x)], [x], [x], [f3(x)]
+#define v8(x) [fe(x)], [f9(x)], [fd(x)], [fb(x)], [fe(x)], [f9(x)], [fd(x)], [x]
+#define w8(x) [x], 0, 0, 0, [x], 0, 0, 0
+
+#define enc_vals(x) \
+ .byte x(0x63),x(0x7c),x(0x77),x(0x7b),x(0xf2),x(0x6b),x(0x6f),x(0xc5); \
+ .byte x(0x30),x(0x01),x(0x67),x(0x2b),x(0xfe),x(0xd7),x(0xab),x(0x76); \
+ .byte x(0xca),x(0x82),x(0xc9),x(0x7d),x(0xfa),x(0x59),x(0x47),x(0xf0); \
+ .byte x(0xad),x(0xd4),x(0xa2),x(0xaf),x(0x9c),x(0xa4),x(0x72),x(0xc0); \
+ .byte x(0xb7),x(0xfd),x(0x93),x(0x26),x(0x36),x(0x3f),x(0xf7),x(0xcc); \
+ .byte x(0x34),x(0xa5),x(0xe5),x(0xf1),x(0x71),x(0xd8),x(0x31),x(0x15); \
+ .byte x(0x04),x(0xc7),x(0x23),x(0xc3),x(0x18),x(0x96),x(0x05),x(0x9a); \
+ .byte x(0x07),x(0x12),x(0x80),x(0xe2),x(0xeb),x(0x27),x(0xb2),x(0x75); \
+ .byte x(0x09),x(0x83),x(0x2c),x(0x1a),x(0x1b),x(0x6e),x(0x5a),x(0xa0); \
+ .byte x(0x52),x(0x3b),x(0xd6),x(0xb3),x(0x29),x(0xe3),x(0x2f),x(0x84); \
+ .byte x(0x53),x(0xd1),x(0x00),x(0xed),x(0x20),x(0xfc),x(0xb1),x(0x5b); \
+ .byte x(0x6a),x(0xcb),x(0xbe),x(0x39),x(0x4a),x(0x4c),x(0x58),x(0xcf); \
+ .byte x(0xd0),x(0xef),x(0xaa),x(0xfb),x(0x43),x(0x4d),x(0x33),x(0x85); \
+ .byte x(0x45),x(0xf9),x(0x02),x(0x7f),x(0x50),x(0x3c),x(0x9f),x(0xa8); \
+ .byte x(0x51),x(0xa3),x(0x40),x(0x8f),x(0x92),x(0x9d),x(0x38),x(0xf5); \
+ .byte x(0xbc),x(0xb6),x(0xda),x(0x21),x(0x10),x(0xff),x(0xf3),x(0xd2); \
+ .byte x(0xcd),x(0x0c),x(0x13),x(0xec),x(0x5f),x(0x97),x(0x44),x(0x17); \
+ .byte x(0xc4),x(0xa7),x(0x7e),x(0x3d),x(0x64),x(0x5d),x(0x19),x(0x73); \
+ .byte x(0x60),x(0x81),x(0x4f),x(0xdc),x(0x22),x(0x2a),x(0x90),x(0x88); \
+ .byte x(0x46),x(0xee),x(0xb8),x(0x14),x(0xde),x(0x5e),x(0x0b),x(0xdb); \
+ .byte x(0xe0),x(0x32),x(0x3a),x(0x0a),x(0x49),x(0x06),x(0x24),x(0x5c); \
+ .byte x(0xc2),x(0xd3),x(0xac),x(0x62),x(0x91),x(0x95),x(0xe4),x(0x79); \
+ .byte x(0xe7),x(0xc8),x(0x37),x(0x6d),x(0x8d),x(0xd5),x(0x4e),x(0xa9); \
+ .byte x(0x6c),x(0x56),x(0xf4),x(0xea),x(0x65),x(0x7a),x(0xae),x(0x08); \
+ .byte x(0xba),x(0x78),x(0x25),x(0x2e),x(0x1c),x(0xa6),x(0xb4),x(0xc6); \
+ .byte x(0xe8),x(0xdd),x(0x74),x(0x1f),x(0x4b),x(0xbd),x(0x8b),x(0x8a); \
+ .byte x(0x70),x(0x3e),x(0xb5),x(0x66),x(0x48),x(0x03),x(0xf6),x(0x0e); \
+ .byte x(0x61),x(0x35),x(0x57),x(0xb9),x(0x86),x(0xc1),x(0x1d),x(0x9e); \
+ .byte x(0xe1),x(0xf8),x(0x98),x(0x11),x(0x69),x(0xd9),x(0x8e),x(0x94); \
+ .byte x(0x9b),x(0x1e),x(0x87),x(0xe9),x(0xce),x(0x55),x(0x28),x(0xdf); \
+ .byte x(0x8c),x(0xa1),x(0x89),x(0x0d),x(0xbf),x(0xe6),x(0x42),x(0x68); \
+ .byte x(0x41),x(0x99),x(0x2d),x(0x0f),x(0xb0),x(0x54),x(0xbb),x(0x16)
+
+#define dec_vals(x) \
+ .byte x(0x52),x(0x09),x(0x6a),x(0xd5),x(0x30),x(0x36),x(0xa5),x(0x38); \
+ .byte x(0xbf),x(0x40),x(0xa3),x(0x9e),x(0x81),x(0xf3),x(0xd7),x(0xfb); \
+ .byte x(0x7c),x(0xe3),x(0x39),x(0x82),x(0x9b),x(0x2f),x(0xff),x(0x87); \
+ .byte x(0x34),x(0x8e),x(0x43),x(0x44),x(0xc4),x(0xde),x(0xe9),x(0xcb); \
+ .byte x(0x54),x(0x7b),x(0x94),x(0x32),x(0xa6),x(0xc2),x(0x23),x(0x3d); \
+ .byte x(0xee),x(0x4c),x(0x95),x(0x0b),x(0x42),x(0xfa),x(0xc3),x(0x4e); \
+ .byte x(0x08),x(0x2e),x(0xa1),x(0x66),x(0x28),x(0xd9),x(0x24),x(0xb2); \
+ .byte x(0x76),x(0x5b),x(0xa2),x(0x49),x(0x6d),x(0x8b),x(0xd1),x(0x25); \
+ .byte x(0x72),x(0xf8),x(0xf6),x(0x64),x(0x86),x(0x68),x(0x98),x(0x16); \
+ .byte x(0xd4),x(0xa4),x(0x5c),x(0xcc),x(0x5d),x(0x65),x(0xb6),x(0x92); \
+ .byte x(0x6c),x(0x70),x(0x48),x(0x50),x(0xfd),x(0xed),x(0xb9),x(0xda); \
+ .byte x(0x5e),x(0x15),x(0x46),x(0x57),x(0xa7),x(0x8d),x(0x9d),x(0x84); \
+ .byte x(0x90),x(0xd8),x(0xab),x(0x00),x(0x8c),x(0xbc),x(0xd3),x(0x0a); \
+ .byte x(0xf7),x(0xe4),x(0x58),x(0x05),x(0xb8),x(0xb3),x(0x45),x(0x06); \
+ .byte x(0xd0),x(0x2c),x(0x1e),x(0x8f),x(0xca),x(0x3f),x(0x0f),x(0x02); \
+ .byte x(0xc1),x(0xaf),x(0xbd),x(0x03),x(0x01),x(0x13),x(0x8a),x(0x6b); \
+ .byte x(0x3a),x(0x91),x(0x11),x(0x41),x(0x4f),x(0x67),x(0xdc),x(0xea); \
+ .byte x(0x97),x(0xf2),x(0xcf),x(0xce),x(0xf0),x(0xb4),x(0xe6),x(0x73); \
+ .byte x(0x96),x(0xac),x(0x74),x(0x22),x(0xe7),x(0xad),x(0x35),x(0x85); \
+ .byte x(0xe2),x(0xf9),x(0x37),x(0xe8),x(0x1c),x(0x75),x(0xdf),x(0x6e); \
+ .byte x(0x47),x(0xf1),x(0x1a),x(0x71),x(0x1d),x(0x29),x(0xc5),x(0x89); \
+ .byte x(0x6f),x(0xb7),x(0x62),x(0x0e),x(0xaa),x(0x18),x(0xbe),x(0x1b); \
+ .byte x(0xfc),x(0x56),x(0x3e),x(0x4b),x(0xc6),x(0xd2),x(0x79),x(0x20); \
+ .byte x(0x9a),x(0xdb),x(0xc0),x(0xfe),x(0x78),x(0xcd),x(0x5a),x(0xf4); \
+ .byte x(0x1f),x(0xdd),x(0xa8),x(0x33),x(0x88),x(0x07),x(0xc7),x(0x31); \
+ .byte x(0xb1),x(0x12),x(0x10),x(0x59),x(0x27),x(0x80),x(0xec),x(0x5f); \
+ .byte x(0x60),x(0x51),x(0x7f),x(0xa9),x(0x19),x(0xb5),x(0x4a),x(0x0d); \
+ .byte x(0x2d),x(0xe5),x(0x7a),x(0x9f),x(0x93),x(0xc9),x(0x9c),x(0xef); \
+ .byte x(0xa0),x(0xe0),x(0x3b),x(0x4d),x(0xae),x(0x2a),x(0xf5),x(0xb0); \
+ .byte x(0xc8),x(0xeb),x(0xbb),x(0x3c),x(0x83),x(0x53),x(0x99),x(0x61); \
+ .byte x(0x17),x(0x2b),x(0x04),x(0x7e),x(0xba),x(0x77),x(0xd6),x(0x26); \
+ .byte x(0xe1),x(0x69),x(0x14),x(0x63),x(0x55),x(0x21),x(0x0c),x(0x7d)
+
+#define tptr %rbp /* table pointer */
+#define kptr %r8 /* key schedule pointer */
+#define fofs 128 /* adjust offset in key schedule to keep |disp| < 128 */
+#define fk_ref(x, y) -16*x+fofs+4*y(kptr)
+
+#ifdef AES_REV_DKS
+#define rofs 128
+#define ik_ref(x, y) -16*x+rofs+4*y(kptr)
+
+#else
+#define rofs -128
+#define ik_ref(x, y) 16*x+rofs+4*y(kptr)
+#endif /* AES_REV_DKS */
+
+#define tab_0(x) (tptr,x,8)
+#define tab_1(x) 3(tptr,x,8)
+#define tab_2(x) 2(tptr,x,8)
+#define tab_3(x) 1(tptr,x,8)
+#define tab_f(x) 1(tptr,x,8)
+#define tab_i(x) 7(tptr,x,8)
+
+#define ff_rnd(p1, p2, p3, p4, round) /* normal forward round */ \
+ mov fk_ref(round,0), p1; \
+ mov fk_ref(round,1), p2; \
+ mov fk_ref(round,2), p3; \
+ mov fk_ref(round,3), p4; \
+ \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ shr $16, %eax; \
+ xor tab_0(%rsi), p1; \
+ xor tab_1(%rdi), p4; \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ xor tab_2(%rsi), p3; \
+ xor tab_3(%rdi), p2; \
+ \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ shr $16, %ebx; \
+ xor tab_0(%rsi), p2; \
+ xor tab_1(%rdi), p1; \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ xor tab_2(%rsi), p4; \
+ xor tab_3(%rdi), p3; \
+ \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ shr $16, %ecx; \
+ xor tab_0(%rsi), p3; \
+ xor tab_1(%rdi), p2; \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ xor tab_2(%rsi), p1; \
+ xor tab_3(%rdi), p4; \
+ \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ shr $16, %edx; \
+ xor tab_0(%rsi), p4; \
+ xor tab_1(%rdi), p3; \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ xor tab_2(%rsi), p2; \
+ xor tab_3(%rdi), p1; \
+ \
+ mov p1, %eax; \
+ mov p2, %ebx; \
+ mov p3, %ecx; \
+ mov p4, %edx
+
+#ifdef LAST_ROUND_TABLES
+
+#define fl_rnd(p1, p2, p3, p4, round) /* last forward round */ \
+ add $2048, tptr; \
+ mov fk_ref(round,0), p1; \
+ mov fk_ref(round,1), p2; \
+ mov fk_ref(round,2), p3; \
+ mov fk_ref(round,3), p4; \
+ \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ shr $16, %eax; \
+ xor tab_0(%rsi), p1; \
+ xor tab_1(%rdi), p4; \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ xor tab_2(%rsi), p3; \
+ xor tab_3(%rdi), p2; \
+ \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ shr $16, %ebx; \
+ xor tab_0(%rsi), p2; \
+ xor tab_1(%rdi), p1; \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ xor tab_2(%rsi), p4; \
+ xor tab_3(%rdi), p3; \
+ \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ shr $16, %ecx; \
+ xor tab_0(%rsi), p3; \
+ xor tab_1(%rdi), p2; \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ xor tab_2(%rsi), p1; \
+ xor tab_3(%rdi), p4; \
+ \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ shr $16, %edx; \
+ xor tab_0(%rsi), p4; \
+ xor tab_1(%rdi), p3; \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ xor tab_2(%rsi), p2; \
+ xor tab_3(%rdi), p1
+
+#else
+
+#define fl_rnd(p1, p2, p3, p4, round) /* last forward round */ \
+ mov fk_ref(round,0), p1; \
+ mov fk_ref(round,1), p2; \
+ mov fk_ref(round,2), p3; \
+ mov fk_ref(round,3), p4; \
+ \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ shr $16, %eax; \
+ movzx tab_f(%rsi), %esi; \
+ movzx tab_f(%rdi), %edi; \
+ xor %esi, p1; \
+ rol $8, %edi; \
+ xor %edi, p4; \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ movzx tab_f(%rsi), %esi; \
+ movzx tab_f(%rdi), %edi; \
+ rol $16, %esi; \
+ rol $24, %edi; \
+ xor %esi, p3; \
+ xor %edi, p2; \
+ \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ shr $16, %ebx; \
+ movzx tab_f(%rsi), %esi; \
+ movzx tab_f(%rdi), %edi; \
+ xor %esi, p2; \
+ rol $8, %edi; \
+ xor %edi, p1; \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ movzx tab_f(%rsi), %esi; \
+ movzx tab_f(%rdi), %edi; \
+ rol $16, %esi; \
+ rol $24, %edi; \
+ xor %esi, p4; \
+ xor %edi, p3; \
+ \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ movzx tab_f(%rsi), %esi; \
+ movzx tab_f(%rdi), %edi; \
+ shr $16, %ecx; \
+ xor %esi, p3; \
+ rol $8, %edi; \
+ xor %edi, p2; \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ movzx tab_f(%rsi), %esi; \
+ movzx tab_f(%rdi), %edi; \
+ rol $16, %esi; \
+ rol $24, %edi; \
+ xor %esi, p1; \
+ xor %edi, p4; \
+ \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ movzx tab_f(%rsi), %esi; \
+ movzx tab_f(%rdi), %edi; \
+ shr $16, %edx; \
+ xor %esi, p4; \
+ rol $8, %edi; \
+ xor %edi, p3; \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ movzx tab_f(%rsi), %esi; \
+ movzx tab_f(%rdi), %edi; \
+ rol $16, %esi; \
+ rol $24, %edi; \
+ xor %esi, p2; \
+ xor %edi, p1
+
+#endif /* LAST_ROUND_TABLES */
+
+#define ii_rnd(p1, p2, p3, p4, round) /* normal inverse round */ \
+ mov ik_ref(round,0), p1; \
+ mov ik_ref(round,1), p2; \
+ mov ik_ref(round,2), p3; \
+ mov ik_ref(round,3), p4; \
+ \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ shr $16, %eax; \
+ xor tab_0(%rsi), p1; \
+ xor tab_1(%rdi), p2; \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ xor tab_2(%rsi), p3; \
+ xor tab_3(%rdi), p4; \
+ \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ shr $16, %ebx; \
+ xor tab_0(%rsi), p2; \
+ xor tab_1(%rdi), p3; \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ xor tab_2(%rsi), p4; \
+ xor tab_3(%rdi), p1; \
+ \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ shr $16, %ecx; \
+ xor tab_0(%rsi), p3; \
+ xor tab_1(%rdi), p4; \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ xor tab_2(%rsi), p1; \
+ xor tab_3(%rdi), p2; \
+ \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ shr $16, %edx; \
+ xor tab_0(%rsi), p4; \
+ xor tab_1(%rdi), p1; \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ xor tab_2(%rsi), p2; \
+ xor tab_3(%rdi), p3; \
+ \
+ mov p1, %eax; \
+ mov p2, %ebx; \
+ mov p3, %ecx; \
+ mov p4, %edx
+
+#ifdef LAST_ROUND_TABLES
+
+#define il_rnd(p1, p2, p3, p4, round) /* last inverse round */ \
+ add $2048, tptr; \
+ mov ik_ref(round,0), p1; \
+ mov ik_ref(round,1), p2; \
+ mov ik_ref(round,2), p3; \
+ mov ik_ref(round,3), p4; \
+ \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ shr $16, %eax; \
+ xor tab_0(%rsi), p1; \
+ xor tab_1(%rdi), p2; \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ xor tab_2(%rsi), p3; \
+ xor tab_3(%rdi), p4; \
+ \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ shr $16, %ebx; \
+ xor tab_0(%rsi), p2; \
+ xor tab_1(%rdi), p3; \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ xor tab_2(%rsi), p4; \
+ xor tab_3(%rdi), p1; \
+ \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ shr $16, %ecx; \
+ xor tab_0(%rsi), p3; \
+ xor tab_1(%rdi), p4; \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ xor tab_2(%rsi), p1; \
+ xor tab_3(%rdi), p2; \
+ \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ shr $16, %edx; \
+ xor tab_0(%rsi), p4; \
+ xor tab_1(%rdi), p1; \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ xor tab_2(%rsi), p2; \
+ xor tab_3(%rdi), p3
+
+#else
+
+#define il_rnd(p1, p2, p3, p4, round) /* last inverse round */ \
+ mov ik_ref(round,0), p1; \
+ mov ik_ref(round,1), p2; \
+ mov ik_ref(round,2), p3; \
+ mov ik_ref(round,3), p4; \
+ \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ movzx tab_i(%rsi), %esi; \
+ movzx tab_i(%rdi), %edi; \
+ shr $16, %eax; \
+ xor %esi, p1; \
+ rol $8, %edi; \
+ xor %edi, p2; \
+ movzx %al, %esi; \
+ movzx %ah, %edi; \
+ movzx tab_i(%rsi), %esi; \
+ movzx tab_i(%rdi), %edi; \
+ rol $16, %esi; \
+ rol $24, %edi; \
+ xor %esi, p3; \
+ xor %edi, p4; \
+ \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ movzx tab_i(%rsi), %esi; \
+ movzx tab_i(%rdi), %edi; \
+ shr $16, %ebx; \
+ xor %esi, p2; \
+ rol $8, %edi; \
+ xor %edi, p3; \
+ movzx %bl, %esi; \
+ movzx %bh, %edi; \
+ movzx tab_i(%rsi), %esi; \
+ movzx tab_i(%rdi), %edi; \
+ rol $16, %esi; \
+ rol $24, %edi; \
+ xor %esi, p4; \
+ xor %edi, p1; \
+ \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ movzx tab_i(%rsi), %esi; \
+ movzx tab_i(%rdi), %edi; \
+ shr $16, %ecx; \
+ xor %esi, p3; \
+ rol $8, %edi; \
+ xor %edi, p4; \
+ movzx %cl, %esi; \
+ movzx %ch, %edi; \
+ movzx tab_i(%rsi), %esi; \
+ movzx tab_i(%rdi), %edi; \
+ rol $16, %esi; \
+ rol $24, %edi; \
+ xor %esi, p1; \
+ xor %edi, p2; \
+ \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ movzx tab_i(%rsi), %esi; \
+ movzx tab_i(%rdi), %edi; \
+ shr $16, %edx; \
+ xor %esi, p4; \
+ rol $8, %edi; \
+ xor %edi, p1; \
+ movzx %dl, %esi; \
+ movzx %dh, %edi; \
+ movzx tab_i(%rsi), %esi; \
+ movzx tab_i(%rdi), %edi; \
+ rol $16, %esi; \
+ rol $24, %edi; \
+ xor %esi, p2; \
+ xor %edi, p3
+
+#endif /* LAST_ROUND_TABLES */
+
+/*
+ * OpenSolaris OS:
+ * void aes_encrypt_amd64(const aes_ks_t *ks, int Nr,
+ * const uint32_t pt[4], uint32_t ct[4])/
+ *
+ * Original interface:
+ * int aes_encrypt(const unsigned char *in,
+ * unsigned char *out, const aes_encrypt_ctx cx[1])/
+ */
+ .align 64
+enc_tab:
+ enc_vals(u8)
+#ifdef LAST_ROUND_TABLES
+ // Last Round Tables:
+ enc_vals(w8)
+#endif
+
+
+ ENTRY_NP(aes_encrypt_amd64)
+#ifdef GLADMAN_INTERFACE
+ // Original interface
+ sub $[4*8], %rsp // gnu/linux/opensolaris binary interface
+ mov %rsi, (%rsp) // output pointer (P2)
+ mov %rdx, %r8 // context (P3)
+
+ mov %rbx, 1*8(%rsp) // P1: input pointer in rdi
+ mov %rbp, 2*8(%rsp) // P2: output pointer in (rsp)
+ mov %r12, 3*8(%rsp) // P3: context in r8
+ movzx 4*KS_LENGTH(kptr), %esi // Get byte key length * 16
+
+#else
+ // OpenSolaris OS interface
+ sub $[4*8], %rsp // Make room on stack to save registers
+ mov %rcx, (%rsp) // Save output pointer (P4) on stack
+ mov %rdi, %r8 // context (P1)
+ mov %rdx, %rdi // P3: save input pointer
+ shl $4, %esi // P2: esi byte key length * 16
+
+ mov %rbx, 1*8(%rsp) // Save registers
+ mov %rbp, 2*8(%rsp)
+ mov %r12, 3*8(%rsp)
+ // P1: context in r8
+ // P2: byte key length * 16 in esi
+ // P3: input pointer in rdi
+ // P4: output pointer in (rsp)
+#endif /* GLADMAN_INTERFACE */
+
+ lea enc_tab(%rip), tptr
+ sub $fofs, kptr
+
+ // Load input block into registers
+ mov (%rdi), %eax
+ mov 1*4(%rdi), %ebx
+ mov 2*4(%rdi), %ecx
+ mov 3*4(%rdi), %edx
+
+ xor fofs(kptr), %eax
+ xor fofs+4(kptr), %ebx
+ xor fofs+8(kptr), %ecx
+ xor fofs+12(kptr), %edx
+
+ lea (kptr,%rsi), kptr
+ // Jump based on byte key length * 16:
+ cmp $[10*16], %esi
+ je 3f
+ cmp $[12*16], %esi
+ je 2f
+ cmp $[14*16], %esi
+ je 1f
+ mov $-1, %rax // error
+ jmp 4f
+
+ // Perform normal forward rounds
+1: ff_rnd(%r9d, %r10d, %r11d, %r12d, 13)
+ ff_rnd(%r9d, %r10d, %r11d, %r12d, 12)
+2: ff_rnd(%r9d, %r10d, %r11d, %r12d, 11)
+ ff_rnd(%r9d, %r10d, %r11d, %r12d, 10)
+3: ff_rnd(%r9d, %r10d, %r11d, %r12d, 9)
+ ff_rnd(%r9d, %r10d, %r11d, %r12d, 8)
+ ff_rnd(%r9d, %r10d, %r11d, %r12d, 7)
+ ff_rnd(%r9d, %r10d, %r11d, %r12d, 6)
+ ff_rnd(%r9d, %r10d, %r11d, %r12d, 5)
+ ff_rnd(%r9d, %r10d, %r11d, %r12d, 4)
+ ff_rnd(%r9d, %r10d, %r11d, %r12d, 3)
+ ff_rnd(%r9d, %r10d, %r11d, %r12d, 2)
+ ff_rnd(%r9d, %r10d, %r11d, %r12d, 1)
+ fl_rnd(%r9d, %r10d, %r11d, %r12d, 0)
+
+ // Copy results
+ mov (%rsp), %rbx
+ mov %r9d, (%rbx)
+ mov %r10d, 4(%rbx)
+ mov %r11d, 8(%rbx)
+ mov %r12d, 12(%rbx)
+ xor %rax, %rax
+4: // Restore registers
+ mov 1*8(%rsp), %rbx
+ mov 2*8(%rsp), %rbp
+ mov 3*8(%rsp), %r12
+ add $[4*8], %rsp
+ ret
+
+ SET_SIZE(aes_encrypt_amd64)
+
+/*
+ * OpenSolaris OS:
+ * void aes_decrypt_amd64(const aes_ks_t *ks, int Nr,
+ * const uint32_t pt[4], uint32_t ct[4])/
+ *
+ * Original interface:
+ * int aes_decrypt(const unsigned char *in,
+ * unsigned char *out, const aes_encrypt_ctx cx[1])/
+ */
+ .align 64
+dec_tab:
+ dec_vals(v8)
+#ifdef LAST_ROUND_TABLES
+ // Last Round Tables:
+ dec_vals(w8)
+#endif
+
+
+ ENTRY_NP(aes_decrypt_amd64)
+#ifdef GLADMAN_INTERFACE
+ // Original interface
+ sub $[4*8], %rsp // gnu/linux/opensolaris binary interface
+ mov %rsi, (%rsp) // output pointer (P2)
+ mov %rdx, %r8 // context (P3)
+
+ mov %rbx, 1*8(%rsp) // P1: input pointer in rdi
+ mov %rbp, 2*8(%rsp) // P2: output pointer in (rsp)
+ mov %r12, 3*8(%rsp) // P3: context in r8
+ movzx 4*KS_LENGTH(kptr), %esi // Get byte key length * 16
+
+#else
+ // OpenSolaris OS interface
+ sub $[4*8], %rsp // Make room on stack to save registers
+ mov %rcx, (%rsp) // Save output pointer (P4) on stack
+ mov %rdi, %r8 // context (P1)
+ mov %rdx, %rdi // P3: save input pointer
+ shl $4, %esi // P2: esi byte key length * 16
+
+ mov %rbx, 1*8(%rsp) // Save registers
+ mov %rbp, 2*8(%rsp)
+ mov %r12, 3*8(%rsp)
+ // P1: context in r8
+ // P2: byte key length * 16 in esi
+ // P3: input pointer in rdi
+ // P4: output pointer in (rsp)
+#endif /* GLADMAN_INTERFACE */
+
+ lea dec_tab(%rip), tptr
+ sub $rofs, kptr
+
+ // Load input block into registers
+ mov (%rdi), %eax
+ mov 1*4(%rdi), %ebx
+ mov 2*4(%rdi), %ecx
+ mov 3*4(%rdi), %edx
+
+#ifdef AES_REV_DKS
+ mov kptr, %rdi
+ lea (kptr,%rsi), kptr
+#else
+ lea (kptr,%rsi), %rdi
+#endif
+
+ xor rofs(%rdi), %eax
+ xor rofs+4(%rdi), %ebx
+ xor rofs+8(%rdi), %ecx
+ xor rofs+12(%rdi), %edx
+
+ // Jump based on byte key length * 16:
+ cmp $[10*16], %esi
+ je 3f
+ cmp $[12*16], %esi
+ je 2f
+ cmp $[14*16], %esi
+ je 1f
+ mov $-1, %rax // error
+ jmp 4f
+
+ // Perform normal inverse rounds
+1: ii_rnd(%r9d, %r10d, %r11d, %r12d, 13)
+ ii_rnd(%r9d, %r10d, %r11d, %r12d, 12)
+2: ii_rnd(%r9d, %r10d, %r11d, %r12d, 11)
+ ii_rnd(%r9d, %r10d, %r11d, %r12d, 10)
+3: ii_rnd(%r9d, %r10d, %r11d, %r12d, 9)
+ ii_rnd(%r9d, %r10d, %r11d, %r12d, 8)
+ ii_rnd(%r9d, %r10d, %r11d, %r12d, 7)
+ ii_rnd(%r9d, %r10d, %r11d, %r12d, 6)
+ ii_rnd(%r9d, %r10d, %r11d, %r12d, 5)
+ ii_rnd(%r9d, %r10d, %r11d, %r12d, 4)
+ ii_rnd(%r9d, %r10d, %r11d, %r12d, 3)
+ ii_rnd(%r9d, %r10d, %r11d, %r12d, 2)
+ ii_rnd(%r9d, %r10d, %r11d, %r12d, 1)
+ il_rnd(%r9d, %r10d, %r11d, %r12d, 0)
+
+ // Copy results
+ mov (%rsp), %rbx
+ mov %r9d, (%rbx)
+ mov %r10d, 4(%rbx)
+ mov %r11d, 8(%rbx)
+ mov %r12d, 12(%rbx)
+ xor %rax, %rax
+4: // Restore registers
+ mov 1*8(%rsp), %rbx
+ mov 2*8(%rsp), %rbp
+ mov 3*8(%rsp), %r12
+ add $[4*8], %rsp
+ ret
+
+ SET_SIZE(aes_decrypt_amd64)
+#endif /* lint || __lint */
--- /dev/null
+/*
+ * ====================================================================
+ * Written by Intel Corporation for the OpenSSL project to add support
+ * for Intel AES-NI instructions. Rights for redistribution and usage
+ * in source and binary forms are granted according to the OpenSSL
+ * license.
+ *
+ * Author: Huang Ying <ying.huang at intel dot com>
+ * Vinodh Gopal <vinodh.gopal at intel dot com>
+ * Kahraman Akdemir
+ *
+ * Intel AES-NI is a new set of Single Instruction Multiple Data (SIMD)
+ * instructions that are going to be introduced in the next generation
+ * of Intel processor, as of 2009. These instructions enable fast and
+ * secure data encryption and decryption, using the Advanced Encryption
+ * Standard (AES), defined by FIPS Publication number 197. The
+ * architecture introduces six instructions that offer full hardware
+ * support for AES. Four of them support high performance data
+ * encryption and decryption, and the other two instructions support
+ * the AES key expansion procedure.
+ * ====================================================================
+ */
+
+/*
+ * ====================================================================
+ * Copyright (c) 1998-2008 The OpenSSL Project. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ * software must display the following acknowledgment:
+ * "This product includes software developed by the OpenSSL Project
+ * for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ * endorse or promote products derived from this software without
+ * prior written permission. For written permission, please contact
+ * openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ * nor may "OpenSSL" appear in their names without prior written
+ * permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ * acknowledgment:
+ * "This product includes software developed by the OpenSSL Project
+ * for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ */
+
+/*
+ * ====================================================================
+ * OpenSolaris OS modifications
+ *
+ * This source originates as files aes-intel.S and eng_aesni_asm.pl, in
+ * patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by
+ * Huang Ying of Intel to the openssl-dev mailing list under the subject
+ * of "Add support to Intel AES-NI instruction set for x86_64 platform".
+ *
+ * This OpenSolaris version has these major changes from the original source:
+ *
+ * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
+ * /usr/include/sys/asm_linkage.h, lint(1B) guards, and dummy C function
+ * definitions for lint.
+ *
+ * 2. Formatted code, added comments, and added #includes and #defines.
+ *
+ * 3. If bit CR0.TS is set, clear and set the TS bit, after and before
+ * calling kpreempt_disable() and kpreempt_enable().
+ * If the TS bit is not set, Save and restore %xmm registers at the beginning
+ * and end of function calls (%xmm* registers are not saved and restored by
+ * during kernel thread preemption).
+ *
+ * 4. Renamed functions, reordered parameters, and changed return value
+ * to match OpenSolaris:
+ *
+ * OpenSSL interface:
+ * int intel_AES_set_encrypt_key(const unsigned char *userKey,
+ * const int bits, AES_KEY *key);
+ * int intel_AES_set_decrypt_key(const unsigned char *userKey,
+ * const int bits, AES_KEY *key);
+ * Return values for above are non-zero on error, 0 on success.
+ *
+ * void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
+ * const AES_KEY *key);
+ * void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
+ * const AES_KEY *key);
+ * typedef struct aes_key_st {
+ * unsigned int rd_key[4 *(AES_MAXNR + 1)];
+ * int rounds;
+ * unsigned int pad[3];
+ * } AES_KEY;
+ * Note: AES_LONG is undefined (that is, Intel uses 32-bit key schedules
+ * (ks32) instead of 64-bit (ks64).
+ * Number of rounds (aka round count) is at offset 240 of AES_KEY.
+ *
+ * OpenSolaris OS interface (#ifdefs removed for readability):
+ * int rijndael_key_setup_dec_intel(uint32_t rk[],
+ * const uint32_t cipherKey[], uint64_t keyBits);
+ * int rijndael_key_setup_enc_intel(uint32_t rk[],
+ * const uint32_t cipherKey[], uint64_t keyBits);
+ * Return values for above are 0 on error, number of rounds on success.
+ *
+ * void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
+ * const uint32_t pt[4], uint32_t ct[4]);
+ * void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
+ * const uint32_t pt[4], uint32_t ct[4]);
+ * typedef union {uint64_t ks64[(MAX_AES_NR + 1) * 4];
+ * uint32_t ks32[(MAX_AES_NR + 1) * 4]; } aes_ks_t;
+ *
+ * typedef union {
+ * uint32_t ks32[((MAX_AES_NR) + 1) * (MAX_AES_NB)];
+ * } aes_ks_t;
+ * typedef struct aes_key {
+ * aes_ks_t encr_ks, decr_ks;
+ * long double align128;
+ * int flags, nr, type;
+ * } aes_key_t;
+ *
+ * Note: ks is the AES key schedule, Nr is number of rounds, pt is plain text,
+ * ct is crypto text, and MAX_AES_NR is 14.
+ * For the x86 64-bit architecture, OpenSolaris OS uses ks32 instead of ks64.
+ *
+ * Note2: aes_ks_t must be aligned on a 0 mod 128 byte boundary.
+ *
+ * ====================================================================
+ */
+
+#if defined(lint) || defined(__lint)
+
+#include <sys/types.h>
+
+/* ARGSUSED */
+void
+aes_encrypt_intel(const uint32_t rk[], int Nr, const uint32_t pt[4],
+ uint32_t ct[4]) {
+}
+/* ARGSUSED */
+void
+aes_decrypt_intel(const uint32_t rk[], int Nr, const uint32_t ct[4],
+ uint32_t pt[4]) {
+}
+/* ARGSUSED */
+int
+rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
+ uint64_t keyBits) {
+ return (0);
+}
+/* ARGSUSED */
+int
+rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
+ uint64_t keyBits) {
+ return (0);
+}
+
+
+#else /* lint */
+
+#define _ASM
+#include <sys/asm_linkage.h>
+
+#ifdef _KERNEL
+ /*
+ * Note: the CLTS macro clobbers P2 (%rsi) under i86xpv. That is,
+ * it calls HYPERVISOR_fpu_taskswitch() which modifies %rsi when it
+ * uses it to pass P2 to syscall.
+ * This also occurs with the STTS macro, but we dont care if
+ * P2 (%rsi) is modified just before function exit.
+ * The CLTS and STTS macros push and pop P1 (%rdi) already.
+ */
+#ifdef __xpv
+#define PROTECTED_CLTS \
+ push %rsi; \
+ CLTS; \
+ pop %rsi
+#else
+#define PROTECTED_CLTS \
+ CLTS
+#endif /* __xpv */
+
+#define CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg) \
+ push %rbp; \
+ mov %rsp, %rbp; \
+ movq %cr0, tmpreg; \
+ testq $CR0_TS, tmpreg; \
+ jnz 1f; \
+ and $-XMM_ALIGN, %rsp; \
+ sub $[XMM_SIZE * 2], %rsp; \
+ movaps %xmm0, 16(%rsp); \
+ movaps %xmm1, (%rsp); \
+ jmp 2f; \
+1: \
+ PROTECTED_CLTS; \
+2:
+
+ /*
+ * If CR0_TS was not set above, pop %xmm0 and %xmm1 off stack,
+ * otherwise set CR0_TS.
+ */
+#define SET_TS_OR_POP_XMM0_XMM1(tmpreg) \
+ testq $CR0_TS, tmpreg; \
+ jnz 1f; \
+ movaps (%rsp), %xmm1; \
+ movaps 16(%rsp), %xmm0; \
+ jmp 2f; \
+1: \
+ STTS(tmpreg); \
+2: \
+ mov %rbp, %rsp; \
+ pop %rbp
+
+ /*
+ * If CR0_TS is not set, align stack (with push %rbp) and push
+ * %xmm0 - %xmm6 on stack, otherwise clear CR0_TS
+ */
+#define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg) \
+ push %rbp; \
+ mov %rsp, %rbp; \
+ movq %cr0, tmpreg; \
+ testq $CR0_TS, tmpreg; \
+ jnz 1f; \
+ and $-XMM_ALIGN, %rsp; \
+ sub $[XMM_SIZE * 7], %rsp; \
+ movaps %xmm0, 96(%rsp); \
+ movaps %xmm1, 80(%rsp); \
+ movaps %xmm2, 64(%rsp); \
+ movaps %xmm3, 48(%rsp); \
+ movaps %xmm4, 32(%rsp); \
+ movaps %xmm5, 16(%rsp); \
+ movaps %xmm6, (%rsp); \
+ jmp 2f; \
+1: \
+ PROTECTED_CLTS; \
+2:
+
+
+ /*
+ * If CR0_TS was not set above, pop %xmm0 - %xmm6 off stack,
+ * otherwise set CR0_TS.
+ */
+#define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg) \
+ testq $CR0_TS, tmpreg; \
+ jnz 1f; \
+ movaps (%rsp), %xmm6; \
+ movaps 16(%rsp), %xmm5; \
+ movaps 32(%rsp), %xmm4; \
+ movaps 48(%rsp), %xmm3; \
+ movaps 64(%rsp), %xmm2; \
+ movaps 80(%rsp), %xmm1; \
+ movaps 96(%rsp), %xmm0; \
+ jmp 2f; \
+1: \
+ STTS(tmpreg); \
+2: \
+ mov %rbp, %rsp; \
+ pop %rbp
+
+
+#else
+#define PROTECTED_CLTS
+#define CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg)
+#define SET_TS_OR_POP_XMM0_XMM1(tmpreg)
+#define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg)
+#define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg)
+#endif /* _KERNEL */
+
+
+/*
+ * _key_expansion_128(), * _key_expansion_192a(), _key_expansion_192b(),
+ * _key_expansion_256a(), _key_expansion_256b()
+ *
+ * Helper functions called by rijndael_key_setup_inc_intel().
+ * Also used indirectly by rijndael_key_setup_dec_intel().
+ *
+ * Input:
+ * %xmm0 User-provided cipher key
+ * %xmm1 Round constant
+ * Output:
+ * (%rcx) AES key
+ */
+
+.align 16
+_key_expansion_128:
+_key_expansion_256a:
+ pshufd $0b11111111, %xmm1, %xmm1
+ shufps $0b00010000, %xmm0, %xmm4
+ pxor %xmm4, %xmm0
+ shufps $0b10001100, %xmm0, %xmm4
+ pxor %xmm4, %xmm0
+ pxor %xmm1, %xmm0
+ movaps %xmm0, (%rcx)
+ add $0x10, %rcx
+ ret
+ SET_SIZE(_key_expansion_128)
+ SET_SIZE(_key_expansion_256a)
+
+.align 16
+_key_expansion_192a:
+ pshufd $0b01010101, %xmm1, %xmm1
+ shufps $0b00010000, %xmm0, %xmm4
+ pxor %xmm4, %xmm0
+ shufps $0b10001100, %xmm0, %xmm4
+ pxor %xmm4, %xmm0
+ pxor %xmm1, %xmm0
+
+ movaps %xmm2, %xmm5
+ movaps %xmm2, %xmm6
+ pslldq $4, %xmm5
+ pshufd $0b11111111, %xmm0, %xmm3
+ pxor %xmm3, %xmm2
+ pxor %xmm5, %xmm2
+
+ movaps %xmm0, %xmm1
+ shufps $0b01000100, %xmm0, %xmm6
+ movaps %xmm6, (%rcx)
+ shufps $0b01001110, %xmm2, %xmm1
+ movaps %xmm1, 0x10(%rcx)
+ add $0x20, %rcx
+ ret
+ SET_SIZE(_key_expansion_192a)
+
+.align 16
+_key_expansion_192b:
+ pshufd $0b01010101, %xmm1, %xmm1
+ shufps $0b00010000, %xmm0, %xmm4
+ pxor %xmm4, %xmm0
+ shufps $0b10001100, %xmm0, %xmm4
+ pxor %xmm4, %xmm0
+ pxor %xmm1, %xmm0
+
+ movaps %xmm2, %xmm5
+ pslldq $4, %xmm5
+ pshufd $0b11111111, %xmm0, %xmm3
+ pxor %xmm3, %xmm2
+ pxor %xmm5, %xmm2
+
+ movaps %xmm0, (%rcx)
+ add $0x10, %rcx
+ ret
+ SET_SIZE(_key_expansion_192b)
+
+.align 16
+_key_expansion_256b:
+ pshufd $0b10101010, %xmm1, %xmm1
+ shufps $0b00010000, %xmm2, %xmm4
+ pxor %xmm4, %xmm2
+ shufps $0b10001100, %xmm2, %xmm4
+ pxor %xmm4, %xmm2
+ pxor %xmm1, %xmm2
+ movaps %xmm2, (%rcx)
+ add $0x10, %rcx
+ ret
+ SET_SIZE(_key_expansion_256b)
+
+
+/*
+ * rijndael_key_setup_enc_intel()
+ * Expand the cipher key into the encryption key schedule.
+ *
+ * For kernel code, caller is responsible for ensuring kpreempt_disable()
+ * has been called. This is because %xmm registers are not saved/restored.
+ * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
+ * on entry. Otherwise, if TS is not set, save and restore %xmm registers
+ * on the stack.
+ *
+ * OpenSolaris interface:
+ * int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
+ * uint64_t keyBits);
+ * Return value is 0 on error, number of rounds on success.
+ *
+ * Original Intel OpenSSL interface:
+ * int intel_AES_set_encrypt_key(const unsigned char *userKey,
+ * const int bits, AES_KEY *key);
+ * Return value is non-zero on error, 0 on success.
+ */
+
+#ifdef OPENSSL_INTERFACE
+#define rijndael_key_setup_enc_intel intel_AES_set_encrypt_key
+#define rijndael_key_setup_dec_intel intel_AES_set_decrypt_key
+
+#define USERCIPHERKEY rdi /* P1, 64 bits */
+#define KEYSIZE32 esi /* P2, 32 bits */
+#define KEYSIZE64 rsi /* P2, 64 bits */
+#define AESKEY rdx /* P3, 64 bits */
+
+#else /* OpenSolaris Interface */
+#define AESKEY rdi /* P1, 64 bits */
+#define USERCIPHERKEY rsi /* P2, 64 bits */
+#define KEYSIZE32 edx /* P3, 32 bits */
+#define KEYSIZE64 rdx /* P3, 64 bits */
+#endif /* OPENSSL_INTERFACE */
+
+#define ROUNDS32 KEYSIZE32 /* temp */
+#define ROUNDS64 KEYSIZE64 /* temp */
+#define ENDAESKEY USERCIPHERKEY /* temp */
+
+ENTRY_NP(rijndael_key_setup_enc_intel)
+rijndael_key_setup_enc_intel_local:
+ CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(%r10)
+
+ // NULL pointer sanity check
+ test %USERCIPHERKEY, %USERCIPHERKEY
+ jz .Lenc_key_invalid_param
+ test %AESKEY, %AESKEY
+ jz .Lenc_key_invalid_param
+
+ movups (%USERCIPHERKEY), %xmm0 // user key (first 16 bytes)
+ movaps %xmm0, (%AESKEY)
+ lea 0x10(%AESKEY), %rcx // key addr
+ pxor %xmm4, %xmm4 // xmm4 is assumed 0 in _key_expansion_x
+
+ cmp $256, %KEYSIZE32
+ jnz .Lenc_key192
+
+ // AES 256: 14 rounds in encryption key schedule
+#ifdef OPENSSL_INTERFACE
+ mov $14, %ROUNDS32
+ movl %ROUNDS32, 240(%AESKEY) // key.rounds = 14
+#endif /* OPENSSL_INTERFACE */
+
+ movups 0x10(%USERCIPHERKEY), %xmm2 // other user key (2nd 16 bytes)
+ movaps %xmm2, (%rcx)
+ add $0x10, %rcx
+
+ aeskeygenassist $0x1, %xmm2, %xmm1 // expand the key
+ call _key_expansion_256a
+ aeskeygenassist $0x1, %xmm0, %xmm1
+ call _key_expansion_256b
+ aeskeygenassist $0x2, %xmm2, %xmm1 // expand the key
+ call _key_expansion_256a
+ aeskeygenassist $0x2, %xmm0, %xmm1
+ call _key_expansion_256b
+ aeskeygenassist $0x4, %xmm2, %xmm1 // expand the key
+ call _key_expansion_256a
+ aeskeygenassist $0x4, %xmm0, %xmm1
+ call _key_expansion_256b
+ aeskeygenassist $0x8, %xmm2, %xmm1 // expand the key
+ call _key_expansion_256a
+ aeskeygenassist $0x8, %xmm0, %xmm1
+ call _key_expansion_256b
+ aeskeygenassist $0x10, %xmm2, %xmm1 // expand the key
+ call _key_expansion_256a
+ aeskeygenassist $0x10, %xmm0, %xmm1
+ call _key_expansion_256b
+ aeskeygenassist $0x20, %xmm2, %xmm1 // expand the key
+ call _key_expansion_256a
+ aeskeygenassist $0x20, %xmm0, %xmm1
+ call _key_expansion_256b
+ aeskeygenassist $0x40, %xmm2, %xmm1 // expand the key
+ call _key_expansion_256a
+
+ SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
+#ifdef OPENSSL_INTERFACE
+ xor %rax, %rax // return 0 (OK)
+#else /* Open Solaris Interface */
+ mov $14, %rax // return # rounds = 14
+#endif
+ ret
+
+.align 4
+.Lenc_key192:
+ cmp $192, %KEYSIZE32
+ jnz .Lenc_key128
+
+ // AES 192: 12 rounds in encryption key schedule
+#ifdef OPENSSL_INTERFACE
+ mov $12, %ROUNDS32
+ movl %ROUNDS32, 240(%AESKEY) // key.rounds = 12
+#endif /* OPENSSL_INTERFACE */
+
+ movq 0x10(%USERCIPHERKEY), %xmm2 // other user key
+ aeskeygenassist $0x1, %xmm2, %xmm1 // expand the key
+ call _key_expansion_192a
+ aeskeygenassist $0x2, %xmm2, %xmm1 // expand the key
+ call _key_expansion_192b
+ aeskeygenassist $0x4, %xmm2, %xmm1 // expand the key
+ call _key_expansion_192a
+ aeskeygenassist $0x8, %xmm2, %xmm1 // expand the key
+ call _key_expansion_192b
+ aeskeygenassist $0x10, %xmm2, %xmm1 // expand the key
+ call _key_expansion_192a
+ aeskeygenassist $0x20, %xmm2, %xmm1 // expand the key
+ call _key_expansion_192b
+ aeskeygenassist $0x40, %xmm2, %xmm1 // expand the key
+ call _key_expansion_192a
+ aeskeygenassist $0x80, %xmm2, %xmm1 // expand the key
+ call _key_expansion_192b
+
+ SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
+#ifdef OPENSSL_INTERFACE
+ xor %rax, %rax // return 0 (OK)
+#else /* OpenSolaris Interface */
+ mov $12, %rax // return # rounds = 12
+#endif
+ ret
+
+.align 4
+.Lenc_key128:
+ cmp $128, %KEYSIZE32
+ jnz .Lenc_key_invalid_key_bits
+
+ // AES 128: 10 rounds in encryption key schedule
+#ifdef OPENSSL_INTERFACE
+ mov $10, %ROUNDS32
+ movl %ROUNDS32, 240(%AESKEY) // key.rounds = 10
+#endif /* OPENSSL_INTERFACE */
+
+ aeskeygenassist $0x1, %xmm0, %xmm1 // expand the key
+ call _key_expansion_128
+ aeskeygenassist $0x2, %xmm0, %xmm1 // expand the key
+ call _key_expansion_128
+ aeskeygenassist $0x4, %xmm0, %xmm1 // expand the key
+ call _key_expansion_128
+ aeskeygenassist $0x8, %xmm0, %xmm1 // expand the key
+ call _key_expansion_128
+ aeskeygenassist $0x10, %xmm0, %xmm1 // expand the key
+ call _key_expansion_128
+ aeskeygenassist $0x20, %xmm0, %xmm1 // expand the key
+ call _key_expansion_128
+ aeskeygenassist $0x40, %xmm0, %xmm1 // expand the key
+ call _key_expansion_128
+ aeskeygenassist $0x80, %xmm0, %xmm1 // expand the key
+ call _key_expansion_128
+ aeskeygenassist $0x1b, %xmm0, %xmm1 // expand the key
+ call _key_expansion_128
+ aeskeygenassist $0x36, %xmm0, %xmm1 // expand the key
+ call _key_expansion_128
+
+ SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
+#ifdef OPENSSL_INTERFACE
+ xor %rax, %rax // return 0 (OK)
+#else /* OpenSolaris Interface */
+ mov $10, %rax // return # rounds = 10
+#endif
+ ret
+
+.Lenc_key_invalid_param:
+#ifdef OPENSSL_INTERFACE
+ SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
+ mov $-1, %rax // user key or AES key pointer is NULL
+ ret
+#else
+ /* FALLTHROUGH */
+#endif /* OPENSSL_INTERFACE */
+
+.Lenc_key_invalid_key_bits:
+ SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
+#ifdef OPENSSL_INTERFACE
+ mov $-2, %rax // keysize is invalid
+#else /* Open Solaris Interface */
+ xor %rax, %rax // a key pointer is NULL or invalid keysize
+#endif /* OPENSSL_INTERFACE */
+
+ ret
+ SET_SIZE(rijndael_key_setup_enc_intel)
+
+
+/*
+ * rijndael_key_setup_dec_intel()
+ * Expand the cipher key into the decryption key schedule.
+ *
+ * For kernel code, caller is responsible for ensuring kpreempt_disable()
+ * has been called. This is because %xmm registers are not saved/restored.
+ * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
+ * on entry. Otherwise, if TS is not set, save and restore %xmm registers
+ * on the stack.
+ *
+ * OpenSolaris interface:
+ * int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
+ * uint64_t keyBits);
+ * Return value is 0 on error, number of rounds on success.
+ * P1->P2, P2->P3, P3->P1
+ *
+ * Original Intel OpenSSL interface:
+ * int intel_AES_set_decrypt_key(const unsigned char *userKey,
+ * const int bits, AES_KEY *key);
+ * Return value is non-zero on error, 0 on success.
+ */
+ENTRY_NP(rijndael_key_setup_dec_intel)
+ // Generate round keys used for encryption
+ call rijndael_key_setup_enc_intel_local
+ test %rax, %rax
+#ifdef OPENSSL_INTERFACE
+ jnz .Ldec_key_exit // Failed if returned non-0
+#else /* OpenSolaris Interface */
+ jz .Ldec_key_exit // Failed if returned 0
+#endif /* OPENSSL_INTERFACE */
+
+ CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
+
+ /*
+ * Convert round keys used for encryption
+ * to a form usable for decryption
+ */
+#ifndef OPENSSL_INTERFACE /* OpenSolaris Interface */
+ mov %rax, %ROUNDS64 // set # rounds (10, 12, or 14)
+ // (already set for OpenSSL)
+#endif
+
+ lea 0x10(%AESKEY), %rcx // key addr
+ shl $4, %ROUNDS32
+ add %AESKEY, %ROUNDS64
+ mov %ROUNDS64, %ENDAESKEY
+
+.align 4
+.Ldec_key_reorder_loop:
+ movaps (%AESKEY), %xmm0
+ movaps (%ROUNDS64), %xmm1
+ movaps %xmm0, (%ROUNDS64)
+ movaps %xmm1, (%AESKEY)
+ lea 0x10(%AESKEY), %AESKEY
+ lea -0x10(%ROUNDS64), %ROUNDS64
+ cmp %AESKEY, %ROUNDS64
+ ja .Ldec_key_reorder_loop
+
+.align 4
+.Ldec_key_inv_loop:
+ movaps (%rcx), %xmm0
+ // Convert an encryption round key to a form usable for decryption
+ // with the "AES Inverse Mix Columns" instruction
+ aesimc %xmm0, %xmm1
+ movaps %xmm1, (%rcx)
+ lea 0x10(%rcx), %rcx
+ cmp %ENDAESKEY, %rcx
+ jnz .Ldec_key_inv_loop
+
+ SET_TS_OR_POP_XMM0_XMM1(%r10)
+
+.Ldec_key_exit:
+ // OpenSolaris: rax = # rounds (10, 12, or 14) or 0 for error
+ // OpenSSL: rax = 0 for OK, or non-zero for error
+ ret
+ SET_SIZE(rijndael_key_setup_dec_intel)
+
+
+/*
+ * aes_encrypt_intel()
+ * Encrypt a single block (in and out can overlap).
+ *
+ * For kernel code, caller is responsible for ensuring kpreempt_disable()
+ * has been called. This is because %xmm registers are not saved/restored.
+ * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
+ * on entry. Otherwise, if TS is not set, save and restore %xmm registers
+ * on the stack.
+ *
+ * Temporary register usage:
+ * %xmm0 State
+ * %xmm1 Key
+ *
+ * Original OpenSolaris Interface:
+ * void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
+ * const uint32_t pt[4], uint32_t ct[4])
+ *
+ * Original Intel OpenSSL Interface:
+ * void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
+ * const AES_KEY *key)
+ */
+
+#ifdef OPENSSL_INTERFACE
+#define aes_encrypt_intel intel_AES_encrypt
+#define aes_decrypt_intel intel_AES_decrypt
+
+#define INP rdi /* P1, 64 bits */
+#define OUTP rsi /* P2, 64 bits */
+#define KEYP rdx /* P3, 64 bits */
+
+/* No NROUNDS parameter--offset 240 from KEYP saved in %ecx: */
+#define NROUNDS32 ecx /* temporary, 32 bits */
+#define NROUNDS cl /* temporary, 8 bits */
+
+#else /* OpenSolaris Interface */
+#define KEYP rdi /* P1, 64 bits */
+#define NROUNDS esi /* P2, 32 bits */
+#define INP rdx /* P3, 64 bits */
+#define OUTP rcx /* P4, 64 bits */
+#endif /* OPENSSL_INTERFACE */
+
+#define STATE xmm0 /* temporary, 128 bits */
+#define KEY xmm1 /* temporary, 128 bits */
+
+ENTRY_NP(aes_encrypt_intel)
+ CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
+
+ movups (%INP), %STATE // input
+ movaps (%KEYP), %KEY // key
+#ifdef OPENSSL_INTERFACE
+ mov 240(%KEYP), %NROUNDS32 // round count
+#else /* OpenSolaris Interface */
+ /* Round count is already present as P2 in %rsi/%esi */
+#endif /* OPENSSL_INTERFACE */
+
+ pxor %KEY, %STATE // round 0
+ lea 0x30(%KEYP), %KEYP
+ cmp $12, %NROUNDS
+ jb .Lenc128
+ lea 0x20(%KEYP), %KEYP
+ je .Lenc192
+
+ // AES 256
+ lea 0x20(%KEYP), %KEYP
+ movaps -0x60(%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps -0x50(%KEYP), %KEY
+ aesenc %KEY, %STATE
+
+.align 4
+.Lenc192:
+ // AES 192 and 256
+ movaps -0x40(%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps -0x30(%KEYP), %KEY
+ aesenc %KEY, %STATE
+
+.align 4
+.Lenc128:
+ // AES 128, 192, and 256
+ movaps -0x20(%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps -0x10(%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps (%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps 0x10(%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps 0x20(%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps 0x30(%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps 0x40(%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps 0x50(%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps 0x60(%KEYP), %KEY
+ aesenc %KEY, %STATE
+ movaps 0x70(%KEYP), %KEY
+ aesenclast %KEY, %STATE // last round
+ movups %STATE, (%OUTP) // output
+
+ SET_TS_OR_POP_XMM0_XMM1(%r10)
+ ret
+ SET_SIZE(aes_encrypt_intel)
+
+
+/*
+ * aes_decrypt_intel()
+ * Decrypt a single block (in and out can overlap).
+ *
+ * For kernel code, caller is responsible for ensuring kpreempt_disable()
+ * has been called. This is because %xmm registers are not saved/restored.
+ * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set
+ * on entry. Otherwise, if TS is not set, save and restore %xmm registers
+ * on the stack.
+ *
+ * Temporary register usage:
+ * %xmm0 State
+ * %xmm1 Key
+ *
+ * Original OpenSolaris Interface:
+ * void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
+ * const uint32_t pt[4], uint32_t ct[4])/
+ *
+ * Original Intel OpenSSL Interface:
+ * void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
+ * const AES_KEY *key);
+ */
+ENTRY_NP(aes_decrypt_intel)
+ CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
+
+ movups (%INP), %STATE // input
+ movaps (%KEYP), %KEY // key
+#ifdef OPENSSL_INTERFACE
+ mov 240(%KEYP), %NROUNDS32 // round count
+#else /* OpenSolaris Interface */
+ /* Round count is already present as P2 in %rsi/%esi */
+#endif /* OPENSSL_INTERFACE */
+
+ pxor %KEY, %STATE // round 0
+ lea 0x30(%KEYP), %KEYP
+ cmp $12, %NROUNDS
+ jb .Ldec128
+ lea 0x20(%KEYP), %KEYP
+ je .Ldec192
+
+ // AES 256
+ lea 0x20(%KEYP), %KEYP
+ movaps -0x60(%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps -0x50(%KEYP), %KEY
+ aesdec %KEY, %STATE
+
+.align 4
+.Ldec192:
+ // AES 192 and 256
+ movaps -0x40(%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps -0x30(%KEYP), %KEY
+ aesdec %KEY, %STATE
+
+.align 4
+.Ldec128:
+ // AES 128, 192, and 256
+ movaps -0x20(%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps -0x10(%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps (%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps 0x10(%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps 0x20(%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps 0x30(%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps 0x40(%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps 0x50(%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps 0x60(%KEYP), %KEY
+ aesdec %KEY, %STATE
+ movaps 0x70(%KEYP), %KEY
+ aesdeclast %KEY, %STATE // last round
+ movups %STATE, (%OUTP) // output
+
+ SET_TS_OR_POP_XMM0_XMM1(%r10)
+ ret
+ SET_SIZE(aes_decrypt_intel)
+
+#endif /* lint || __lint */
--- /dev/null
+/*
+ * ---------------------------------------------------------------------------
+ * Copyright (c) 1998-2007, Brian Gladman, Worcester, UK. All rights reserved.
+ *
+ * LICENSE TERMS
+ *
+ * The free distribution and use of this software is allowed (with or without
+ * changes) provided that:
+ *
+ * 1. source code distributions include the above copyright notice, this
+ * list of conditions and the following disclaimer;
+ *
+ * 2. binary distributions include the above copyright notice, this list
+ * of conditions and the following disclaimer in their documentation;
+ *
+ * 3. the name of the copyright holder is not used to endorse products
+ * built using this software without specific written permission.
+ *
+ * DISCLAIMER
+ *
+ * This software is provided 'as is' with no explicit or implied warranties
+ * in respect of its properties, including, but not limited to, correctness
+ * and/or fitness for purpose.
+ * ---------------------------------------------------------------------------
+ * Issue Date: 20/12/2007
+ */
+
+#include <aes/aes_impl.h>
+#include "aesopt.h"
+#include "aestab.h"
+#include "aestab2.h"
+
+/*
+ * Initialise the key schedule from the user supplied key. The key
+ * length can be specified in bytes, with legal values of 16, 24
+ * and 32, or in bits, with legal values of 128, 192 and 256. These
+ * values correspond with Nk values of 4, 6 and 8 respectively.
+ *
+ * The following macros implement a single cycle in the key
+ * schedule generation process. The number of cycles needed
+ * for each cx->n_col and nk value is:
+ *
+ * nk = 4 5 6 7 8
+ * ------------------------------
+ * cx->n_col = 4 10 9 8 7 7
+ * cx->n_col = 5 14 11 10 9 9
+ * cx->n_col = 6 19 15 12 11 11
+ * cx->n_col = 7 21 19 16 13 14
+ * cx->n_col = 8 29 23 19 17 14
+ */
+
+/*
+ * OpenSolaris changes
+ * 1. Added header files aes_impl.h and aestab2.h
+ * 2. Changed uint_8t and uint_32t to uint8_t and uint32_t
+ * 3. Remove code under ifdef USE_VIA_ACE_IF_PRESENT (always undefined)
+ * 4. Removed always-defined ifdefs FUNCS_IN_C, ENC_KEYING_IN_C,
+ * AES_128, AES_192, AES_256, AES_VAR defines
+ * 5. Changed aes_encrypt_key* aes_decrypt_key* functions to "static void"
+ * 6. Changed N_COLS to MAX_AES_NB
+ * 7. Replaced functions aes_encrypt_key and aes_decrypt_key with
+ * OpenSolaris-compatible functions rijndael_key_setup_enc_amd64 and
+ * rijndael_key_setup_dec_amd64
+ * 8. cstyled code and removed lint warnings
+ */
+
+#if defined(REDUCE_CODE_SIZE)
+#define ls_box ls_sub
+ uint32_t ls_sub(const uint32_t t, const uint32_t n);
+#define inv_mcol im_sub
+ uint32_t im_sub(const uint32_t x);
+#ifdef ENC_KS_UNROLL
+#undef ENC_KS_UNROLL
+#endif
+#ifdef DEC_KS_UNROLL
+#undef DEC_KS_UNROLL
+#endif
+#endif /* REDUCE_CODE_SIZE */
+
+
+#define ke4(k, i) \
+{ k[4 * (i) + 4] = ss[0] ^= ls_box(ss[3], 3) ^ t_use(r, c)[i]; \
+ k[4 * (i) + 5] = ss[1] ^= ss[0]; \
+ k[4 * (i) + 6] = ss[2] ^= ss[1]; \
+ k[4 * (i) + 7] = ss[3] ^= ss[2]; \
+}
+
+static void
+aes_encrypt_key128(const unsigned char *key, uint32_t rk[])
+{
+ uint32_t ss[4];
+
+ rk[0] = ss[0] = word_in(key, 0);
+ rk[1] = ss[1] = word_in(key, 1);
+ rk[2] = ss[2] = word_in(key, 2);
+ rk[3] = ss[3] = word_in(key, 3);
+
+#ifdef ENC_KS_UNROLL
+ ke4(rk, 0); ke4(rk, 1);
+ ke4(rk, 2); ke4(rk, 3);
+ ke4(rk, 4); ke4(rk, 5);
+ ke4(rk, 6); ke4(rk, 7);
+ ke4(rk, 8);
+#else
+ {
+ uint32_t i;
+ for (i = 0; i < 9; ++i)
+ ke4(rk, i);
+ }
+#endif /* ENC_KS_UNROLL */
+ ke4(rk, 9);
+}
+
+
+#define kef6(k, i) \
+{ k[6 * (i) + 6] = ss[0] ^= ls_box(ss[5], 3) ^ t_use(r, c)[i]; \
+ k[6 * (i) + 7] = ss[1] ^= ss[0]; \
+ k[6 * (i) + 8] = ss[2] ^= ss[1]; \
+ k[6 * (i) + 9] = ss[3] ^= ss[2]; \
+}
+
+#define ke6(k, i) \
+{ kef6(k, i); \
+ k[6 * (i) + 10] = ss[4] ^= ss[3]; \
+ k[6 * (i) + 11] = ss[5] ^= ss[4]; \
+}
+
+static void
+aes_encrypt_key192(const unsigned char *key, uint32_t rk[])
+{
+ uint32_t ss[6];
+
+ rk[0] = ss[0] = word_in(key, 0);
+ rk[1] = ss[1] = word_in(key, 1);
+ rk[2] = ss[2] = word_in(key, 2);
+ rk[3] = ss[3] = word_in(key, 3);
+ rk[4] = ss[4] = word_in(key, 4);
+ rk[5] = ss[5] = word_in(key, 5);
+
+#ifdef ENC_KS_UNROLL
+ ke6(rk, 0); ke6(rk, 1);
+ ke6(rk, 2); ke6(rk, 3);
+ ke6(rk, 4); ke6(rk, 5);
+ ke6(rk, 6);
+#else
+ {
+ uint32_t i;
+ for (i = 0; i < 7; ++i)
+ ke6(rk, i);
+ }
+#endif /* ENC_KS_UNROLL */
+ kef6(rk, 7);
+}
+
+
+
+#define kef8(k, i) \
+{ k[8 * (i) + 8] = ss[0] ^= ls_box(ss[7], 3) ^ t_use(r, c)[i]; \
+ k[8 * (i) + 9] = ss[1] ^= ss[0]; \
+ k[8 * (i) + 10] = ss[2] ^= ss[1]; \
+ k[8 * (i) + 11] = ss[3] ^= ss[2]; \
+}
+
+#define ke8(k, i) \
+{ kef8(k, i); \
+ k[8 * (i) + 12] = ss[4] ^= ls_box(ss[3], 0); \
+ k[8 * (i) + 13] = ss[5] ^= ss[4]; \
+ k[8 * (i) + 14] = ss[6] ^= ss[5]; \
+ k[8 * (i) + 15] = ss[7] ^= ss[6]; \
+}
+
+static void
+aes_encrypt_key256(const unsigned char *key, uint32_t rk[])
+{
+ uint32_t ss[8];
+
+ rk[0] = ss[0] = word_in(key, 0);
+ rk[1] = ss[1] = word_in(key, 1);
+ rk[2] = ss[2] = word_in(key, 2);
+ rk[3] = ss[3] = word_in(key, 3);
+ rk[4] = ss[4] = word_in(key, 4);
+ rk[5] = ss[5] = word_in(key, 5);
+ rk[6] = ss[6] = word_in(key, 6);
+ rk[7] = ss[7] = word_in(key, 7);
+
+#ifdef ENC_KS_UNROLL
+ ke8(rk, 0); ke8(rk, 1);
+ ke8(rk, 2); ke8(rk, 3);
+ ke8(rk, 4); ke8(rk, 5);
+#else
+ {
+ uint32_t i;
+ for (i = 0; i < 6; ++i)
+ ke8(rk, i);
+ }
+#endif /* ENC_KS_UNROLL */
+ kef8(rk, 6);
+}
+
+
+/*
+ * Expand the cipher key into the encryption key schedule.
+ *
+ * Return the number of rounds for the given cipher key size.
+ * The size of the key schedule depends on the number of rounds
+ * (which can be computed from the size of the key), i.e. 4 * (Nr + 1).
+ *
+ * Parameters:
+ * rk AES key schedule 32-bit array to be initialized
+ * cipherKey User key
+ * keyBits AES key size (128, 192, or 256 bits)
+ */
+int
+rijndael_key_setup_enc_amd64(uint32_t rk[], const uint32_t cipherKey[],
+ int keyBits)
+{
+ switch (keyBits) {
+ case 128:
+ aes_encrypt_key128((unsigned char *)&cipherKey[0], rk);
+ return (10);
+ case 192:
+ aes_encrypt_key192((unsigned char *)&cipherKey[0], rk);
+ return (12);
+ case 256:
+ aes_encrypt_key256((unsigned char *)&cipherKey[0], rk);
+ return (14);
+ default: /* should never get here */
+ break;
+ }
+
+ return (0);
+}
+
+
+/* this is used to store the decryption round keys */
+/* in forward or reverse order */
+
+#ifdef AES_REV_DKS
+#define v(n, i) ((n) - (i) + 2 * ((i) & 3))
+#else
+#define v(n, i) (i)
+#endif
+
+#if DEC_ROUND == NO_TABLES
+#define ff(x) (x)
+#else
+#define ff(x) inv_mcol(x)
+#if defined(dec_imvars)
+#define d_vars dec_imvars
+#endif
+#endif /* FUNCS_IN_C & DEC_KEYING_IN_C */
+
+
+#define k4e(k, i) \
+{ k[v(40, (4 * (i)) + 4)] = ss[0] ^= ls_box(ss[3], 3) ^ t_use(r, c)[i]; \
+ k[v(40, (4 * (i)) + 5)] = ss[1] ^= ss[0]; \
+ k[v(40, (4 * (i)) + 6)] = ss[2] ^= ss[1]; \
+ k[v(40, (4 * (i)) + 7)] = ss[3] ^= ss[2]; \
+}
+
+#if 1
+
+#define kdf4(k, i) \
+{ ss[0] = ss[0] ^ ss[2] ^ ss[1] ^ ss[3]; \
+ ss[1] = ss[1] ^ ss[3]; \
+ ss[2] = ss[2] ^ ss[3]; \
+ ss[4] = ls_box(ss[(i + 3) % 4], 3) ^ t_use(r, c)[i]; \
+ ss[i % 4] ^= ss[4]; \
+ ss[4] ^= k[v(40, (4 * (i)))]; k[v(40, (4 * (i)) + 4)] = ff(ss[4]); \
+ ss[4] ^= k[v(40, (4 * (i)) + 1)]; k[v(40, (4 * (i)) + 5)] = ff(ss[4]); \
+ ss[4] ^= k[v(40, (4 * (i)) + 2)]; k[v(40, (4 * (i)) + 6)] = ff(ss[4]); \
+ ss[4] ^= k[v(40, (4 * (i)) + 3)]; k[v(40, (4 * (i)) + 7)] = ff(ss[4]); \
+}
+
+#define kd4(k, i) \
+{ ss[4] = ls_box(ss[(i + 3) % 4], 3) ^ t_use(r, c)[i]; \
+ ss[i % 4] ^= ss[4]; ss[4] = ff(ss[4]); \
+ k[v(40, (4 * (i)) + 4)] = ss[4] ^= k[v(40, (4 * (i)))]; \
+ k[v(40, (4 * (i)) + 5)] = ss[4] ^= k[v(40, (4 * (i)) + 1)]; \
+ k[v(40, (4 * (i)) + 6)] = ss[4] ^= k[v(40, (4 * (i)) + 2)]; \
+ k[v(40, (4 * (i)) + 7)] = ss[4] ^= k[v(40, (4 * (i)) + 3)]; \
+}
+
+#define kdl4(k, i) \
+{ ss[4] = ls_box(ss[(i + 3) % 4], 3) ^ t_use(r, c)[i]; \
+ ss[i % 4] ^= ss[4]; \
+ k[v(40, (4 * (i)) + 4)] = (ss[0] ^= ss[1]) ^ ss[2] ^ ss[3]; \
+ k[v(40, (4 * (i)) + 5)] = ss[1] ^ ss[3]; \
+ k[v(40, (4 * (i)) + 6)] = ss[0]; \
+ k[v(40, (4 * (i)) + 7)] = ss[1]; \
+}
+
+#else
+
+#define kdf4(k, i) \
+{ ss[0] ^= ls_box(ss[3], 3) ^ t_use(r, c)[i]; \
+ k[v(40, (4 * (i)) + 4)] = ff(ss[0]); \
+ ss[1] ^= ss[0]; k[v(40, (4 * (i)) + 5)] = ff(ss[1]); \
+ ss[2] ^= ss[1]; k[v(40, (4 * (i)) + 6)] = ff(ss[2]); \
+ ss[3] ^= ss[2]; k[v(40, (4 * (i)) + 7)] = ff(ss[3]); \
+}
+
+#define kd4(k, i) \
+{ ss[4] = ls_box(ss[3], 3) ^ t_use(r, c)[i]; \
+ ss[0] ^= ss[4]; \
+ ss[4] = ff(ss[4]); \
+ k[v(40, (4 * (i)) + 4)] = ss[4] ^= k[v(40, (4 * (i)))]; \
+ ss[1] ^= ss[0]; \
+ k[v(40, (4 * (i)) + 5)] = ss[4] ^= k[v(40, (4 * (i)) + 1)]; \
+ ss[2] ^= ss[1]; \
+ k[v(40, (4 * (i)) + 6)] = ss[4] ^= k[v(40, (4 * (i)) + 2)]; \
+ ss[3] ^= ss[2]; \
+ k[v(40, (4 * (i)) + 7)] = ss[4] ^= k[v(40, (4 * (i)) + 3)]; \
+}
+
+#define kdl4(k, i) \
+{ ss[0] ^= ls_box(ss[3], 3) ^ t_use(r, c)[i]; \
+ k[v(40, (4 * (i)) + 4)] = ss[0]; \
+ ss[1] ^= ss[0]; k[v(40, (4 * (i)) + 5)] = ss[1]; \
+ ss[2] ^= ss[1]; k[v(40, (4 * (i)) + 6)] = ss[2]; \
+ ss[3] ^= ss[2]; k[v(40, (4 * (i)) + 7)] = ss[3]; \
+}
+
+#endif
+
+static void
+aes_decrypt_key128(const unsigned char *key, uint32_t rk[])
+{
+ uint32_t ss[5];
+#if defined(d_vars)
+ d_vars;
+#endif
+ rk[v(40, (0))] = ss[0] = word_in(key, 0);
+ rk[v(40, (1))] = ss[1] = word_in(key, 1);
+ rk[v(40, (2))] = ss[2] = word_in(key, 2);
+ rk[v(40, (3))] = ss[3] = word_in(key, 3);
+
+#ifdef DEC_KS_UNROLL
+ kdf4(rk, 0); kd4(rk, 1);
+ kd4(rk, 2); kd4(rk, 3);
+ kd4(rk, 4); kd4(rk, 5);
+ kd4(rk, 6); kd4(rk, 7);
+ kd4(rk, 8); kdl4(rk, 9);
+#else
+ {
+ uint32_t i;
+ for (i = 0; i < 10; ++i)
+ k4e(rk, i);
+#if !(DEC_ROUND == NO_TABLES)
+ for (i = MAX_AES_NB; i < 10 * MAX_AES_NB; ++i)
+ rk[i] = inv_mcol(rk[i]);
+#endif
+ }
+#endif /* DEC_KS_UNROLL */
+}
+
+
+
+#define k6ef(k, i) \
+{ k[v(48, (6 * (i)) + 6)] = ss[0] ^= ls_box(ss[5], 3) ^ t_use(r, c)[i]; \
+ k[v(48, (6 * (i)) + 7)] = ss[1] ^= ss[0]; \
+ k[v(48, (6 * (i)) + 8)] = ss[2] ^= ss[1]; \
+ k[v(48, (6 * (i)) + 9)] = ss[3] ^= ss[2]; \
+}
+
+#define k6e(k, i) \
+{ k6ef(k, i); \
+ k[v(48, (6 * (i)) + 10)] = ss[4] ^= ss[3]; \
+ k[v(48, (6 * (i)) + 11)] = ss[5] ^= ss[4]; \
+}
+
+#define kdf6(k, i) \
+{ ss[0] ^= ls_box(ss[5], 3) ^ t_use(r, c)[i]; \
+ k[v(48, (6 * (i)) + 6)] = ff(ss[0]); \
+ ss[1] ^= ss[0]; k[v(48, (6 * (i)) + 7)] = ff(ss[1]); \
+ ss[2] ^= ss[1]; k[v(48, (6 * (i)) + 8)] = ff(ss[2]); \
+ ss[3] ^= ss[2]; k[v(48, (6 * (i)) + 9)] = ff(ss[3]); \
+ ss[4] ^= ss[3]; k[v(48, (6 * (i)) + 10)] = ff(ss[4]); \
+ ss[5] ^= ss[4]; k[v(48, (6 * (i)) + 11)] = ff(ss[5]); \
+}
+
+#define kd6(k, i) \
+{ ss[6] = ls_box(ss[5], 3) ^ t_use(r, c)[i]; \
+ ss[0] ^= ss[6]; ss[6] = ff(ss[6]); \
+ k[v(48, (6 * (i)) + 6)] = ss[6] ^= k[v(48, (6 * (i)))]; \
+ ss[1] ^= ss[0]; \
+ k[v(48, (6 * (i)) + 7)] = ss[6] ^= k[v(48, (6 * (i)) + 1)]; \
+ ss[2] ^= ss[1]; \
+ k[v(48, (6 * (i)) + 8)] = ss[6] ^= k[v(48, (6 * (i)) + 2)]; \
+ ss[3] ^= ss[2]; \
+ k[v(48, (6 * (i)) + 9)] = ss[6] ^= k[v(48, (6 * (i)) + 3)]; \
+ ss[4] ^= ss[3]; \
+ k[v(48, (6 * (i)) + 10)] = ss[6] ^= k[v(48, (6 * (i)) + 4)]; \
+ ss[5] ^= ss[4]; \
+ k[v(48, (6 * (i)) + 11)] = ss[6] ^= k[v(48, (6 * (i)) + 5)]; \
+}
+
+#define kdl6(k, i) \
+{ ss[0] ^= ls_box(ss[5], 3) ^ t_use(r, c)[i]; \
+ k[v(48, (6 * (i)) + 6)] = ss[0]; \
+ ss[1] ^= ss[0]; k[v(48, (6 * (i)) + 7)] = ss[1]; \
+ ss[2] ^= ss[1]; k[v(48, (6 * (i)) + 8)] = ss[2]; \
+ ss[3] ^= ss[2]; k[v(48, (6 * (i)) + 9)] = ss[3]; \
+}
+
+static void
+aes_decrypt_key192(const unsigned char *key, uint32_t rk[])
+{
+ uint32_t ss[7];
+#if defined(d_vars)
+ d_vars;
+#endif
+ rk[v(48, (0))] = ss[0] = word_in(key, 0);
+ rk[v(48, (1))] = ss[1] = word_in(key, 1);
+ rk[v(48, (2))] = ss[2] = word_in(key, 2);
+ rk[v(48, (3))] = ss[3] = word_in(key, 3);
+
+#ifdef DEC_KS_UNROLL
+ ss[4] = word_in(key, 4);
+ rk[v(48, (4))] = ff(ss[4]);
+ ss[5] = word_in(key, 5);
+ rk[v(48, (5))] = ff(ss[5]);
+ kdf6(rk, 0); kd6(rk, 1);
+ kd6(rk, 2); kd6(rk, 3);
+ kd6(rk, 4); kd6(rk, 5);
+ kd6(rk, 6); kdl6(rk, 7);
+#else
+ rk[v(48, (4))] = ss[4] = word_in(key, 4);
+ rk[v(48, (5))] = ss[5] = word_in(key, 5);
+ {
+ uint32_t i;
+
+ for (i = 0; i < 7; ++i)
+ k6e(rk, i);
+ k6ef(rk, 7);
+#if !(DEC_ROUND == NO_TABLES)
+ for (i = MAX_AES_NB; i < 12 * MAX_AES_NB; ++i)
+ rk[i] = inv_mcol(rk[i]);
+#endif
+ }
+#endif
+}
+
+
+
+#define k8ef(k, i) \
+{ k[v(56, (8 * (i)) + 8)] = ss[0] ^= ls_box(ss[7], 3) ^ t_use(r, c)[i]; \
+ k[v(56, (8 * (i)) + 9)] = ss[1] ^= ss[0]; \
+ k[v(56, (8 * (i)) + 10)] = ss[2] ^= ss[1]; \
+ k[v(56, (8 * (i)) + 11)] = ss[3] ^= ss[2]; \
+}
+
+#define k8e(k, i) \
+{ k8ef(k, i); \
+ k[v(56, (8 * (i)) + 12)] = ss[4] ^= ls_box(ss[3], 0); \
+ k[v(56, (8 * (i)) + 13)] = ss[5] ^= ss[4]; \
+ k[v(56, (8 * (i)) + 14)] = ss[6] ^= ss[5]; \
+ k[v(56, (8 * (i)) + 15)] = ss[7] ^= ss[6]; \
+}
+
+#define kdf8(k, i) \
+{ ss[0] ^= ls_box(ss[7], 3) ^ t_use(r, c)[i]; \
+ k[v(56, (8 * (i)) + 8)] = ff(ss[0]); \
+ ss[1] ^= ss[0]; k[v(56, (8 * (i)) + 9)] = ff(ss[1]); \
+ ss[2] ^= ss[1]; k[v(56, (8 * (i)) + 10)] = ff(ss[2]); \
+ ss[3] ^= ss[2]; k[v(56, (8 * (i)) + 11)] = ff(ss[3]); \
+ ss[4] ^= ls_box(ss[3], 0); k[v(56, (8 * (i)) + 12)] = ff(ss[4]); \
+ ss[5] ^= ss[4]; k[v(56, (8 * (i)) + 13)] = ff(ss[5]); \
+ ss[6] ^= ss[5]; k[v(56, (8 * (i)) + 14)] = ff(ss[6]); \
+ ss[7] ^= ss[6]; k[v(56, (8 * (i)) + 15)] = ff(ss[7]); \
+}
+
+#define kd8(k, i) \
+{ ss[8] = ls_box(ss[7], 3) ^ t_use(r, c)[i]; \
+ ss[0] ^= ss[8]; \
+ ss[8] = ff(ss[8]); \
+ k[v(56, (8 * (i)) + 8)] = ss[8] ^= k[v(56, (8 * (i)))]; \
+ ss[1] ^= ss[0]; \
+ k[v(56, (8 * (i)) + 9)] = ss[8] ^= k[v(56, (8 * (i)) + 1)]; \
+ ss[2] ^= ss[1]; \
+ k[v(56, (8 * (i)) + 10)] = ss[8] ^= k[v(56, (8 * (i)) + 2)]; \
+ ss[3] ^= ss[2]; \
+ k[v(56, (8 * (i)) + 11)] = ss[8] ^= k[v(56, (8 * (i)) + 3)]; \
+ ss[8] = ls_box(ss[3], 0); \
+ ss[4] ^= ss[8]; \
+ ss[8] = ff(ss[8]); \
+ k[v(56, (8 * (i)) + 12)] = ss[8] ^= k[v(56, (8 * (i)) + 4)]; \
+ ss[5] ^= ss[4]; \
+ k[v(56, (8 * (i)) + 13)] = ss[8] ^= k[v(56, (8 * (i)) + 5)]; \
+ ss[6] ^= ss[5]; \
+ k[v(56, (8 * (i)) + 14)] = ss[8] ^= k[v(56, (8 * (i)) + 6)]; \
+ ss[7] ^= ss[6]; \
+ k[v(56, (8 * (i)) + 15)] = ss[8] ^= k[v(56, (8 * (i)) + 7)]; \
+}
+
+#define kdl8(k, i) \
+{ ss[0] ^= ls_box(ss[7], 3) ^ t_use(r, c)[i]; \
+ k[v(56, (8 * (i)) + 8)] = ss[0]; \
+ ss[1] ^= ss[0]; k[v(56, (8 * (i)) + 9)] = ss[1]; \
+ ss[2] ^= ss[1]; k[v(56, (8 * (i)) + 10)] = ss[2]; \
+ ss[3] ^= ss[2]; k[v(56, (8 * (i)) + 11)] = ss[3]; \
+}
+
+static void
+aes_decrypt_key256(const unsigned char *key, uint32_t rk[])
+{
+ uint32_t ss[9];
+#if defined(d_vars)
+ d_vars;
+#endif
+ rk[v(56, (0))] = ss[0] = word_in(key, 0);
+ rk[v(56, (1))] = ss[1] = word_in(key, 1);
+ rk[v(56, (2))] = ss[2] = word_in(key, 2);
+ rk[v(56, (3))] = ss[3] = word_in(key, 3);
+
+#ifdef DEC_KS_UNROLL
+ ss[4] = word_in(key, 4);
+ rk[v(56, (4))] = ff(ss[4]);
+ ss[5] = word_in(key, 5);
+ rk[v(56, (5))] = ff(ss[5]);
+ ss[6] = word_in(key, 6);
+ rk[v(56, (6))] = ff(ss[6]);
+ ss[7] = word_in(key, 7);
+ rk[v(56, (7))] = ff(ss[7]);
+ kdf8(rk, 0); kd8(rk, 1);
+ kd8(rk, 2); kd8(rk, 3);
+ kd8(rk, 4); kd8(rk, 5);
+ kdl8(rk, 6);
+#else
+ rk[v(56, (4))] = ss[4] = word_in(key, 4);
+ rk[v(56, (5))] = ss[5] = word_in(key, 5);
+ rk[v(56, (6))] = ss[6] = word_in(key, 6);
+ rk[v(56, (7))] = ss[7] = word_in(key, 7);
+ {
+ uint32_t i;
+
+ for (i = 0; i < 6; ++i)
+ k8e(rk, i);
+ k8ef(rk, 6);
+#if !(DEC_ROUND == NO_TABLES)
+ for (i = MAX_AES_NB; i < 14 * MAX_AES_NB; ++i)
+ rk[i] = inv_mcol(rk[i]);
+#endif
+ }
+#endif /* DEC_KS_UNROLL */
+}
+
+
+/*
+ * Expand the cipher key into the decryption key schedule.
+ *
+ * Return the number of rounds for the given cipher key size.
+ * The size of the key schedule depends on the number of rounds
+ * (which can be computed from the size of the key), i.e. 4 * (Nr + 1).
+ *
+ * Parameters:
+ * rk AES key schedule 32-bit array to be initialized
+ * cipherKey User key
+ * keyBits AES key size (128, 192, or 256 bits)
+ */
+int
+rijndael_key_setup_dec_amd64(uint32_t rk[], const uint32_t cipherKey[],
+ int keyBits)
+{
+ switch (keyBits) {
+ case 128:
+ aes_decrypt_key128((unsigned char *)&cipherKey[0], rk);
+ return (10);
+ case 192:
+ aes_decrypt_key192((unsigned char *)&cipherKey[0], rk);
+ return (12);
+ case 256:
+ aes_decrypt_key256((unsigned char *)&cipherKey[0], rk);
+ return (14);
+ default: /* should never get here */
+ break;
+ }
+
+ return (0);
+}
--- /dev/null
+/*
+ * ---------------------------------------------------------------------------
+ * Copyright (c) 1998-2007, Brian Gladman, Worcester, UK. All rights reserved.
+ *
+ * LICENSE TERMS
+ *
+ * The free distribution and use of this software is allowed (with or without
+ * changes) provided that:
+ *
+ * 1. source code distributions include the above copyright notice, this
+ * list of conditions and the following disclaimer;
+ *
+ * 2. binary distributions include the above copyright notice, this list
+ * of conditions and the following disclaimer in their documentation;
+ *
+ * 3. the name of the copyright holder is not used to endorse products
+ * built using this software without specific written permission.
+ *
+ * DISCLAIMER
+ *
+ * This software is provided 'as is' with no explicit or implied warranties
+ * in respect of its properties, including, but not limited to, correctness
+ * and/or fitness for purpose.
+ * ---------------------------------------------------------------------------
+ * Issue Date: 20/12/2007
+ *
+ * This file contains the compilation options for AES (Rijndael) and code
+ * that is common across encryption, key scheduling and table generation.
+ *
+ * OPERATION
+ *
+ * These source code files implement the AES algorithm Rijndael designed by
+ * Joan Daemen and Vincent Rijmen. This version is designed for the standard
+ * block size of 16 bytes and for key sizes of 128, 192 and 256 bits (16, 24
+ * and 32 bytes).
+ *
+ * This version is designed for flexibility and speed using operations on
+ * 32-bit words rather than operations on bytes. It can be compiled with
+ * either big or little endian internal byte order but is faster when the
+ * native byte order for the processor is used.
+ *
+ * THE CIPHER INTERFACE
+ *
+ * The cipher interface is implemented as an array of bytes in which lower
+ * AES bit sequence indexes map to higher numeric significance within bytes.
+ */
+
+/*
+ * OpenSolaris changes
+ * 1. Added __cplusplus and _AESTAB_H header guards
+ * 2. Added header files sys/types.h and aes_impl.h
+ * 3. Added defines for AES_ENCRYPT, AES_DECRYPT, AES_REV_DKS, and ASM_AMD64_C
+ * 4. Moved defines for IS_BIG_ENDIAN, IS_LITTLE_ENDIAN, PLATFORM_BYTE_ORDER
+ * from brg_endian.h
+ * 5. Undefined VIA_ACE_POSSIBLE and ASSUME_VIA_ACE_PRESENT
+ * 6. Changed uint_8t and uint_32t to uint8_t and uint32_t
+ * 7. Defined aes_sw32 as htonl() for byte swapping
+ * 8. Cstyled and hdrchk code
+ *
+ */
+
+#ifndef _AESOPT_H
+#define _AESOPT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+#include <aes/aes_impl.h>
+
+/* SUPPORT FEATURES */
+#define AES_ENCRYPT /* if support for encryption is needed */
+#define AES_DECRYPT /* if support for decryption is needed */
+
+/* PLATFORM-SPECIFIC FEATURES */
+#define IS_BIG_ENDIAN 4321 /* byte 0 is most significant (mc68k) */
+#define IS_LITTLE_ENDIAN 1234 /* byte 0 is least significant (i386) */
+#define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN
+#define AES_REV_DKS /* define to reverse decryption key schedule */
+
+
+/*
+ * CONFIGURATION - THE USE OF DEFINES
+ * Later in this section there are a number of defines that control the
+ * operation of the code. In each section, the purpose of each define is
+ * explained so that the relevant form can be included or excluded by
+ * setting either 1's or 0's respectively on the branches of the related
+ * #if clauses. The following local defines should not be changed.
+ */
+
+#define ENCRYPTION_IN_C 1
+#define DECRYPTION_IN_C 2
+#define ENC_KEYING_IN_C 4
+#define DEC_KEYING_IN_C 8
+
+#define NO_TABLES 0
+#define ONE_TABLE 1
+#define FOUR_TABLES 4
+#define NONE 0
+#define PARTIAL 1
+#define FULL 2
+
+/* --- START OF USER CONFIGURED OPTIONS --- */
+
+/*
+ * 1. BYTE ORDER WITHIN 32 BIT WORDS
+ *
+ * The fundamental data processing units in Rijndael are 8-bit bytes. The
+ * input, output and key input are all enumerated arrays of bytes in which
+ * bytes are numbered starting at zero and increasing to one less than the
+ * number of bytes in the array in question. This enumeration is only used
+ * for naming bytes and does not imply any adjacency or order relationship
+ * from one byte to another. When these inputs and outputs are considered
+ * as bit sequences, bits 8*n to 8*n+7 of the bit sequence are mapped to
+ * byte[n] with bit 8n+i in the sequence mapped to bit 7-i within the byte.
+ * In this implementation bits are numbered from 0 to 7 starting at the
+ * numerically least significant end of each byte. Bit n represents 2^n.
+ *
+ * However, Rijndael can be implemented more efficiently using 32-bit
+ * words by packing bytes into words so that bytes 4*n to 4*n+3 are placed
+ * into word[n]. While in principle these bytes can be assembled into words
+ * in any positions, this implementation only supports the two formats in
+ * which bytes in adjacent positions within words also have adjacent byte
+ * numbers. This order is called big-endian if the lowest numbered bytes
+ * in words have the highest numeric significance and little-endian if the
+ * opposite applies.
+ *
+ * This code can work in either order irrespective of the order used by the
+ * machine on which it runs. Normally the internal byte order will be set
+ * to the order of the processor on which the code is to be run but this
+ * define can be used to reverse this in special situations
+ *
+ * WARNING: Assembler code versions rely on PLATFORM_BYTE_ORDER being set.
+ * This define will hence be redefined later (in section 4) if necessary
+ */
+
+#if 1
+#define ALGORITHM_BYTE_ORDER PLATFORM_BYTE_ORDER
+#elif 0
+#define ALGORITHM_BYTE_ORDER IS_LITTLE_ENDIAN
+#elif 0
+#define ALGORITHM_BYTE_ORDER IS_BIG_ENDIAN
+#else
+#error The algorithm byte order is not defined
+#endif
+
+/* 2. VIA ACE SUPPORT */
+
+#if defined(__GNUC__) && defined(__i386__) || \
+ defined(_WIN32) && defined(_M_IX86) && \
+ !(defined(_WIN64) || defined(_WIN32_WCE) || \
+ defined(_MSC_VER) && (_MSC_VER <= 800))
+#define VIA_ACE_POSSIBLE
+#endif
+
+/*
+ * Define this option if support for the VIA ACE is required. This uses
+ * inline assembler instructions and is only implemented for the Microsoft,
+ * Intel and GCC compilers. If VIA ACE is known to be present, then defining
+ * ASSUME_VIA_ACE_PRESENT will remove the ordinary encryption/decryption
+ * code. If USE_VIA_ACE_IF_PRESENT is defined then VIA ACE will be used if
+ * it is detected (both present and enabled) but the normal AES code will
+ * also be present.
+ *
+ * When VIA ACE is to be used, all AES encryption contexts MUST be 16 byte
+ * aligned; other input/output buffers do not need to be 16 byte aligned
+ * but there are very large performance gains if this can be arranged.
+ * VIA ACE also requires the decryption key schedule to be in reverse
+ * order (which later checks below ensure).
+ */
+
+/* VIA ACE is not used here for OpenSolaris: */
+#undef VIA_ACE_POSSIBLE
+#undef ASSUME_VIA_ACE_PRESENT
+
+#if 0 && defined(VIA_ACE_POSSIBLE) && !defined(USE_VIA_ACE_IF_PRESENT)
+#define USE_VIA_ACE_IF_PRESENT
+#endif
+
+#if 0 && defined(VIA_ACE_POSSIBLE) && !defined(ASSUME_VIA_ACE_PRESENT)
+#define ASSUME_VIA_ACE_PRESENT
+#endif
+
+
+/*
+ * 3. ASSEMBLER SUPPORT
+ *
+ * This define (which can be on the command line) enables the use of the
+ * assembler code routines for encryption, decryption and key scheduling
+ * as follows:
+ *
+ * ASM_X86_V1C uses the assembler (aes_x86_v1.asm) with large tables for
+ * encryption and decryption and but with key scheduling in C
+ * ASM_X86_V2 uses assembler (aes_x86_v2.asm) with compressed tables for
+ * encryption, decryption and key scheduling
+ * ASM_X86_V2C uses assembler (aes_x86_v2.asm) with compressed tables for
+ * encryption and decryption and but with key scheduling in C
+ * ASM_AMD64_C uses assembler (aes_amd64.asm) with compressed tables for
+ * encryption and decryption and but with key scheduling in C
+ *
+ * Change one 'if 0' below to 'if 1' to select the version or define
+ * as a compilation option.
+ */
+
+#if 0 && !defined(ASM_X86_V1C)
+#define ASM_X86_V1C
+#elif 0 && !defined(ASM_X86_V2)
+#define ASM_X86_V2
+#elif 0 && !defined(ASM_X86_V2C)
+#define ASM_X86_V2C
+#elif 1 && !defined(ASM_AMD64_C)
+#define ASM_AMD64_C
+#endif
+
+#if (defined(ASM_X86_V1C) || defined(ASM_X86_V2) || defined(ASM_X86_V2C)) && \
+ !defined(_M_IX86) || defined(ASM_AMD64_C) && !defined(_M_X64) && \
+ !defined(__amd64)
+#error Assembler code is only available for x86 and AMD64 systems
+#endif
+
+/*
+ * 4. FAST INPUT/OUTPUT OPERATIONS.
+ *
+ * On some machines it is possible to improve speed by transferring the
+ * bytes in the input and output arrays to and from the internal 32-bit
+ * variables by addressing these arrays as if they are arrays of 32-bit
+ * words. On some machines this will always be possible but there may
+ * be a large performance penalty if the byte arrays are not aligned on
+ * the normal word boundaries. On other machines this technique will
+ * lead to memory access errors when such 32-bit word accesses are not
+ * properly aligned. The option SAFE_IO avoids such problems but will
+ * often be slower on those machines that support misaligned access
+ * (especially so if care is taken to align the input and output byte
+ * arrays on 32-bit word boundaries). If SAFE_IO is not defined it is
+ * assumed that access to byte arrays as if they are arrays of 32-bit
+ * words will not cause problems when such accesses are misaligned.
+ */
+#if 1 && !defined(_MSC_VER)
+#define SAFE_IO
+#endif
+
+/*
+ * 5. LOOP UNROLLING
+ *
+ * The code for encryption and decryption cycles through a number of rounds
+ * that can be implemented either in a loop or by expanding the code into a
+ * long sequence of instructions, the latter producing a larger program but
+ * one that will often be much faster. The latter is called loop unrolling.
+ * There are also potential speed advantages in expanding two iterations in
+ * a loop with half the number of iterations, which is called partial loop
+ * unrolling. The following options allow partial or full loop unrolling
+ * to be set independently for encryption and decryption
+ */
+#if 1
+#define ENC_UNROLL FULL
+#elif 0
+#define ENC_UNROLL PARTIAL
+#else
+#define ENC_UNROLL NONE
+#endif
+
+#if 1
+#define DEC_UNROLL FULL
+#elif 0
+#define DEC_UNROLL PARTIAL
+#else
+#define DEC_UNROLL NONE
+#endif
+
+#if 1
+#define ENC_KS_UNROLL
+#endif
+
+#if 1
+#define DEC_KS_UNROLL
+#endif
+
+/*
+ * 6. FAST FINITE FIELD OPERATIONS
+ *
+ * If this section is included, tables are used to provide faster finite
+ * field arithmetic. This has no effect if FIXED_TABLES is defined.
+ */
+#if 1
+#define FF_TABLES
+#endif
+
+/*
+ * 7. INTERNAL STATE VARIABLE FORMAT
+ *
+ * The internal state of Rijndael is stored in a number of local 32-bit
+ * word variables which can be defined either as an array or as individual
+ * names variables. Include this section if you want to store these local
+ * variables in arrays. Otherwise individual local variables will be used.
+ */
+#if 1
+#define ARRAYS
+#endif
+
+/*
+ * 8. FIXED OR DYNAMIC TABLES
+ *
+ * When this section is included the tables used by the code are compiled
+ * statically into the binary file. Otherwise the subroutine aes_init()
+ * must be called to compute them before the code is first used.
+ */
+#if 1 && !(defined(_MSC_VER) && (_MSC_VER <= 800))
+#define FIXED_TABLES
+#endif
+
+/*
+ * 9. MASKING OR CASTING FROM LONGER VALUES TO BYTES
+ *
+ * In some systems it is better to mask longer values to extract bytes
+ * rather than using a cast. This option allows this choice.
+ */
+#if 0
+#define to_byte(x) ((uint8_t)(x))
+#else
+#define to_byte(x) ((x) & 0xff)
+#endif
+
+/*
+ * 10. TABLE ALIGNMENT
+ *
+ * On some systems speed will be improved by aligning the AES large lookup
+ * tables on particular boundaries. This define should be set to a power of
+ * two giving the desired alignment. It can be left undefined if alignment
+ * is not needed. This option is specific to the Micrsoft VC++ compiler -
+ * it seems to sometimes cause trouble for the VC++ version 6 compiler.
+ */
+
+#if 1 && defined(_MSC_VER) && (_MSC_VER >= 1300)
+#define TABLE_ALIGN 32
+#endif
+
+/*
+ * 11. REDUCE CODE AND TABLE SIZE
+ *
+ * This replaces some expanded macros with function calls if AES_ASM_V2 or
+ * AES_ASM_V2C are defined
+ */
+
+#if 1 && (defined(ASM_X86_V2) || defined(ASM_X86_V2C))
+#define REDUCE_CODE_SIZE
+#endif
+
+/*
+ * 12. TABLE OPTIONS
+ *
+ * This cipher proceeds by repeating in a number of cycles known as rounds
+ * which are implemented by a round function which is optionally be speeded
+ * up using tables. The basic tables are 256 32-bit words, with either
+ * one or four tables being required for each round function depending on
+ * how much speed is required. Encryption and decryption round functions
+ * are different and the last encryption and decryption round functions are
+ * different again making four different round functions in all.
+ *
+ * This means that:
+ * 1. Normal encryption and decryption rounds can each use either 0, 1
+ * or 4 tables and table spaces of 0, 1024 or 4096 bytes each.
+ * 2. The last encryption and decryption rounds can also use either 0, 1
+ * or 4 tables and table spaces of 0, 1024 or 4096 bytes each.
+ *
+ * Include or exclude the appropriate definitions below to set the number
+ * of tables used by this implementation.
+ */
+
+#if 1 /* set tables for the normal encryption round */
+#define ENC_ROUND FOUR_TABLES
+#elif 0
+#define ENC_ROUND ONE_TABLE
+#else
+#define ENC_ROUND NO_TABLES
+#endif
+
+#if 1 /* set tables for the last encryption round */
+#define LAST_ENC_ROUND FOUR_TABLES
+#elif 0
+#define LAST_ENC_ROUND ONE_TABLE
+#else
+#define LAST_ENC_ROUND NO_TABLES
+#endif
+
+#if 1 /* set tables for the normal decryption round */
+#define DEC_ROUND FOUR_TABLES
+#elif 0
+#define DEC_ROUND ONE_TABLE
+#else
+#define DEC_ROUND NO_TABLES
+#endif
+
+#if 1 /* set tables for the last decryption round */
+#define LAST_DEC_ROUND FOUR_TABLES
+#elif 0
+#define LAST_DEC_ROUND ONE_TABLE
+#else
+#define LAST_DEC_ROUND NO_TABLES
+#endif
+
+/*
+ * The decryption key schedule can be speeded up with tables in the same
+ * way that the round functions can. Include or exclude the following
+ * defines to set this requirement.
+ */
+#if 1
+#define KEY_SCHED FOUR_TABLES
+#elif 0
+#define KEY_SCHED ONE_TABLE
+#else
+#define KEY_SCHED NO_TABLES
+#endif
+
+/* ---- END OF USER CONFIGURED OPTIONS ---- */
+
+/* VIA ACE support is only available for VC++ and GCC */
+
+#if !defined(_MSC_VER) && !defined(__GNUC__)
+#if defined(ASSUME_VIA_ACE_PRESENT)
+#undef ASSUME_VIA_ACE_PRESENT
+#endif
+#if defined(USE_VIA_ACE_IF_PRESENT)
+#undef USE_VIA_ACE_IF_PRESENT
+#endif
+#endif
+
+#if defined(ASSUME_VIA_ACE_PRESENT) && !defined(USE_VIA_ACE_IF_PRESENT)
+#define USE_VIA_ACE_IF_PRESENT
+#endif
+
+#if defined(USE_VIA_ACE_IF_PRESENT) && !defined(AES_REV_DKS)
+#define AES_REV_DKS
+#endif
+
+/* Assembler support requires the use of platform byte order */
+
+#if (defined(ASM_X86_V1C) || defined(ASM_X86_V2C) || defined(ASM_AMD64_C)) && \
+ (ALGORITHM_BYTE_ORDER != PLATFORM_BYTE_ORDER)
+#undef ALGORITHM_BYTE_ORDER
+#define ALGORITHM_BYTE_ORDER PLATFORM_BYTE_ORDER
+#endif
+
+/*
+ * In this implementation the columns of the state array are each held in
+ * 32-bit words. The state array can be held in various ways: in an array
+ * of words, in a number of individual word variables or in a number of
+ * processor registers. The following define maps a variable name x and
+ * a column number c to the way the state array variable is to be held.
+ * The first define below maps the state into an array x[c] whereas the
+ * second form maps the state into a number of individual variables x0,
+ * x1, etc. Another form could map individual state columns to machine
+ * register names.
+ */
+
+#if defined(ARRAYS)
+#define s(x, c) x[c]
+#else
+#define s(x, c) x##c
+#endif
+
+/*
+ * This implementation provides subroutines for encryption, decryption
+ * and for setting the three key lengths (separately) for encryption
+ * and decryption. Since not all functions are needed, masks are set
+ * up here to determine which will be implemented in C
+ */
+
+#if !defined(AES_ENCRYPT)
+#define EFUNCS_IN_C 0
+#elif defined(ASSUME_VIA_ACE_PRESENT) || defined(ASM_X86_V1C) || \
+ defined(ASM_X86_V2C) || defined(ASM_AMD64_C)
+#define EFUNCS_IN_C ENC_KEYING_IN_C
+#elif !defined(ASM_X86_V2)
+#define EFUNCS_IN_C (ENCRYPTION_IN_C | ENC_KEYING_IN_C)
+#else
+#define EFUNCS_IN_C 0
+#endif
+
+#if !defined(AES_DECRYPT)
+#define DFUNCS_IN_C 0
+#elif defined(ASSUME_VIA_ACE_PRESENT) || defined(ASM_X86_V1C) || \
+ defined(ASM_X86_V2C) || defined(ASM_AMD64_C)
+#define DFUNCS_IN_C DEC_KEYING_IN_C
+#elif !defined(ASM_X86_V2)
+#define DFUNCS_IN_C (DECRYPTION_IN_C | DEC_KEYING_IN_C)
+#else
+#define DFUNCS_IN_C 0
+#endif
+
+#define FUNCS_IN_C (EFUNCS_IN_C | DFUNCS_IN_C)
+
+/* END OF CONFIGURATION OPTIONS */
+
+/* Disable or report errors on some combinations of options */
+
+#if ENC_ROUND == NO_TABLES && LAST_ENC_ROUND != NO_TABLES
+#undef LAST_ENC_ROUND
+#define LAST_ENC_ROUND NO_TABLES
+#elif ENC_ROUND == ONE_TABLE && LAST_ENC_ROUND == FOUR_TABLES
+#undef LAST_ENC_ROUND
+#define LAST_ENC_ROUND ONE_TABLE
+#endif
+
+#if ENC_ROUND == NO_TABLES && ENC_UNROLL != NONE
+#undef ENC_UNROLL
+#define ENC_UNROLL NONE
+#endif
+
+#if DEC_ROUND == NO_TABLES && LAST_DEC_ROUND != NO_TABLES
+#undef LAST_DEC_ROUND
+#define LAST_DEC_ROUND NO_TABLES
+#elif DEC_ROUND == ONE_TABLE && LAST_DEC_ROUND == FOUR_TABLES
+#undef LAST_DEC_ROUND
+#define LAST_DEC_ROUND ONE_TABLE
+#endif
+
+#if DEC_ROUND == NO_TABLES && DEC_UNROLL != NONE
+#undef DEC_UNROLL
+#define DEC_UNROLL NONE
+#endif
+
+#if (ALGORITHM_BYTE_ORDER == IS_LITTLE_ENDIAN)
+#define aes_sw32 htonl
+#elif defined(bswap32)
+#define aes_sw32 bswap32
+#elif defined(bswap_32)
+#define aes_sw32 bswap_32
+#else
+#define brot(x, n) (((uint32_t)(x) << (n)) | ((uint32_t)(x) >> (32 - (n))))
+#define aes_sw32(x) ((brot((x), 8) & 0x00ff00ff) | (brot((x), 24) & 0xff00ff00))
+#endif
+
+
+/*
+ * upr(x, n): rotates bytes within words by n positions, moving bytes to
+ * higher index positions with wrap around into low positions
+ * ups(x, n): moves bytes by n positions to higher index positions in
+ * words but without wrap around
+ * bval(x, n): extracts a byte from a word
+ *
+ * WARNING: The definitions given here are intended only for use with
+ * unsigned variables and with shift counts that are compile
+ * time constants
+ */
+
+#if (ALGORITHM_BYTE_ORDER == IS_LITTLE_ENDIAN)
+#define upr(x, n) (((uint32_t)(x) << (8 * (n))) | \
+ ((uint32_t)(x) >> (32 - 8 * (n))))
+#define ups(x, n) ((uint32_t)(x) << (8 * (n)))
+#define bval(x, n) to_byte((x) >> (8 * (n)))
+#define bytes2word(b0, b1, b2, b3) \
+ (((uint32_t)(b3) << 24) | ((uint32_t)(b2) << 16) | \
+ ((uint32_t)(b1) << 8) | (b0))
+#endif
+
+#if (ALGORITHM_BYTE_ORDER == IS_BIG_ENDIAN)
+#define upr(x, n) (((uint32_t)(x) >> (8 * (n))) | \
+ ((uint32_t)(x) << (32 - 8 * (n))))
+#define ups(x, n) ((uint32_t)(x) >> (8 * (n)))
+#define bval(x, n) to_byte((x) >> (24 - 8 * (n)))
+#define bytes2word(b0, b1, b2, b3) \
+ (((uint32_t)(b0) << 24) | ((uint32_t)(b1) << 16) | \
+ ((uint32_t)(b2) << 8) | (b3))
+#endif
+
+#if defined(SAFE_IO)
+#define word_in(x, c) bytes2word(((const uint8_t *)(x) + 4 * c)[0], \
+ ((const uint8_t *)(x) + 4 * c)[1], \
+ ((const uint8_t *)(x) + 4 * c)[2], \
+ ((const uint8_t *)(x) + 4 * c)[3])
+#define word_out(x, c, v) { ((uint8_t *)(x) + 4 * c)[0] = bval(v, 0); \
+ ((uint8_t *)(x) + 4 * c)[1] = bval(v, 1); \
+ ((uint8_t *)(x) + 4 * c)[2] = bval(v, 2); \
+ ((uint8_t *)(x) + 4 * c)[3] = bval(v, 3); }
+#elif (ALGORITHM_BYTE_ORDER == PLATFORM_BYTE_ORDER)
+#define word_in(x, c) (*((uint32_t *)(x) + (c)))
+#define word_out(x, c, v) (*((uint32_t *)(x) + (c)) = (v))
+#else
+#define word_in(x, c) aes_sw32(*((uint32_t *)(x) + (c)))
+#define word_out(x, c, v) (*((uint32_t *)(x) + (c)) = aes_sw32(v))
+#endif
+
+/* the finite field modular polynomial and elements */
+
+#define WPOLY 0x011b
+#define BPOLY 0x1b
+
+/* multiply four bytes in GF(2^8) by 'x' {02} in parallel */
+
+#define m1 0x80808080
+#define m2 0x7f7f7f7f
+#define gf_mulx(x) ((((x) & m2) << 1) ^ ((((x) & m1) >> 7) * BPOLY))
+
+/*
+ * The following defines provide alternative definitions of gf_mulx that might
+ * give improved performance if a fast 32-bit multiply is not available. Note
+ * that a temporary variable u needs to be defined where gf_mulx is used.
+ *
+ * #define gf_mulx(x) (u = (x) & m1, u |= (u >> 1), ((x) & m2) << 1) ^ \
+ * ((u >> 3) | (u >> 6))
+ * #define m4 (0x01010101 * BPOLY)
+ * #define gf_mulx(x) (u = (x) & m1, ((x) & m2) << 1) ^ ((u - (u >> 7)) \
+ * & m4)
+ */
+
+/* Work out which tables are needed for the different options */
+
+#if defined(ASM_X86_V1C)
+#if defined(ENC_ROUND)
+#undef ENC_ROUND
+#endif
+#define ENC_ROUND FOUR_TABLES
+#if defined(LAST_ENC_ROUND)
+#undef LAST_ENC_ROUND
+#endif
+#define LAST_ENC_ROUND FOUR_TABLES
+#if defined(DEC_ROUND)
+#undef DEC_ROUND
+#endif
+#define DEC_ROUND FOUR_TABLES
+#if defined(LAST_DEC_ROUND)
+#undef LAST_DEC_ROUND
+#endif
+#define LAST_DEC_ROUND FOUR_TABLES
+#if defined(KEY_SCHED)
+#undef KEY_SCHED
+#define KEY_SCHED FOUR_TABLES
+#endif
+#endif
+
+#if (FUNCS_IN_C & ENCRYPTION_IN_C) || defined(ASM_X86_V1C)
+#if ENC_ROUND == ONE_TABLE
+#define FT1_SET
+#elif ENC_ROUND == FOUR_TABLES
+#define FT4_SET
+#else
+#define SBX_SET
+#endif
+#if LAST_ENC_ROUND == ONE_TABLE
+#define FL1_SET
+#elif LAST_ENC_ROUND == FOUR_TABLES
+#define FL4_SET
+#elif !defined(SBX_SET)
+#define SBX_SET
+#endif
+#endif
+
+#if (FUNCS_IN_C & DECRYPTION_IN_C) || defined(ASM_X86_V1C)
+#if DEC_ROUND == ONE_TABLE
+#define IT1_SET
+#elif DEC_ROUND == FOUR_TABLES
+#define IT4_SET
+#else
+#define ISB_SET
+#endif
+#if LAST_DEC_ROUND == ONE_TABLE
+#define IL1_SET
+#elif LAST_DEC_ROUND == FOUR_TABLES
+#define IL4_SET
+#elif !defined(ISB_SET)
+#define ISB_SET
+#endif
+#endif
+
+
+#if !(defined(REDUCE_CODE_SIZE) && (defined(ASM_X86_V2) || \
+ defined(ASM_X86_V2C)))
+#if ((FUNCS_IN_C & ENC_KEYING_IN_C) || (FUNCS_IN_C & DEC_KEYING_IN_C))
+#if KEY_SCHED == ONE_TABLE
+#if !defined(FL1_SET) && !defined(FL4_SET)
+#define LS1_SET
+#endif
+#elif KEY_SCHED == FOUR_TABLES
+#if !defined(FL4_SET)
+#define LS4_SET
+#endif
+#elif !defined(SBX_SET)
+#define SBX_SET
+#endif
+#endif
+#if (FUNCS_IN_C & DEC_KEYING_IN_C)
+#if KEY_SCHED == ONE_TABLE
+#define IM1_SET
+#elif KEY_SCHED == FOUR_TABLES
+#define IM4_SET
+#elif !defined(SBX_SET)
+#define SBX_SET
+#endif
+#endif
+#endif
+
+/* generic definitions of Rijndael macros that use tables */
+
+#define no_table(x, box, vf, rf, c) bytes2word(\
+ box[bval(vf(x, 0, c), rf(0, c))], \
+ box[bval(vf(x, 1, c), rf(1, c))], \
+ box[bval(vf(x, 2, c), rf(2, c))], \
+ box[bval(vf(x, 3, c), rf(3, c))])
+
+#define one_table(x, op, tab, vf, rf, c) \
+ (tab[bval(vf(x, 0, c), rf(0, c))] \
+ ^ op(tab[bval(vf(x, 1, c), rf(1, c))], 1) \
+ ^ op(tab[bval(vf(x, 2, c), rf(2, c))], 2) \
+ ^ op(tab[bval(vf(x, 3, c), rf(3, c))], 3))
+
+#define four_tables(x, tab, vf, rf, c) \
+ (tab[0][bval(vf(x, 0, c), rf(0, c))] \
+ ^ tab[1][bval(vf(x, 1, c), rf(1, c))] \
+ ^ tab[2][bval(vf(x, 2, c), rf(2, c))] \
+ ^ tab[3][bval(vf(x, 3, c), rf(3, c))])
+
+#define vf1(x, r, c) (x)
+#define rf1(r, c) (r)
+#define rf2(r, c) ((8+r-c)&3)
+
+/*
+ * Perform forward and inverse column mix operation on four bytes in long word
+ * x in parallel. NOTE: x must be a simple variable, NOT an expression in
+ * these macros.
+ */
+
+#if !(defined(REDUCE_CODE_SIZE) && (defined(ASM_X86_V2) || \
+ defined(ASM_X86_V2C)))
+
+#if defined(FM4_SET) /* not currently used */
+#define fwd_mcol(x) four_tables(x, t_use(f, m), vf1, rf1, 0)
+#elif defined(FM1_SET) /* not currently used */
+#define fwd_mcol(x) one_table(x, upr, t_use(f, m), vf1, rf1, 0)
+#else
+#define dec_fmvars uint32_t g2
+#define fwd_mcol(x) (g2 = gf_mulx(x), g2 ^ upr((x) ^ g2, 3) ^ \
+ upr((x), 2) ^ upr((x), 1))
+#endif
+
+#if defined(IM4_SET)
+#define inv_mcol(x) four_tables(x, t_use(i, m), vf1, rf1, 0)
+#elif defined(IM1_SET)
+#define inv_mcol(x) one_table(x, upr, t_use(i, m), vf1, rf1, 0)
+#else
+#define dec_imvars uint32_t g2, g4, g9
+#define inv_mcol(x) (g2 = gf_mulx(x), g4 = gf_mulx(g2), g9 = \
+ (x) ^ gf_mulx(g4), g4 ^= g9, \
+ (x) ^ g2 ^ g4 ^ upr(g2 ^ g9, 3) ^ \
+ upr(g4, 2) ^ upr(g9, 1))
+#endif
+
+#if defined(FL4_SET)
+#define ls_box(x, c) four_tables(x, t_use(f, l), vf1, rf2, c)
+#elif defined(LS4_SET)
+#define ls_box(x, c) four_tables(x, t_use(l, s), vf1, rf2, c)
+#elif defined(FL1_SET)
+#define ls_box(x, c) one_table(x, upr, t_use(f, l), vf1, rf2, c)
+#elif defined(LS1_SET)
+#define ls_box(x, c) one_table(x, upr, t_use(l, s), vf1, rf2, c)
+#else
+#define ls_box(x, c) no_table(x, t_use(s, box), vf1, rf2, c)
+#endif
+
+#endif
+
+#if defined(ASM_X86_V1C) && defined(AES_DECRYPT) && !defined(ISB_SET)
+#define ISB_SET
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _AESOPT_H */
--- /dev/null
+/*
+ * ---------------------------------------------------------------------------
+ * Copyright (c) 1998-2007, Brian Gladman, Worcester, UK. All rights reserved.
+ *
+ * LICENSE TERMS
+ *
+ * The free distribution and use of this software is allowed (with or without
+ * changes) provided that:
+ *
+ * 1. source code distributions include the above copyright notice, this
+ * list of conditions and the following disclaimer;
+ *
+ * 2. binary distributions include the above copyright notice, this list
+ * of conditions and the following disclaimer in their documentation;
+ *
+ * 3. the name of the copyright holder is not used to endorse products
+ * built using this software without specific written permission.
+ *
+ * DISCLAIMER
+ *
+ * This software is provided 'as is' with no explicit or implied warranties
+ * in respect of its properties, including, but not limited to, correctness
+ * and/or fitness for purpose.
+ * ---------------------------------------------------------------------------
+ * Issue Date: 20/12/2007
+ *
+ * This file contains the code for declaring the tables needed to implement
+ * AES. The file aesopt.h is assumed to be included before this header file.
+ * If there are no global variables, the definitions here can be used to put
+ * the AES tables in a structure so that a pointer can then be added to the
+ * AES context to pass them to the AES routines that need them. If this
+ * facility is used, the calling program has to ensure that this pointer is
+ * managed appropriately. In particular, the value of the t_dec(in, it) item
+ * in the table structure must be set to zero in order to ensure that the
+ * tables are initialised. In practice the three code sequences in aeskey.c
+ * that control the calls to aes_init() and the aes_init() routine itself will
+ * have to be changed for a specific implementation. If global variables are
+ * available it will generally be preferable to use them with the precomputed
+ * FIXED_TABLES option that uses static global tables.
+ *
+ * The following defines can be used to control the way the tables
+ * are defined, initialised and used in embedded environments that
+ * require special features for these purposes
+ *
+ * the 't_dec' construction is used to declare fixed table arrays
+ * the 't_set' construction is used to set fixed table values
+ * the 't_use' construction is used to access fixed table values
+ *
+ * 256 byte tables:
+ *
+ * t_xxx(s, box) => forward S box
+ * t_xxx(i, box) => inverse S box
+ *
+ * 256 32-bit word OR 4 x 256 32-bit word tables:
+ *
+ * t_xxx(f, n) => forward normal round
+ * t_xxx(f, l) => forward last round
+ * t_xxx(i, n) => inverse normal round
+ * t_xxx(i, l) => inverse last round
+ * t_xxx(l, s) => key schedule table
+ * t_xxx(i, m) => key schedule table
+ *
+ * Other variables and tables:
+ *
+ * t_xxx(r, c) => the rcon table
+ */
+
+/*
+ * OpenSolaris OS modifications
+ *
+ * 1. Added __cplusplus and _AESTAB_H header guards
+ * 2. Added header file sys/types.h
+ * 3. Remove code defined for _MSC_VER
+ * 4. Changed all variables to "static const"
+ * 5. Changed uint_8t and uint_32t to uint8_t and uint32_t
+ * 6. Cstyled and hdrchk code
+ */
+
+#ifndef _AESTAB_H
+#define _AESTAB_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/types.h>
+
+#define t_dec(m, n) t_##m##n
+#define t_set(m, n) t_##m##n
+#define t_use(m, n) t_##m##n
+
+#if defined(DO_TABLES) && defined(FIXED_TABLES)
+#define d_1(t, n, b, e) static const t n[256] = b(e)
+#define d_4(t, n, b, e, f, g, h) static const t n[4][256] = \
+ {b(e), b(f), b(g), b(h)}
+static const uint32_t t_dec(r, c)[RC_LENGTH] = rc_data(w0);
+#else
+#define d_1(t, n, b, e) static const t n[256]
+#define d_4(t, n, b, e, f, g, h) static const t n[4][256]
+static const uint32_t t_dec(r, c)[RC_LENGTH];
+#endif
+
+#if defined(SBX_SET)
+ d_1(uint8_t, t_dec(s, box), sb_data, h0);
+#endif
+#if defined(ISB_SET)
+ d_1(uint8_t, t_dec(i, box), isb_data, h0);
+#endif
+
+#if defined(FT1_SET)
+ d_1(uint32_t, t_dec(f, n), sb_data, u0);
+#endif
+#if defined(FT4_SET)
+ d_4(uint32_t, t_dec(f, n), sb_data, u0, u1, u2, u3);
+#endif
+
+#if defined(FL1_SET)
+ d_1(uint32_t, t_dec(f, l), sb_data, w0);
+#endif
+#if defined(FL4_SET)
+ d_4(uint32_t, t_dec(f, l), sb_data, w0, w1, w2, w3);
+#endif
+
+#if defined(IT1_SET)
+ d_1(uint32_t, t_dec(i, n), isb_data, v0);
+#endif
+#if defined(IT4_SET)
+ d_4(uint32_t, t_dec(i, n), isb_data, v0, v1, v2, v3);
+#endif
+
+#if defined(IL1_SET)
+ d_1(uint32_t, t_dec(i, l), isb_data, w0);
+#endif
+#if defined(IL4_SET)
+ d_4(uint32_t, t_dec(i, l), isb_data, w0, w1, w2, w3);
+#endif
+
+#if defined(LS1_SET)
+#if defined(FL1_SET)
+#undef LS1_SET
+#else
+ d_1(uint32_t, t_dec(l, s), sb_data, w0);
+#endif
+#endif
+
+#if defined(LS4_SET)
+#if defined(FL4_SET)
+#undef LS4_SET
+#else
+ d_4(uint32_t, t_dec(l, s), sb_data, w0, w1, w2, w3);
+#endif
+#endif
+
+#if defined(IM1_SET)
+ d_1(uint32_t, t_dec(i, m), mm_data, v0);
+#endif
+#if defined(IM4_SET)
+ d_4(uint32_t, t_dec(i, m), mm_data, v0, v1, v2, v3);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _AESTAB_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _AESTAB2_H
+#define _AESTAB2_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * To create this file for OpenSolaris:
+ * 1. Compile and run tablegen.c, from aes-src-04-03-08.zip,
+ * after defining ASM_AMD64_C
+ * 2. mv aestab2.c aestab2.h
+ * 3. Add __cplusplus and _AESTAB2_H header guards
+ * 3. Add #include <aes_impl.h>
+ * 4. Change "uint_32t" to "uint32_t"
+ * 5. Change all variables to "static const"
+ * 6. Cstyle and hdrchk this file
+ */
+
+#include <aes/aes_impl.h>
+
+static const uint32_t t_rc[RC_LENGTH] =
+{
+ 0x00000001, 0x00000002, 0x00000004, 0x00000008,
+ 0x00000010, 0x00000020, 0x00000040, 0x00000080,
+ 0x0000001b, 0x00000036
+};
+
+static const uint32_t t_ls[4][256] =
+{
+ {
+ 0x00000063, 0x0000007c, 0x00000077, 0x0000007b,
+ 0x000000f2, 0x0000006b, 0x0000006f, 0x000000c5,
+ 0x00000030, 0x00000001, 0x00000067, 0x0000002b,
+ 0x000000fe, 0x000000d7, 0x000000ab, 0x00000076,
+ 0x000000ca, 0x00000082, 0x000000c9, 0x0000007d,
+ 0x000000fa, 0x00000059, 0x00000047, 0x000000f0,
+ 0x000000ad, 0x000000d4, 0x000000a2, 0x000000af,
+ 0x0000009c, 0x000000a4, 0x00000072, 0x000000c0,
+ 0x000000b7, 0x000000fd, 0x00000093, 0x00000026,
+ 0x00000036, 0x0000003f, 0x000000f7, 0x000000cc,
+ 0x00000034, 0x000000a5, 0x000000e5, 0x000000f1,
+ 0x00000071, 0x000000d8, 0x00000031, 0x00000015,
+ 0x00000004, 0x000000c7, 0x00000023, 0x000000c3,
+ 0x00000018, 0x00000096, 0x00000005, 0x0000009a,
+ 0x00000007, 0x00000012, 0x00000080, 0x000000e2,
+ 0x000000eb, 0x00000027, 0x000000b2, 0x00000075,
+ 0x00000009, 0x00000083, 0x0000002c, 0x0000001a,
+ 0x0000001b, 0x0000006e, 0x0000005a, 0x000000a0,
+ 0x00000052, 0x0000003b, 0x000000d6, 0x000000b3,
+ 0x00000029, 0x000000e3, 0x0000002f, 0x00000084,
+ 0x00000053, 0x000000d1, 0x00000000, 0x000000ed,
+ 0x00000020, 0x000000fc, 0x000000b1, 0x0000005b,
+ 0x0000006a, 0x000000cb, 0x000000be, 0x00000039,
+ 0x0000004a, 0x0000004c, 0x00000058, 0x000000cf,
+ 0x000000d0, 0x000000ef, 0x000000aa, 0x000000fb,
+ 0x00000043, 0x0000004d, 0x00000033, 0x00000085,
+ 0x00000045, 0x000000f9, 0x00000002, 0x0000007f,
+ 0x00000050, 0x0000003c, 0x0000009f, 0x000000a8,
+ 0x00000051, 0x000000a3, 0x00000040, 0x0000008f,
+ 0x00000092, 0x0000009d, 0x00000038, 0x000000f5,
+ 0x000000bc, 0x000000b6, 0x000000da, 0x00000021,
+ 0x00000010, 0x000000ff, 0x000000f3, 0x000000d2,
+ 0x000000cd, 0x0000000c, 0x00000013, 0x000000ec,
+ 0x0000005f, 0x00000097, 0x00000044, 0x00000017,
+ 0x000000c4, 0x000000a7, 0x0000007e, 0x0000003d,
+ 0x00000064, 0x0000005d, 0x00000019, 0x00000073,
+ 0x00000060, 0x00000081, 0x0000004f, 0x000000dc,
+ 0x00000022, 0x0000002a, 0x00000090, 0x00000088,
+ 0x00000046, 0x000000ee, 0x000000b8, 0x00000014,
+ 0x000000de, 0x0000005e, 0x0000000b, 0x000000db,
+ 0x000000e0, 0x00000032, 0x0000003a, 0x0000000a,
+ 0x00000049, 0x00000006, 0x00000024, 0x0000005c,
+ 0x000000c2, 0x000000d3, 0x000000ac, 0x00000062,
+ 0x00000091, 0x00000095, 0x000000e4, 0x00000079,
+ 0x000000e7, 0x000000c8, 0x00000037, 0x0000006d,
+ 0x0000008d, 0x000000d5, 0x0000004e, 0x000000a9,
+ 0x0000006c, 0x00000056, 0x000000f4, 0x000000ea,
+ 0x00000065, 0x0000007a, 0x000000ae, 0x00000008,
+ 0x000000ba, 0x00000078, 0x00000025, 0x0000002e,
+ 0x0000001c, 0x000000a6, 0x000000b4, 0x000000c6,
+ 0x000000e8, 0x000000dd, 0x00000074, 0x0000001f,
+ 0x0000004b, 0x000000bd, 0x0000008b, 0x0000008a,
+ 0x00000070, 0x0000003e, 0x000000b5, 0x00000066,
+ 0x00000048, 0x00000003, 0x000000f6, 0x0000000e,
+ 0x00000061, 0x00000035, 0x00000057, 0x000000b9,
+ 0x00000086, 0x000000c1, 0x0000001d, 0x0000009e,
+ 0x000000e1, 0x000000f8, 0x00000098, 0x00000011,
+ 0x00000069, 0x000000d9, 0x0000008e, 0x00000094,
+ 0x0000009b, 0x0000001e, 0x00000087, 0x000000e9,
+ 0x000000ce, 0x00000055, 0x00000028, 0x000000df,
+ 0x0000008c, 0x000000a1, 0x00000089, 0x0000000d,
+ 0x000000bf, 0x000000e6, 0x00000042, 0x00000068,
+ 0x00000041, 0x00000099, 0x0000002d, 0x0000000f,
+ 0x000000b0, 0x00000054, 0x000000bb, 0x00000016
+ },
+ {
+ 0x00006300, 0x00007c00, 0x00007700, 0x00007b00,
+ 0x0000f200, 0x00006b00, 0x00006f00, 0x0000c500,
+ 0x00003000, 0x00000100, 0x00006700, 0x00002b00,
+ 0x0000fe00, 0x0000d700, 0x0000ab00, 0x00007600,
+ 0x0000ca00, 0x00008200, 0x0000c900, 0x00007d00,
+ 0x0000fa00, 0x00005900, 0x00004700, 0x0000f000,
+ 0x0000ad00, 0x0000d400, 0x0000a200, 0x0000af00,
+ 0x00009c00, 0x0000a400, 0x00007200, 0x0000c000,
+ 0x0000b700, 0x0000fd00, 0x00009300, 0x00002600,
+ 0x00003600, 0x00003f00, 0x0000f700, 0x0000cc00,
+ 0x00003400, 0x0000a500, 0x0000e500, 0x0000f100,
+ 0x00007100, 0x0000d800, 0x00003100, 0x00001500,
+ 0x00000400, 0x0000c700, 0x00002300, 0x0000c300,
+ 0x00001800, 0x00009600, 0x00000500, 0x00009a00,
+ 0x00000700, 0x00001200, 0x00008000, 0x0000e200,
+ 0x0000eb00, 0x00002700, 0x0000b200, 0x00007500,
+ 0x00000900, 0x00008300, 0x00002c00, 0x00001a00,
+ 0x00001b00, 0x00006e00, 0x00005a00, 0x0000a000,
+ 0x00005200, 0x00003b00, 0x0000d600, 0x0000b300,
+ 0x00002900, 0x0000e300, 0x00002f00, 0x00008400,
+ 0x00005300, 0x0000d100, 0x00000000, 0x0000ed00,
+ 0x00002000, 0x0000fc00, 0x0000b100, 0x00005b00,
+ 0x00006a00, 0x0000cb00, 0x0000be00, 0x00003900,
+ 0x00004a00, 0x00004c00, 0x00005800, 0x0000cf00,
+ 0x0000d000, 0x0000ef00, 0x0000aa00, 0x0000fb00,
+ 0x00004300, 0x00004d00, 0x00003300, 0x00008500,
+ 0x00004500, 0x0000f900, 0x00000200, 0x00007f00,
+ 0x00005000, 0x00003c00, 0x00009f00, 0x0000a800,
+ 0x00005100, 0x0000a300, 0x00004000, 0x00008f00,
+ 0x00009200, 0x00009d00, 0x00003800, 0x0000f500,
+ 0x0000bc00, 0x0000b600, 0x0000da00, 0x00002100,
+ 0x00001000, 0x0000ff00, 0x0000f300, 0x0000d200,
+ 0x0000cd00, 0x00000c00, 0x00001300, 0x0000ec00,
+ 0x00005f00, 0x00009700, 0x00004400, 0x00001700,
+ 0x0000c400, 0x0000a700, 0x00007e00, 0x00003d00,
+ 0x00006400, 0x00005d00, 0x00001900, 0x00007300,
+ 0x00006000, 0x00008100, 0x00004f00, 0x0000dc00,
+ 0x00002200, 0x00002a00, 0x00009000, 0x00008800,
+ 0x00004600, 0x0000ee00, 0x0000b800, 0x00001400,
+ 0x0000de00, 0x00005e00, 0x00000b00, 0x0000db00,
+ 0x0000e000, 0x00003200, 0x00003a00, 0x00000a00,
+ 0x00004900, 0x00000600, 0x00002400, 0x00005c00,
+ 0x0000c200, 0x0000d300, 0x0000ac00, 0x00006200,
+ 0x00009100, 0x00009500, 0x0000e400, 0x00007900,
+ 0x0000e700, 0x0000c800, 0x00003700, 0x00006d00,
+ 0x00008d00, 0x0000d500, 0x00004e00, 0x0000a900,
+ 0x00006c00, 0x00005600, 0x0000f400, 0x0000ea00,
+ 0x00006500, 0x00007a00, 0x0000ae00, 0x00000800,
+ 0x0000ba00, 0x00007800, 0x00002500, 0x00002e00,
+ 0x00001c00, 0x0000a600, 0x0000b400, 0x0000c600,
+ 0x0000e800, 0x0000dd00, 0x00007400, 0x00001f00,
+ 0x00004b00, 0x0000bd00, 0x00008b00, 0x00008a00,
+ 0x00007000, 0x00003e00, 0x0000b500, 0x00006600,
+ 0x00004800, 0x00000300, 0x0000f600, 0x00000e00,
+ 0x00006100, 0x00003500, 0x00005700, 0x0000b900,
+ 0x00008600, 0x0000c100, 0x00001d00, 0x00009e00,
+ 0x0000e100, 0x0000f800, 0x00009800, 0x00001100,
+ 0x00006900, 0x0000d900, 0x00008e00, 0x00009400,
+ 0x00009b00, 0x00001e00, 0x00008700, 0x0000e900,
+ 0x0000ce00, 0x00005500, 0x00002800, 0x0000df00,
+ 0x00008c00, 0x0000a100, 0x00008900, 0x00000d00,
+ 0x0000bf00, 0x0000e600, 0x00004200, 0x00006800,
+ 0x00004100, 0x00009900, 0x00002d00, 0x00000f00,
+ 0x0000b000, 0x00005400, 0x0000bb00, 0x00001600
+ },
+ {
+ 0x00630000, 0x007c0000, 0x00770000, 0x007b0000,
+ 0x00f20000, 0x006b0000, 0x006f0000, 0x00c50000,
+ 0x00300000, 0x00010000, 0x00670000, 0x002b0000,
+ 0x00fe0000, 0x00d70000, 0x00ab0000, 0x00760000,
+ 0x00ca0000, 0x00820000, 0x00c90000, 0x007d0000,
+ 0x00fa0000, 0x00590000, 0x00470000, 0x00f00000,
+ 0x00ad0000, 0x00d40000, 0x00a20000, 0x00af0000,
+ 0x009c0000, 0x00a40000, 0x00720000, 0x00c00000,
+ 0x00b70000, 0x00fd0000, 0x00930000, 0x00260000,
+ 0x00360000, 0x003f0000, 0x00f70000, 0x00cc0000,
+ 0x00340000, 0x00a50000, 0x00e50000, 0x00f10000,
+ 0x00710000, 0x00d80000, 0x00310000, 0x00150000,
+ 0x00040000, 0x00c70000, 0x00230000, 0x00c30000,
+ 0x00180000, 0x00960000, 0x00050000, 0x009a0000,
+ 0x00070000, 0x00120000, 0x00800000, 0x00e20000,
+ 0x00eb0000, 0x00270000, 0x00b20000, 0x00750000,
+ 0x00090000, 0x00830000, 0x002c0000, 0x001a0000,
+ 0x001b0000, 0x006e0000, 0x005a0000, 0x00a00000,
+ 0x00520000, 0x003b0000, 0x00d60000, 0x00b30000,
+ 0x00290000, 0x00e30000, 0x002f0000, 0x00840000,
+ 0x00530000, 0x00d10000, 0x00000000, 0x00ed0000,
+ 0x00200000, 0x00fc0000, 0x00b10000, 0x005b0000,
+ 0x006a0000, 0x00cb0000, 0x00be0000, 0x00390000,
+ 0x004a0000, 0x004c0000, 0x00580000, 0x00cf0000,
+ 0x00d00000, 0x00ef0000, 0x00aa0000, 0x00fb0000,
+ 0x00430000, 0x004d0000, 0x00330000, 0x00850000,
+ 0x00450000, 0x00f90000, 0x00020000, 0x007f0000,
+ 0x00500000, 0x003c0000, 0x009f0000, 0x00a80000,
+ 0x00510000, 0x00a30000, 0x00400000, 0x008f0000,
+ 0x00920000, 0x009d0000, 0x00380000, 0x00f50000,
+ 0x00bc0000, 0x00b60000, 0x00da0000, 0x00210000,
+ 0x00100000, 0x00ff0000, 0x00f30000, 0x00d20000,
+ 0x00cd0000, 0x000c0000, 0x00130000, 0x00ec0000,
+ 0x005f0000, 0x00970000, 0x00440000, 0x00170000,
+ 0x00c40000, 0x00a70000, 0x007e0000, 0x003d0000,
+ 0x00640000, 0x005d0000, 0x00190000, 0x00730000,
+ 0x00600000, 0x00810000, 0x004f0000, 0x00dc0000,
+ 0x00220000, 0x002a0000, 0x00900000, 0x00880000,
+ 0x00460000, 0x00ee0000, 0x00b80000, 0x00140000,
+ 0x00de0000, 0x005e0000, 0x000b0000, 0x00db0000,
+ 0x00e00000, 0x00320000, 0x003a0000, 0x000a0000,
+ 0x00490000, 0x00060000, 0x00240000, 0x005c0000,
+ 0x00c20000, 0x00d30000, 0x00ac0000, 0x00620000,
+ 0x00910000, 0x00950000, 0x00e40000, 0x00790000,
+ 0x00e70000, 0x00c80000, 0x00370000, 0x006d0000,
+ 0x008d0000, 0x00d50000, 0x004e0000, 0x00a90000,
+ 0x006c0000, 0x00560000, 0x00f40000, 0x00ea0000,
+ 0x00650000, 0x007a0000, 0x00ae0000, 0x00080000,
+ 0x00ba0000, 0x00780000, 0x00250000, 0x002e0000,
+ 0x001c0000, 0x00a60000, 0x00b40000, 0x00c60000,
+ 0x00e80000, 0x00dd0000, 0x00740000, 0x001f0000,
+ 0x004b0000, 0x00bd0000, 0x008b0000, 0x008a0000,
+ 0x00700000, 0x003e0000, 0x00b50000, 0x00660000,
+ 0x00480000, 0x00030000, 0x00f60000, 0x000e0000,
+ 0x00610000, 0x00350000, 0x00570000, 0x00b90000,
+ 0x00860000, 0x00c10000, 0x001d0000, 0x009e0000,
+ 0x00e10000, 0x00f80000, 0x00980000, 0x00110000,
+ 0x00690000, 0x00d90000, 0x008e0000, 0x00940000,
+ 0x009b0000, 0x001e0000, 0x00870000, 0x00e90000,
+ 0x00ce0000, 0x00550000, 0x00280000, 0x00df0000,
+ 0x008c0000, 0x00a10000, 0x00890000, 0x000d0000,
+ 0x00bf0000, 0x00e60000, 0x00420000, 0x00680000,
+ 0x00410000, 0x00990000, 0x002d0000, 0x000f0000,
+ 0x00b00000, 0x00540000, 0x00bb0000, 0x00160000
+ },
+ {
+ 0x63000000, 0x7c000000, 0x77000000, 0x7b000000,
+ 0xf2000000, 0x6b000000, 0x6f000000, 0xc5000000,
+ 0x30000000, 0x01000000, 0x67000000, 0x2b000000,
+ 0xfe000000, 0xd7000000, 0xab000000, 0x76000000,
+ 0xca000000, 0x82000000, 0xc9000000, 0x7d000000,
+ 0xfa000000, 0x59000000, 0x47000000, 0xf0000000,
+ 0xad000000, 0xd4000000, 0xa2000000, 0xaf000000,
+ 0x9c000000, 0xa4000000, 0x72000000, 0xc0000000,
+ 0xb7000000, 0xfd000000, 0x93000000, 0x26000000,
+ 0x36000000, 0x3f000000, 0xf7000000, 0xcc000000,
+ 0x34000000, 0xa5000000, 0xe5000000, 0xf1000000,
+ 0x71000000, 0xd8000000, 0x31000000, 0x15000000,
+ 0x04000000, 0xc7000000, 0x23000000, 0xc3000000,
+ 0x18000000, 0x96000000, 0x05000000, 0x9a000000,
+ 0x07000000, 0x12000000, 0x80000000, 0xe2000000,
+ 0xeb000000, 0x27000000, 0xb2000000, 0x75000000,
+ 0x09000000, 0x83000000, 0x2c000000, 0x1a000000,
+ 0x1b000000, 0x6e000000, 0x5a000000, 0xa0000000,
+ 0x52000000, 0x3b000000, 0xd6000000, 0xb3000000,
+ 0x29000000, 0xe3000000, 0x2f000000, 0x84000000,
+ 0x53000000, 0xd1000000, 0x00000000, 0xed000000,
+ 0x20000000, 0xfc000000, 0xb1000000, 0x5b000000,
+ 0x6a000000, 0xcb000000, 0xbe000000, 0x39000000,
+ 0x4a000000, 0x4c000000, 0x58000000, 0xcf000000,
+ 0xd0000000, 0xef000000, 0xaa000000, 0xfb000000,
+ 0x43000000, 0x4d000000, 0x33000000, 0x85000000,
+ 0x45000000, 0xf9000000, 0x02000000, 0x7f000000,
+ 0x50000000, 0x3c000000, 0x9f000000, 0xa8000000,
+ 0x51000000, 0xa3000000, 0x40000000, 0x8f000000,
+ 0x92000000, 0x9d000000, 0x38000000, 0xf5000000,
+ 0xbc000000, 0xb6000000, 0xda000000, 0x21000000,
+ 0x10000000, 0xff000000, 0xf3000000, 0xd2000000,
+ 0xcd000000, 0x0c000000, 0x13000000, 0xec000000,
+ 0x5f000000, 0x97000000, 0x44000000, 0x17000000,
+ 0xc4000000, 0xa7000000, 0x7e000000, 0x3d000000,
+ 0x64000000, 0x5d000000, 0x19000000, 0x73000000,
+ 0x60000000, 0x81000000, 0x4f000000, 0xdc000000,
+ 0x22000000, 0x2a000000, 0x90000000, 0x88000000,
+ 0x46000000, 0xee000000, 0xb8000000, 0x14000000,
+ 0xde000000, 0x5e000000, 0x0b000000, 0xdb000000,
+ 0xe0000000, 0x32000000, 0x3a000000, 0x0a000000,
+ 0x49000000, 0x06000000, 0x24000000, 0x5c000000,
+ 0xc2000000, 0xd3000000, 0xac000000, 0x62000000,
+ 0x91000000, 0x95000000, 0xe4000000, 0x79000000,
+ 0xe7000000, 0xc8000000, 0x37000000, 0x6d000000,
+ 0x8d000000, 0xd5000000, 0x4e000000, 0xa9000000,
+ 0x6c000000, 0x56000000, 0xf4000000, 0xea000000,
+ 0x65000000, 0x7a000000, 0xae000000, 0x08000000,
+ 0xba000000, 0x78000000, 0x25000000, 0x2e000000,
+ 0x1c000000, 0xa6000000, 0xb4000000, 0xc6000000,
+ 0xe8000000, 0xdd000000, 0x74000000, 0x1f000000,
+ 0x4b000000, 0xbd000000, 0x8b000000, 0x8a000000,
+ 0x70000000, 0x3e000000, 0xb5000000, 0x66000000,
+ 0x48000000, 0x03000000, 0xf6000000, 0x0e000000,
+ 0x61000000, 0x35000000, 0x57000000, 0xb9000000,
+ 0x86000000, 0xc1000000, 0x1d000000, 0x9e000000,
+ 0xe1000000, 0xf8000000, 0x98000000, 0x11000000,
+ 0x69000000, 0xd9000000, 0x8e000000, 0x94000000,
+ 0x9b000000, 0x1e000000, 0x87000000, 0xe9000000,
+ 0xce000000, 0x55000000, 0x28000000, 0xdf000000,
+ 0x8c000000, 0xa1000000, 0x89000000, 0x0d000000,
+ 0xbf000000, 0xe6000000, 0x42000000, 0x68000000,
+ 0x41000000, 0x99000000, 0x2d000000, 0x0f000000,
+ 0xb0000000, 0x54000000, 0xbb000000, 0x16000000
+ }
+};
+
+static const uint32_t t_im[4][256] =
+{
+ {
+ 0x00000000, 0x0b0d090e, 0x161a121c, 0x1d171b12,
+ 0x2c342438, 0x27392d36, 0x3a2e3624, 0x31233f2a,
+ 0x58684870, 0x5365417e, 0x4e725a6c, 0x457f5362,
+ 0x745c6c48, 0x7f516546, 0x62467e54, 0x694b775a,
+ 0xb0d090e0, 0xbbdd99ee, 0xa6ca82fc, 0xadc78bf2,
+ 0x9ce4b4d8, 0x97e9bdd6, 0x8afea6c4, 0x81f3afca,
+ 0xe8b8d890, 0xe3b5d19e, 0xfea2ca8c, 0xf5afc382,
+ 0xc48cfca8, 0xcf81f5a6, 0xd296eeb4, 0xd99be7ba,
+ 0x7bbb3bdb, 0x70b632d5, 0x6da129c7, 0x66ac20c9,
+ 0x578f1fe3, 0x5c8216ed, 0x41950dff, 0x4a9804f1,
+ 0x23d373ab, 0x28de7aa5, 0x35c961b7, 0x3ec468b9,
+ 0x0fe75793, 0x04ea5e9d, 0x19fd458f, 0x12f04c81,
+ 0xcb6bab3b, 0xc066a235, 0xdd71b927, 0xd67cb029,
+ 0xe75f8f03, 0xec52860d, 0xf1459d1f, 0xfa489411,
+ 0x9303e34b, 0x980eea45, 0x8519f157, 0x8e14f859,
+ 0xbf37c773, 0xb43ace7d, 0xa92dd56f, 0xa220dc61,
+ 0xf66d76ad, 0xfd607fa3, 0xe07764b1, 0xeb7a6dbf,
+ 0xda595295, 0xd1545b9b, 0xcc434089, 0xc74e4987,
+ 0xae053edd, 0xa50837d3, 0xb81f2cc1, 0xb31225cf,
+ 0x82311ae5, 0x893c13eb, 0x942b08f9, 0x9f2601f7,
+ 0x46bde64d, 0x4db0ef43, 0x50a7f451, 0x5baafd5f,
+ 0x6a89c275, 0x6184cb7b, 0x7c93d069, 0x779ed967,
+ 0x1ed5ae3d, 0x15d8a733, 0x08cfbc21, 0x03c2b52f,
+ 0x32e18a05, 0x39ec830b, 0x24fb9819, 0x2ff69117,
+ 0x8dd64d76, 0x86db4478, 0x9bcc5f6a, 0x90c15664,
+ 0xa1e2694e, 0xaaef6040, 0xb7f87b52, 0xbcf5725c,
+ 0xd5be0506, 0xdeb30c08, 0xc3a4171a, 0xc8a91e14,
+ 0xf98a213e, 0xf2872830, 0xef903322, 0xe49d3a2c,
+ 0x3d06dd96, 0x360bd498, 0x2b1ccf8a, 0x2011c684,
+ 0x1132f9ae, 0x1a3ff0a0, 0x0728ebb2, 0x0c25e2bc,
+ 0x656e95e6, 0x6e639ce8, 0x737487fa, 0x78798ef4,
+ 0x495ab1de, 0x4257b8d0, 0x5f40a3c2, 0x544daacc,
+ 0xf7daec41, 0xfcd7e54f, 0xe1c0fe5d, 0xeacdf753,
+ 0xdbeec879, 0xd0e3c177, 0xcdf4da65, 0xc6f9d36b,
+ 0xafb2a431, 0xa4bfad3f, 0xb9a8b62d, 0xb2a5bf23,
+ 0x83868009, 0x888b8907, 0x959c9215, 0x9e919b1b,
+ 0x470a7ca1, 0x4c0775af, 0x51106ebd, 0x5a1d67b3,
+ 0x6b3e5899, 0x60335197, 0x7d244a85, 0x7629438b,
+ 0x1f6234d1, 0x146f3ddf, 0x097826cd, 0x02752fc3,
+ 0x335610e9, 0x385b19e7, 0x254c02f5, 0x2e410bfb,
+ 0x8c61d79a, 0x876cde94, 0x9a7bc586, 0x9176cc88,
+ 0xa055f3a2, 0xab58faac, 0xb64fe1be, 0xbd42e8b0,
+ 0xd4099fea, 0xdf0496e4, 0xc2138df6, 0xc91e84f8,
+ 0xf83dbbd2, 0xf330b2dc, 0xee27a9ce, 0xe52aa0c0,
+ 0x3cb1477a, 0x37bc4e74, 0x2aab5566, 0x21a65c68,
+ 0x10856342, 0x1b886a4c, 0x069f715e, 0x0d927850,
+ 0x64d90f0a, 0x6fd40604, 0x72c31d16, 0x79ce1418,
+ 0x48ed2b32, 0x43e0223c, 0x5ef7392e, 0x55fa3020,
+ 0x01b79aec, 0x0aba93e2, 0x17ad88f0, 0x1ca081fe,
+ 0x2d83bed4, 0x268eb7da, 0x3b99acc8, 0x3094a5c6,
+ 0x59dfd29c, 0x52d2db92, 0x4fc5c080, 0x44c8c98e,
+ 0x75ebf6a4, 0x7ee6ffaa, 0x63f1e4b8, 0x68fcedb6,
+ 0xb1670a0c, 0xba6a0302, 0xa77d1810, 0xac70111e,
+ 0x9d532e34, 0x965e273a, 0x8b493c28, 0x80443526,
+ 0xe90f427c, 0xe2024b72, 0xff155060, 0xf418596e,
+ 0xc53b6644, 0xce366f4a, 0xd3217458, 0xd82c7d56,
+ 0x7a0ca137, 0x7101a839, 0x6c16b32b, 0x671bba25,
+ 0x5638850f, 0x5d358c01, 0x40229713, 0x4b2f9e1d,
+ 0x2264e947, 0x2969e049, 0x347efb5b, 0x3f73f255,
+ 0x0e50cd7f, 0x055dc471, 0x184adf63, 0x1347d66d,
+ 0xcadc31d7, 0xc1d138d9, 0xdcc623cb, 0xd7cb2ac5,
+ 0xe6e815ef, 0xede51ce1, 0xf0f207f3, 0xfbff0efd,
+ 0x92b479a7, 0x99b970a9, 0x84ae6bbb, 0x8fa362b5,
+ 0xbe805d9f, 0xb58d5491, 0xa89a4f83, 0xa397468d
+ },
+ {
+ 0x00000000, 0x0d090e0b, 0x1a121c16, 0x171b121d,
+ 0x3424382c, 0x392d3627, 0x2e36243a, 0x233f2a31,
+ 0x68487058, 0x65417e53, 0x725a6c4e, 0x7f536245,
+ 0x5c6c4874, 0x5165467f, 0x467e5462, 0x4b775a69,
+ 0xd090e0b0, 0xdd99eebb, 0xca82fca6, 0xc78bf2ad,
+ 0xe4b4d89c, 0xe9bdd697, 0xfea6c48a, 0xf3afca81,
+ 0xb8d890e8, 0xb5d19ee3, 0xa2ca8cfe, 0xafc382f5,
+ 0x8cfca8c4, 0x81f5a6cf, 0x96eeb4d2, 0x9be7bad9,
+ 0xbb3bdb7b, 0xb632d570, 0xa129c76d, 0xac20c966,
+ 0x8f1fe357, 0x8216ed5c, 0x950dff41, 0x9804f14a,
+ 0xd373ab23, 0xde7aa528, 0xc961b735, 0xc468b93e,
+ 0xe757930f, 0xea5e9d04, 0xfd458f19, 0xf04c8112,
+ 0x6bab3bcb, 0x66a235c0, 0x71b927dd, 0x7cb029d6,
+ 0x5f8f03e7, 0x52860dec, 0x459d1ff1, 0x489411fa,
+ 0x03e34b93, 0x0eea4598, 0x19f15785, 0x14f8598e,
+ 0x37c773bf, 0x3ace7db4, 0x2dd56fa9, 0x20dc61a2,
+ 0x6d76adf6, 0x607fa3fd, 0x7764b1e0, 0x7a6dbfeb,
+ 0x595295da, 0x545b9bd1, 0x434089cc, 0x4e4987c7,
+ 0x053eddae, 0x0837d3a5, 0x1f2cc1b8, 0x1225cfb3,
+ 0x311ae582, 0x3c13eb89, 0x2b08f994, 0x2601f79f,
+ 0xbde64d46, 0xb0ef434d, 0xa7f45150, 0xaafd5f5b,
+ 0x89c2756a, 0x84cb7b61, 0x93d0697c, 0x9ed96777,
+ 0xd5ae3d1e, 0xd8a73315, 0xcfbc2108, 0xc2b52f03,
+ 0xe18a0532, 0xec830b39, 0xfb981924, 0xf691172f,
+ 0xd64d768d, 0xdb447886, 0xcc5f6a9b, 0xc1566490,
+ 0xe2694ea1, 0xef6040aa, 0xf87b52b7, 0xf5725cbc,
+ 0xbe0506d5, 0xb30c08de, 0xa4171ac3, 0xa91e14c8,
+ 0x8a213ef9, 0x872830f2, 0x903322ef, 0x9d3a2ce4,
+ 0x06dd963d, 0x0bd49836, 0x1ccf8a2b, 0x11c68420,
+ 0x32f9ae11, 0x3ff0a01a, 0x28ebb207, 0x25e2bc0c,
+ 0x6e95e665, 0x639ce86e, 0x7487fa73, 0x798ef478,
+ 0x5ab1de49, 0x57b8d042, 0x40a3c25f, 0x4daacc54,
+ 0xdaec41f7, 0xd7e54ffc, 0xc0fe5de1, 0xcdf753ea,
+ 0xeec879db, 0xe3c177d0, 0xf4da65cd, 0xf9d36bc6,
+ 0xb2a431af, 0xbfad3fa4, 0xa8b62db9, 0xa5bf23b2,
+ 0x86800983, 0x8b890788, 0x9c921595, 0x919b1b9e,
+ 0x0a7ca147, 0x0775af4c, 0x106ebd51, 0x1d67b35a,
+ 0x3e58996b, 0x33519760, 0x244a857d, 0x29438b76,
+ 0x6234d11f, 0x6f3ddf14, 0x7826cd09, 0x752fc302,
+ 0x5610e933, 0x5b19e738, 0x4c02f525, 0x410bfb2e,
+ 0x61d79a8c, 0x6cde9487, 0x7bc5869a, 0x76cc8891,
+ 0x55f3a2a0, 0x58faacab, 0x4fe1beb6, 0x42e8b0bd,
+ 0x099fead4, 0x0496e4df, 0x138df6c2, 0x1e84f8c9,
+ 0x3dbbd2f8, 0x30b2dcf3, 0x27a9ceee, 0x2aa0c0e5,
+ 0xb1477a3c, 0xbc4e7437, 0xab55662a, 0xa65c6821,
+ 0x85634210, 0x886a4c1b, 0x9f715e06, 0x9278500d,
+ 0xd90f0a64, 0xd406046f, 0xc31d1672, 0xce141879,
+ 0xed2b3248, 0xe0223c43, 0xf7392e5e, 0xfa302055,
+ 0xb79aec01, 0xba93e20a, 0xad88f017, 0xa081fe1c,
+ 0x83bed42d, 0x8eb7da26, 0x99acc83b, 0x94a5c630,
+ 0xdfd29c59, 0xd2db9252, 0xc5c0804f, 0xc8c98e44,
+ 0xebf6a475, 0xe6ffaa7e, 0xf1e4b863, 0xfcedb668,
+ 0x670a0cb1, 0x6a0302ba, 0x7d1810a7, 0x70111eac,
+ 0x532e349d, 0x5e273a96, 0x493c288b, 0x44352680,
+ 0x0f427ce9, 0x024b72e2, 0x155060ff, 0x18596ef4,
+ 0x3b6644c5, 0x366f4ace, 0x217458d3, 0x2c7d56d8,
+ 0x0ca1377a, 0x01a83971, 0x16b32b6c, 0x1bba2567,
+ 0x38850f56, 0x358c015d, 0x22971340, 0x2f9e1d4b,
+ 0x64e94722, 0x69e04929, 0x7efb5b34, 0x73f2553f,
+ 0x50cd7f0e, 0x5dc47105, 0x4adf6318, 0x47d66d13,
+ 0xdc31d7ca, 0xd138d9c1, 0xc623cbdc, 0xcb2ac5d7,
+ 0xe815efe6, 0xe51ce1ed, 0xf207f3f0, 0xff0efdfb,
+ 0xb479a792, 0xb970a999, 0xae6bbb84, 0xa362b58f,
+ 0x805d9fbe, 0x8d5491b5, 0x9a4f83a8, 0x97468da3
+ },
+ {
+ 0x00000000, 0x090e0b0d, 0x121c161a, 0x1b121d17,
+ 0x24382c34, 0x2d362739, 0x36243a2e, 0x3f2a3123,
+ 0x48705868, 0x417e5365, 0x5a6c4e72, 0x5362457f,
+ 0x6c48745c, 0x65467f51, 0x7e546246, 0x775a694b,
+ 0x90e0b0d0, 0x99eebbdd, 0x82fca6ca, 0x8bf2adc7,
+ 0xb4d89ce4, 0xbdd697e9, 0xa6c48afe, 0xafca81f3,
+ 0xd890e8b8, 0xd19ee3b5, 0xca8cfea2, 0xc382f5af,
+ 0xfca8c48c, 0xf5a6cf81, 0xeeb4d296, 0xe7bad99b,
+ 0x3bdb7bbb, 0x32d570b6, 0x29c76da1, 0x20c966ac,
+ 0x1fe3578f, 0x16ed5c82, 0x0dff4195, 0x04f14a98,
+ 0x73ab23d3, 0x7aa528de, 0x61b735c9, 0x68b93ec4,
+ 0x57930fe7, 0x5e9d04ea, 0x458f19fd, 0x4c8112f0,
+ 0xab3bcb6b, 0xa235c066, 0xb927dd71, 0xb029d67c,
+ 0x8f03e75f, 0x860dec52, 0x9d1ff145, 0x9411fa48,
+ 0xe34b9303, 0xea45980e, 0xf1578519, 0xf8598e14,
+ 0xc773bf37, 0xce7db43a, 0xd56fa92d, 0xdc61a220,
+ 0x76adf66d, 0x7fa3fd60, 0x64b1e077, 0x6dbfeb7a,
+ 0x5295da59, 0x5b9bd154, 0x4089cc43, 0x4987c74e,
+ 0x3eddae05, 0x37d3a508, 0x2cc1b81f, 0x25cfb312,
+ 0x1ae58231, 0x13eb893c, 0x08f9942b, 0x01f79f26,
+ 0xe64d46bd, 0xef434db0, 0xf45150a7, 0xfd5f5baa,
+ 0xc2756a89, 0xcb7b6184, 0xd0697c93, 0xd967779e,
+ 0xae3d1ed5, 0xa73315d8, 0xbc2108cf, 0xb52f03c2,
+ 0x8a0532e1, 0x830b39ec, 0x981924fb, 0x91172ff6,
+ 0x4d768dd6, 0x447886db, 0x5f6a9bcc, 0x566490c1,
+ 0x694ea1e2, 0x6040aaef, 0x7b52b7f8, 0x725cbcf5,
+ 0x0506d5be, 0x0c08deb3, 0x171ac3a4, 0x1e14c8a9,
+ 0x213ef98a, 0x2830f287, 0x3322ef90, 0x3a2ce49d,
+ 0xdd963d06, 0xd498360b, 0xcf8a2b1c, 0xc6842011,
+ 0xf9ae1132, 0xf0a01a3f, 0xebb20728, 0xe2bc0c25,
+ 0x95e6656e, 0x9ce86e63, 0x87fa7374, 0x8ef47879,
+ 0xb1de495a, 0xb8d04257, 0xa3c25f40, 0xaacc544d,
+ 0xec41f7da, 0xe54ffcd7, 0xfe5de1c0, 0xf753eacd,
+ 0xc879dbee, 0xc177d0e3, 0xda65cdf4, 0xd36bc6f9,
+ 0xa431afb2, 0xad3fa4bf, 0xb62db9a8, 0xbf23b2a5,
+ 0x80098386, 0x8907888b, 0x9215959c, 0x9b1b9e91,
+ 0x7ca1470a, 0x75af4c07, 0x6ebd5110, 0x67b35a1d,
+ 0x58996b3e, 0x51976033, 0x4a857d24, 0x438b7629,
+ 0x34d11f62, 0x3ddf146f, 0x26cd0978, 0x2fc30275,
+ 0x10e93356, 0x19e7385b, 0x02f5254c, 0x0bfb2e41,
+ 0xd79a8c61, 0xde94876c, 0xc5869a7b, 0xcc889176,
+ 0xf3a2a055, 0xfaacab58, 0xe1beb64f, 0xe8b0bd42,
+ 0x9fead409, 0x96e4df04, 0x8df6c213, 0x84f8c91e,
+ 0xbbd2f83d, 0xb2dcf330, 0xa9ceee27, 0xa0c0e52a,
+ 0x477a3cb1, 0x4e7437bc, 0x55662aab, 0x5c6821a6,
+ 0x63421085, 0x6a4c1b88, 0x715e069f, 0x78500d92,
+ 0x0f0a64d9, 0x06046fd4, 0x1d1672c3, 0x141879ce,
+ 0x2b3248ed, 0x223c43e0, 0x392e5ef7, 0x302055fa,
+ 0x9aec01b7, 0x93e20aba, 0x88f017ad, 0x81fe1ca0,
+ 0xbed42d83, 0xb7da268e, 0xacc83b99, 0xa5c63094,
+ 0xd29c59df, 0xdb9252d2, 0xc0804fc5, 0xc98e44c8,
+ 0xf6a475eb, 0xffaa7ee6, 0xe4b863f1, 0xedb668fc,
+ 0x0a0cb167, 0x0302ba6a, 0x1810a77d, 0x111eac70,
+ 0x2e349d53, 0x273a965e, 0x3c288b49, 0x35268044,
+ 0x427ce90f, 0x4b72e202, 0x5060ff15, 0x596ef418,
+ 0x6644c53b, 0x6f4ace36, 0x7458d321, 0x7d56d82c,
+ 0xa1377a0c, 0xa8397101, 0xb32b6c16, 0xba25671b,
+ 0x850f5638, 0x8c015d35, 0x97134022, 0x9e1d4b2f,
+ 0xe9472264, 0xe0492969, 0xfb5b347e, 0xf2553f73,
+ 0xcd7f0e50, 0xc471055d, 0xdf63184a, 0xd66d1347,
+ 0x31d7cadc, 0x38d9c1d1, 0x23cbdcc6, 0x2ac5d7cb,
+ 0x15efe6e8, 0x1ce1ede5, 0x07f3f0f2, 0x0efdfbff,
+ 0x79a792b4, 0x70a999b9, 0x6bbb84ae, 0x62b58fa3,
+ 0x5d9fbe80, 0x5491b58d, 0x4f83a89a, 0x468da397
+ },
+ {
+ 0x00000000, 0x0e0b0d09, 0x1c161a12, 0x121d171b,
+ 0x382c3424, 0x3627392d, 0x243a2e36, 0x2a31233f,
+ 0x70586848, 0x7e536541, 0x6c4e725a, 0x62457f53,
+ 0x48745c6c, 0x467f5165, 0x5462467e, 0x5a694b77,
+ 0xe0b0d090, 0xeebbdd99, 0xfca6ca82, 0xf2adc78b,
+ 0xd89ce4b4, 0xd697e9bd, 0xc48afea6, 0xca81f3af,
+ 0x90e8b8d8, 0x9ee3b5d1, 0x8cfea2ca, 0x82f5afc3,
+ 0xa8c48cfc, 0xa6cf81f5, 0xb4d296ee, 0xbad99be7,
+ 0xdb7bbb3b, 0xd570b632, 0xc76da129, 0xc966ac20,
+ 0xe3578f1f, 0xed5c8216, 0xff41950d, 0xf14a9804,
+ 0xab23d373, 0xa528de7a, 0xb735c961, 0xb93ec468,
+ 0x930fe757, 0x9d04ea5e, 0x8f19fd45, 0x8112f04c,
+ 0x3bcb6bab, 0x35c066a2, 0x27dd71b9, 0x29d67cb0,
+ 0x03e75f8f, 0x0dec5286, 0x1ff1459d, 0x11fa4894,
+ 0x4b9303e3, 0x45980eea, 0x578519f1, 0x598e14f8,
+ 0x73bf37c7, 0x7db43ace, 0x6fa92dd5, 0x61a220dc,
+ 0xadf66d76, 0xa3fd607f, 0xb1e07764, 0xbfeb7a6d,
+ 0x95da5952, 0x9bd1545b, 0x89cc4340, 0x87c74e49,
+ 0xddae053e, 0xd3a50837, 0xc1b81f2c, 0xcfb31225,
+ 0xe582311a, 0xeb893c13, 0xf9942b08, 0xf79f2601,
+ 0x4d46bde6, 0x434db0ef, 0x5150a7f4, 0x5f5baafd,
+ 0x756a89c2, 0x7b6184cb, 0x697c93d0, 0x67779ed9,
+ 0x3d1ed5ae, 0x3315d8a7, 0x2108cfbc, 0x2f03c2b5,
+ 0x0532e18a, 0x0b39ec83, 0x1924fb98, 0x172ff691,
+ 0x768dd64d, 0x7886db44, 0x6a9bcc5f, 0x6490c156,
+ 0x4ea1e269, 0x40aaef60, 0x52b7f87b, 0x5cbcf572,
+ 0x06d5be05, 0x08deb30c, 0x1ac3a417, 0x14c8a91e,
+ 0x3ef98a21, 0x30f28728, 0x22ef9033, 0x2ce49d3a,
+ 0x963d06dd, 0x98360bd4, 0x8a2b1ccf, 0x842011c6,
+ 0xae1132f9, 0xa01a3ff0, 0xb20728eb, 0xbc0c25e2,
+ 0xe6656e95, 0xe86e639c, 0xfa737487, 0xf478798e,
+ 0xde495ab1, 0xd04257b8, 0xc25f40a3, 0xcc544daa,
+ 0x41f7daec, 0x4ffcd7e5, 0x5de1c0fe, 0x53eacdf7,
+ 0x79dbeec8, 0x77d0e3c1, 0x65cdf4da, 0x6bc6f9d3,
+ 0x31afb2a4, 0x3fa4bfad, 0x2db9a8b6, 0x23b2a5bf,
+ 0x09838680, 0x07888b89, 0x15959c92, 0x1b9e919b,
+ 0xa1470a7c, 0xaf4c0775, 0xbd51106e, 0xb35a1d67,
+ 0x996b3e58, 0x97603351, 0x857d244a, 0x8b762943,
+ 0xd11f6234, 0xdf146f3d, 0xcd097826, 0xc302752f,
+ 0xe9335610, 0xe7385b19, 0xf5254c02, 0xfb2e410b,
+ 0x9a8c61d7, 0x94876cde, 0x869a7bc5, 0x889176cc,
+ 0xa2a055f3, 0xacab58fa, 0xbeb64fe1, 0xb0bd42e8,
+ 0xead4099f, 0xe4df0496, 0xf6c2138d, 0xf8c91e84,
+ 0xd2f83dbb, 0xdcf330b2, 0xceee27a9, 0xc0e52aa0,
+ 0x7a3cb147, 0x7437bc4e, 0x662aab55, 0x6821a65c,
+ 0x42108563, 0x4c1b886a, 0x5e069f71, 0x500d9278,
+ 0x0a64d90f, 0x046fd406, 0x1672c31d, 0x1879ce14,
+ 0x3248ed2b, 0x3c43e022, 0x2e5ef739, 0x2055fa30,
+ 0xec01b79a, 0xe20aba93, 0xf017ad88, 0xfe1ca081,
+ 0xd42d83be, 0xda268eb7, 0xc83b99ac, 0xc63094a5,
+ 0x9c59dfd2, 0x9252d2db, 0x804fc5c0, 0x8e44c8c9,
+ 0xa475ebf6, 0xaa7ee6ff, 0xb863f1e4, 0xb668fced,
+ 0x0cb1670a, 0x02ba6a03, 0x10a77d18, 0x1eac7011,
+ 0x349d532e, 0x3a965e27, 0x288b493c, 0x26804435,
+ 0x7ce90f42, 0x72e2024b, 0x60ff1550, 0x6ef41859,
+ 0x44c53b66, 0x4ace366f, 0x58d32174, 0x56d82c7d,
+ 0x377a0ca1, 0x397101a8, 0x2b6c16b3, 0x25671bba,
+ 0x0f563885, 0x015d358c, 0x13402297, 0x1d4b2f9e,
+ 0x472264e9, 0x492969e0, 0x5b347efb, 0x553f73f2,
+ 0x7f0e50cd, 0x71055dc4, 0x63184adf, 0x6d1347d6,
+ 0xd7cadc31, 0xd9c1d138, 0xcbdcc623, 0xc5d7cb2a,
+ 0xefe6e815, 0xe1ede51c, 0xf3f0f207, 0xfdfbff0e,
+ 0xa792b479, 0xa999b970, 0xbb84ae6b, 0xb58fa362,
+ 0x9fbe805d, 0x91b58d54, 0x83a89a4f, 0x8da39746
+ }
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _AESTAB2_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2009 Intel Corporation
+ * All Rights Reserved.
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
+ * instructions. This file contains an accelerated
+ * Galois Field Multiplication implementation.
+ *
+ * PCLMULQDQ is used to accelerate the most time-consuming part of GHASH,
+ * carry-less multiplication. More information about PCLMULQDQ can be
+ * found at:
+ * http://software.intel.com/en-us/articles/
+ * carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/
+ *
+ */
+
+/*
+ * ====================================================================
+ * OpenSolaris OS modifications
+ *
+ * This source originates as file galois_hash_asm.c from
+ * Intel Corporation dated September 21, 2009.
+ *
+ * This OpenSolaris version has these major changes from the original source:
+ *
+ * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
+ * /usr/include/sys/asm_linkage.h, lint(1B) guards, and a dummy C function
+ * definition for lint.
+ *
+ * 2. Formatted code, added comments, and added #includes and #defines.
+ *
+ * 3. If bit CR0.TS is set, clear and set the TS bit, after and before
+ * calling kpreempt_disable() and kpreempt_enable().
+ * If the TS bit is not set, Save and restore %xmm registers at the beginning
+ * and end of function calls (%xmm* registers are not saved and restored by
+ * during kernel thread preemption).
+ *
+ * 4. Removed code to perform hashing. This is already done with C macro
+ * GHASH in gcm.c. For better performance, this removed code should be
+ * reintegrated in the future to replace the C GHASH macro.
+ *
+ * 5. Added code to byte swap 16-byte input and output.
+ *
+ * 6. Folded in comments from the original C source with embedded assembly
+ * (SB_w_shift_xor.c)
+ *
+ * 7. Renamed function and reordered parameters to match OpenSolaris:
+ * Intel interface:
+ * void galois_hash_asm(unsigned char *hk, unsigned char *s,
+ * unsigned char *d, int length)
+ * OpenSolaris OS interface:
+ * void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
+ * ====================================================================
+ */
+
+
+#if defined(lint) || defined(__lint)
+
+#include <sys/types.h>
+
+/* ARGSUSED */
+void
+gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res) {
+}
+
+#else /* lint */
+
+#define _ASM
+#include <sys/asm_linkage.h>
+
+#ifdef _KERNEL
+ /*
+ * Note: the CLTS macro clobbers P2 (%rsi) under i86xpv. That is,
+ * it calls HYPERVISOR_fpu_taskswitch() which modifies %rsi when it
+ * uses it to pass P2 to syscall.
+ * This also occurs with the STTS macro, but we dont care if
+ * P2 (%rsi) is modified just before function exit.
+ * The CLTS and STTS macros push and pop P1 (%rdi) already.
+ */
+#ifdef __xpv
+#define PROTECTED_CLTS \
+ push %rsi; \
+ CLTS; \
+ pop %rsi
+#else
+#define PROTECTED_CLTS \
+ CLTS
+#endif /* __xpv */
+
+ /*
+ * If CR0_TS is not set, align stack (with push %rbp) and push
+ * %xmm0 - %xmm10 on stack, otherwise clear CR0_TS
+ */
+#define CLEAR_TS_OR_PUSH_XMM_REGISTERS(tmpreg) \
+ push %rbp; \
+ mov %rsp, %rbp; \
+ movq %cr0, tmpreg; \
+ testq $CR0_TS, tmpreg; \
+ jnz 1f; \
+ and $-XMM_ALIGN, %rsp; \
+ sub $[XMM_SIZE * 11], %rsp; \
+ movaps %xmm0, 160(%rsp); \
+ movaps %xmm1, 144(%rsp); \
+ movaps %xmm2, 128(%rsp); \
+ movaps %xmm3, 112(%rsp); \
+ movaps %xmm4, 96(%rsp); \
+ movaps %xmm5, 80(%rsp); \
+ movaps %xmm6, 64(%rsp); \
+ movaps %xmm7, 48(%rsp); \
+ movaps %xmm8, 32(%rsp); \
+ movaps %xmm9, 16(%rsp); \
+ movaps %xmm10, (%rsp); \
+ jmp 2f; \
+1: \
+ PROTECTED_CLTS; \
+2:
+
+
+ /*
+ * If CR0_TS was not set above, pop %xmm0 - %xmm10 off stack,
+ * otherwise set CR0_TS.
+ */
+#define SET_TS_OR_POP_XMM_REGISTERS(tmpreg) \
+ testq $CR0_TS, tmpreg; \
+ jnz 1f; \
+ movaps (%rsp), %xmm10; \
+ movaps 16(%rsp), %xmm9; \
+ movaps 32(%rsp), %xmm8; \
+ movaps 48(%rsp), %xmm7; \
+ movaps 64(%rsp), %xmm6; \
+ movaps 80(%rsp), %xmm5; \
+ movaps 96(%rsp), %xmm4; \
+ movaps 112(%rsp), %xmm3; \
+ movaps 128(%rsp), %xmm2; \
+ movaps 144(%rsp), %xmm1; \
+ movaps 160(%rsp), %xmm0; \
+ jmp 2f; \
+1: \
+ STTS(tmpreg); \
+2: \
+ mov %rbp, %rsp; \
+ pop %rbp
+
+
+#else
+#define PROTECTED_CLTS
+#define CLEAR_TS_OR_PUSH_XMM_REGISTERS(tmpreg)
+#define SET_TS_OR_POP_XMM_REGISTERS(tmpreg)
+#endif /* _KERNEL */
+
+/*
+ * Use this mask to byte-swap a 16-byte integer with the pshufb instruction
+ */
+
+// static uint8_t byte_swap16_mask[] = {
+// 15, 14, 13, 12, 11, 10, 9, 8, 7, 6 ,5, 4, 3, 2, 1, 0 };
+.text
+.align XMM_ALIGN
+.Lbyte_swap16_mask:
+ .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+
+
+
+/*
+ * void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
+ *
+ * Perform a carry-less multiplication (that is, use XOR instead of the
+ * multiply operator) on P1 and P2 and place the result in P3.
+ *
+ * Byte swap the input and the output.
+ *
+ * Note: x_in, y, and res all point to a block of 20-byte numbers
+ * (an array of two 64-bit integers).
+ *
+ * Note2: For kernel code, caller is responsible for ensuring
+ * kpreempt_disable() has been called. This is because %xmm registers are
+ * not saved/restored. Clear and set the CR0.TS bit on entry and exit,
+ * respectively, if TS is set on entry. Otherwise, if TS is not set,
+ * save and restore %xmm registers on the stack.
+ *
+ * Note3: Original Intel definition:
+ * void galois_hash_asm(unsigned char *hk, unsigned char *s,
+ * unsigned char *d, int length)
+ *
+ * Note4: Register/parameter mapping:
+ * Intel:
+ * Parameter 1: %rcx (copied to %xmm0) hk or x_in
+ * Parameter 2: %rdx (copied to %xmm1) s or y
+ * Parameter 3: %rdi (result) d or res
+ * OpenSolaris:
+ * Parameter 1: %rdi (copied to %xmm0) x_in
+ * Parameter 2: %rsi (copied to %xmm1) y
+ * Parameter 3: %rdx (result) res
+ */
+
+ENTRY_NP(gcm_mul_pclmulqdq)
+ CLEAR_TS_OR_PUSH_XMM_REGISTERS(%r10)
+
+ //
+ // Copy Parameters
+ //
+ movdqu (%rdi), %xmm0 // P1
+ movdqu (%rsi), %xmm1 // P2
+
+ //
+ // Byte swap 16-byte input
+ //
+ lea .Lbyte_swap16_mask(%rip), %rax
+ movaps (%rax), %xmm10
+ pshufb %xmm10, %xmm0
+ pshufb %xmm10, %xmm1
+
+
+ //
+ // Multiply with the hash key
+ //
+ movdqu %xmm0, %xmm3
+ pclmulqdq $0, %xmm1, %xmm3 // xmm3 holds a0*b0
+
+ movdqu %xmm0, %xmm4
+ pclmulqdq $16, %xmm1, %xmm4 // xmm4 holds a0*b1
+
+ movdqu %xmm0, %xmm5
+ pclmulqdq $1, %xmm1, %xmm5 // xmm5 holds a1*b0
+ movdqu %xmm0, %xmm6
+ pclmulqdq $17, %xmm1, %xmm6 // xmm6 holds a1*b1
+
+ pxor %xmm5, %xmm4 // xmm4 holds a0*b1 + a1*b0
+
+ movdqu %xmm4, %xmm5 // move the contents of xmm4 to xmm5
+ psrldq $8, %xmm4 // shift by xmm4 64 bits to the right
+ pslldq $8, %xmm5 // shift by xmm5 64 bits to the left
+ pxor %xmm5, %xmm3
+ pxor %xmm4, %xmm6 // Register pair <xmm6:xmm3> holds the result
+ // of the carry-less multiplication of
+ // xmm0 by xmm1.
+
+ // We shift the result of the multiplication by one bit position
+ // to the left to cope for the fact that the bits are reversed.
+ movdqu %xmm3, %xmm7
+ movdqu %xmm6, %xmm8
+ pslld $1, %xmm3
+ pslld $1, %xmm6
+ psrld $31, %xmm7
+ psrld $31, %xmm8
+ movdqu %xmm7, %xmm9
+ pslldq $4, %xmm8
+ pslldq $4, %xmm7
+ psrldq $12, %xmm9
+ por %xmm7, %xmm3
+ por %xmm8, %xmm6
+ por %xmm9, %xmm6
+
+ //
+ // First phase of the reduction
+ //
+ // Move xmm3 into xmm7, xmm8, xmm9 in order to perform the shifts
+ // independently.
+ movdqu %xmm3, %xmm7
+ movdqu %xmm3, %xmm8
+ movdqu %xmm3, %xmm9
+ pslld $31, %xmm7 // packed right shift shifting << 31
+ pslld $30, %xmm8 // packed right shift shifting << 30
+ pslld $25, %xmm9 // packed right shift shifting << 25
+ pxor %xmm8, %xmm7 // xor the shifted versions
+ pxor %xmm9, %xmm7
+ movdqu %xmm7, %xmm8
+ pslldq $12, %xmm7
+ psrldq $4, %xmm8
+ pxor %xmm7, %xmm3 // first phase of the reduction complete
+
+ //
+ // Second phase of the reduction
+ //
+ // Make 3 copies of xmm3 in xmm2, xmm4, xmm5 for doing these
+ // shift operations.
+ movdqu %xmm3, %xmm2
+ movdqu %xmm3, %xmm4 // packed left shifting >> 1
+ movdqu %xmm3, %xmm5
+ psrld $1, %xmm2
+ psrld $2, %xmm4 // packed left shifting >> 2
+ psrld $7, %xmm5 // packed left shifting >> 7
+ pxor %xmm4, %xmm2 // xor the shifted versions
+ pxor %xmm5, %xmm2
+ pxor %xmm8, %xmm2
+ pxor %xmm2, %xmm3
+ pxor %xmm3, %xmm6 // the result is in xmm6
+
+ //
+ // Byte swap 16-byte result
+ //
+ pshufb %xmm10, %xmm6 // %xmm10 has the swap mask
+
+ //
+ // Store the result
+ //
+ movdqu %xmm6, (%rdx) // P3
+
+
+ //
+ // Cleanup and Return
+ //
+ SET_TS_OR_POP_XMM_REGISTERS(%r10)
+ ret
+ SET_SIZE(gcm_mul_pclmulqdq)
+
+#endif /* lint || __lint */
--- /dev/null
+/*
+ * !/usr/bin/env perl
+ *
+ * ====================================================================
+ * Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
+ * project. The module is, however, dual licensed under OpenSSL and
+ * CRYPTOGAMS licenses depending on where you obtain it. For further
+ * details see http://www.openssl.org/~appro/cryptogams/.
+ * ====================================================================
+ *
+ * sha1_block procedure for x86_64.
+ *
+ * It was brought to my attention that on EM64T compiler-generated code
+ * was far behind 32-bit assembler implementation. This is unlike on
+ * Opteron where compiler-generated code was only 15% behind 32-bit
+ * assembler, which originally made it hard to motivate the effort.
+ * There was suggestion to mechanically translate 32-bit code, but I
+ * dismissed it, reasoning that x86_64 offers enough register bank
+ * capacity to fully utilize SHA-1 parallelism. Therefore this fresh
+ * implementation:-) However! While 64-bit code does performs better
+ * on Opteron, I failed to beat 32-bit assembler on EM64T core. Well,
+ * x86_64 does offer larger *addressable* bank, but out-of-order core
+ * reaches for even more registers through dynamic aliasing, and EM64T
+ * core must have managed to run-time optimize even 32-bit code just as
+ * good as 64-bit one. Performance improvement is summarized in the
+ * following table:
+ *
+ * gcc 3.4 32-bit asm cycles/byte
+ * Opteron +45% +20% 6.8
+ * Xeon P4 +65% +0% 9.9
+ * Core2 +60% +10% 7.0
+ *
+ *
+ * OpenSolaris OS modifications
+ *
+ * Sun elects to use this software under the BSD license.
+ *
+ * This source originates from OpenSSL file sha1-x86_64.pl at
+ * ftp://ftp.openssl.org/snapshot/openssl-0.9.8-stable-SNAP-20080131.tar.gz
+ * (presumably for future OpenSSL release 0.9.8h), with these changes:
+ *
+ * 1. Added perl "use strict" and declared variables.
+ *
+ * 2. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
+ * /usr/include/sys/asm_linkage.h, .ident keywords, and lint(1B) guards.
+ *
+ * 3. Removed x86_64-xlate.pl script (not needed for as(1) or gas(1)
+ * assemblers).
+ *
+ */
+
+/*
+ * This file was generated by a perl script (sha1-x86_64.pl). The comments from
+ * the original file have been pasted above.
+ */
+
+#if defined(lint) || defined(__lint)
+#include <sys/stdint.h>
+#include <sys/sha1.h>
+
+/* ARGSUSED */
+void
+sha1_block_data_order(SHA1_CTX *ctx, const void *inpp, size_t blocks)
+{
+}
+
+#else
+#define _ASM
+#include <sys/asm_linkage.h>
+ENTRY_NP(sha1_block_data_order)
+ push %rbx
+ push %rbp
+ push %r12
+ mov %rsp,%rax
+ mov %rdi,%r8 # reassigned argument
+ sub $72,%rsp
+ mov %rsi,%r9 # reassigned argument
+ and $-64,%rsp
+ mov %rdx,%r10 # reassigned argument
+ mov %rax,64(%rsp)
+
+ mov 0(%r8),%edx
+ mov 4(%r8),%esi
+ mov 8(%r8),%edi
+ mov 12(%r8),%ebp
+ mov 16(%r8),%r11d
+.align 4
+.Lloop:
+ mov 0(%r9),%eax
+ bswap %eax
+ mov %eax,0(%rsp)
+ lea 0x5a827999(%eax,%r11d),%r12d
+ mov %edi,%ebx
+ mov 4(%r9),%eax
+ mov %edx,%r11d
+ xor %ebp,%ebx
+ bswap %eax
+ rol $5,%r11d
+ and %esi,%ebx
+ mov %eax,4(%rsp)
+ add %r11d,%r12d
+ xor %ebp,%ebx
+ rol $30,%esi
+ add %ebx,%r12d
+ lea 0x5a827999(%eax,%ebp),%r11d
+ mov %esi,%ebx
+ mov 8(%r9),%eax
+ mov %r12d,%ebp
+ xor %edi,%ebx
+ bswap %eax
+ rol $5,%ebp
+ and %edx,%ebx
+ mov %eax,8(%rsp)
+ add %ebp,%r11d
+ xor %edi,%ebx
+ rol $30,%edx
+ add %ebx,%r11d
+ lea 0x5a827999(%eax,%edi),%ebp
+ mov %edx,%ebx
+ mov 12(%r9),%eax
+ mov %r11d,%edi
+ xor %esi,%ebx
+ bswap %eax
+ rol $5,%edi
+ and %r12d,%ebx
+ mov %eax,12(%rsp)
+ add %edi,%ebp
+ xor %esi,%ebx
+ rol $30,%r12d
+ add %ebx,%ebp
+ lea 0x5a827999(%eax,%esi),%edi
+ mov %r12d,%ebx
+ mov 16(%r9),%eax
+ mov %ebp,%esi
+ xor %edx,%ebx
+ bswap %eax
+ rol $5,%esi
+ and %r11d,%ebx
+ mov %eax,16(%rsp)
+ add %esi,%edi
+ xor %edx,%ebx
+ rol $30,%r11d
+ add %ebx,%edi
+ lea 0x5a827999(%eax,%edx),%esi
+ mov %r11d,%ebx
+ mov 20(%r9),%eax
+ mov %edi,%edx
+ xor %r12d,%ebx
+ bswap %eax
+ rol $5,%edx
+ and %ebp,%ebx
+ mov %eax,20(%rsp)
+ add %edx,%esi
+ xor %r12d,%ebx
+ rol $30,%ebp
+ add %ebx,%esi
+ lea 0x5a827999(%eax,%r12d),%edx
+ mov %ebp,%ebx
+ mov 24(%r9),%eax
+ mov %esi,%r12d
+ xor %r11d,%ebx
+ bswap %eax
+ rol $5,%r12d
+ and %edi,%ebx
+ mov %eax,24(%rsp)
+ add %r12d,%edx
+ xor %r11d,%ebx
+ rol $30,%edi
+ add %ebx,%edx
+ lea 0x5a827999(%eax,%r11d),%r12d
+ mov %edi,%ebx
+ mov 28(%r9),%eax
+ mov %edx,%r11d
+ xor %ebp,%ebx
+ bswap %eax
+ rol $5,%r11d
+ and %esi,%ebx
+ mov %eax,28(%rsp)
+ add %r11d,%r12d
+ xor %ebp,%ebx
+ rol $30,%esi
+ add %ebx,%r12d
+ lea 0x5a827999(%eax,%ebp),%r11d
+ mov %esi,%ebx
+ mov 32(%r9),%eax
+ mov %r12d,%ebp
+ xor %edi,%ebx
+ bswap %eax
+ rol $5,%ebp
+ and %edx,%ebx
+ mov %eax,32(%rsp)
+ add %ebp,%r11d
+ xor %edi,%ebx
+ rol $30,%edx
+ add %ebx,%r11d
+ lea 0x5a827999(%eax,%edi),%ebp
+ mov %edx,%ebx
+ mov 36(%r9),%eax
+ mov %r11d,%edi
+ xor %esi,%ebx
+ bswap %eax
+ rol $5,%edi
+ and %r12d,%ebx
+ mov %eax,36(%rsp)
+ add %edi,%ebp
+ xor %esi,%ebx
+ rol $30,%r12d
+ add %ebx,%ebp
+ lea 0x5a827999(%eax,%esi),%edi
+ mov %r12d,%ebx
+ mov 40(%r9),%eax
+ mov %ebp,%esi
+ xor %edx,%ebx
+ bswap %eax
+ rol $5,%esi
+ and %r11d,%ebx
+ mov %eax,40(%rsp)
+ add %esi,%edi
+ xor %edx,%ebx
+ rol $30,%r11d
+ add %ebx,%edi
+ lea 0x5a827999(%eax,%edx),%esi
+ mov %r11d,%ebx
+ mov 44(%r9),%eax
+ mov %edi,%edx
+ xor %r12d,%ebx
+ bswap %eax
+ rol $5,%edx
+ and %ebp,%ebx
+ mov %eax,44(%rsp)
+ add %edx,%esi
+ xor %r12d,%ebx
+ rol $30,%ebp
+ add %ebx,%esi
+ lea 0x5a827999(%eax,%r12d),%edx
+ mov %ebp,%ebx
+ mov 48(%r9),%eax
+ mov %esi,%r12d
+ xor %r11d,%ebx
+ bswap %eax
+ rol $5,%r12d
+ and %edi,%ebx
+ mov %eax,48(%rsp)
+ add %r12d,%edx
+ xor %r11d,%ebx
+ rol $30,%edi
+ add %ebx,%edx
+ lea 0x5a827999(%eax,%r11d),%r12d
+ mov %edi,%ebx
+ mov 52(%r9),%eax
+ mov %edx,%r11d
+ xor %ebp,%ebx
+ bswap %eax
+ rol $5,%r11d
+ and %esi,%ebx
+ mov %eax,52(%rsp)
+ add %r11d,%r12d
+ xor %ebp,%ebx
+ rol $30,%esi
+ add %ebx,%r12d
+ lea 0x5a827999(%eax,%ebp),%r11d
+ mov %esi,%ebx
+ mov 56(%r9),%eax
+ mov %r12d,%ebp
+ xor %edi,%ebx
+ bswap %eax
+ rol $5,%ebp
+ and %edx,%ebx
+ mov %eax,56(%rsp)
+ add %ebp,%r11d
+ xor %edi,%ebx
+ rol $30,%edx
+ add %ebx,%r11d
+ lea 0x5a827999(%eax,%edi),%ebp
+ mov %edx,%ebx
+ mov 60(%r9),%eax
+ mov %r11d,%edi
+ xor %esi,%ebx
+ bswap %eax
+ rol $5,%edi
+ and %r12d,%ebx
+ mov %eax,60(%rsp)
+ add %edi,%ebp
+ xor %esi,%ebx
+ rol $30,%r12d
+ add %ebx,%ebp
+ lea 0x5a827999(%eax,%esi),%edi
+ mov 0(%rsp),%eax
+ mov %r12d,%ebx
+ mov %ebp,%esi
+ xor 8(%rsp),%eax
+ xor %edx,%ebx
+ rol $5,%esi
+ xor 32(%rsp),%eax
+ and %r11d,%ebx
+ add %esi,%edi
+ xor 52(%rsp),%eax
+ xor %edx,%ebx
+ rol $30,%r11d
+ add %ebx,%edi
+ rol $1,%eax
+ mov %eax,0(%rsp)
+ lea 0x5a827999(%eax,%edx),%esi
+ mov 4(%rsp),%eax
+ mov %r11d,%ebx
+ mov %edi,%edx
+ xor 12(%rsp),%eax
+ xor %r12d,%ebx
+ rol $5,%edx
+ xor 36(%rsp),%eax
+ and %ebp,%ebx
+ add %edx,%esi
+ xor 56(%rsp),%eax
+ xor %r12d,%ebx
+ rol $30,%ebp
+ add %ebx,%esi
+ rol $1,%eax
+ mov %eax,4(%rsp)
+ lea 0x5a827999(%eax,%r12d),%edx
+ mov 8(%rsp),%eax
+ mov %ebp,%ebx
+ mov %esi,%r12d
+ xor 16(%rsp),%eax
+ xor %r11d,%ebx
+ rol $5,%r12d
+ xor 40(%rsp),%eax
+ and %edi,%ebx
+ add %r12d,%edx
+ xor 60(%rsp),%eax
+ xor %r11d,%ebx
+ rol $30,%edi
+ add %ebx,%edx
+ rol $1,%eax
+ mov %eax,8(%rsp)
+ lea 0x5a827999(%eax,%r11d),%r12d
+ mov 12(%rsp),%eax
+ mov %edi,%ebx
+ mov %edx,%r11d
+ xor 20(%rsp),%eax
+ xor %ebp,%ebx
+ rol $5,%r11d
+ xor 44(%rsp),%eax
+ and %esi,%ebx
+ add %r11d,%r12d
+ xor 0(%rsp),%eax
+ xor %ebp,%ebx
+ rol $30,%esi
+ add %ebx,%r12d
+ rol $1,%eax
+ mov %eax,12(%rsp)
+ lea 0x5a827999(%eax,%ebp),%r11d
+ mov 16(%rsp),%eax
+ mov %esi,%ebx
+ mov %r12d,%ebp
+ xor 24(%rsp),%eax
+ xor %edi,%ebx
+ rol $5,%ebp
+ xor 48(%rsp),%eax
+ and %edx,%ebx
+ add %ebp,%r11d
+ xor 4(%rsp),%eax
+ xor %edi,%ebx
+ rol $30,%edx
+ add %ebx,%r11d
+ rol $1,%eax
+ mov %eax,16(%rsp)
+ lea 0x6ed9eba1(%eax,%edi),%ebp
+ mov 20(%rsp),%eax
+ mov %edx,%ebx
+ mov %r11d,%edi
+ xor 28(%rsp),%eax
+ xor %r12d,%ebx
+ rol $5,%edi
+ xor 52(%rsp),%eax
+ xor %esi,%ebx
+ add %edi,%ebp
+ xor 8(%rsp),%eax
+ rol $30,%r12d
+ add %ebx,%ebp
+ rol $1,%eax
+ mov %eax,20(%rsp)
+ lea 0x6ed9eba1(%eax,%esi),%edi
+ mov 24(%rsp),%eax
+ mov %r12d,%ebx
+ mov %ebp,%esi
+ xor 32(%rsp),%eax
+ xor %r11d,%ebx
+ rol $5,%esi
+ xor 56(%rsp),%eax
+ xor %edx,%ebx
+ add %esi,%edi
+ xor 12(%rsp),%eax
+ rol $30,%r11d
+ add %ebx,%edi
+ rol $1,%eax
+ mov %eax,24(%rsp)
+ lea 0x6ed9eba1(%eax,%edx),%esi
+ mov 28(%rsp),%eax
+ mov %r11d,%ebx
+ mov %edi,%edx
+ xor 36(%rsp),%eax
+ xor %ebp,%ebx
+ rol $5,%edx
+ xor 60(%rsp),%eax
+ xor %r12d,%ebx
+ add %edx,%esi
+ xor 16(%rsp),%eax
+ rol $30,%ebp
+ add %ebx,%esi
+ rol $1,%eax
+ mov %eax,28(%rsp)
+ lea 0x6ed9eba1(%eax,%r12d),%edx
+ mov 32(%rsp),%eax
+ mov %ebp,%ebx
+ mov %esi,%r12d
+ xor 40(%rsp),%eax
+ xor %edi,%ebx
+ rol $5,%r12d
+ xor 0(%rsp),%eax
+ xor %r11d,%ebx
+ add %r12d,%edx
+ xor 20(%rsp),%eax
+ rol $30,%edi
+ add %ebx,%edx
+ rol $1,%eax
+ mov %eax,32(%rsp)
+ lea 0x6ed9eba1(%eax,%r11d),%r12d
+ mov 36(%rsp),%eax
+ mov %edi,%ebx
+ mov %edx,%r11d
+ xor 44(%rsp),%eax
+ xor %esi,%ebx
+ rol $5,%r11d
+ xor 4(%rsp),%eax
+ xor %ebp,%ebx
+ add %r11d,%r12d
+ xor 24(%rsp),%eax
+ rol $30,%esi
+ add %ebx,%r12d
+ rol $1,%eax
+ mov %eax,36(%rsp)
+ lea 0x6ed9eba1(%eax,%ebp),%r11d
+ mov 40(%rsp),%eax
+ mov %esi,%ebx
+ mov %r12d,%ebp
+ xor 48(%rsp),%eax
+ xor %edx,%ebx
+ rol $5,%ebp
+ xor 8(%rsp),%eax
+ xor %edi,%ebx
+ add %ebp,%r11d
+ xor 28(%rsp),%eax
+ rol $30,%edx
+ add %ebx,%r11d
+ rol $1,%eax
+ mov %eax,40(%rsp)
+ lea 0x6ed9eba1(%eax,%edi),%ebp
+ mov 44(%rsp),%eax
+ mov %edx,%ebx
+ mov %r11d,%edi
+ xor 52(%rsp),%eax
+ xor %r12d,%ebx
+ rol $5,%edi
+ xor 12(%rsp),%eax
+ xor %esi,%ebx
+ add %edi,%ebp
+ xor 32(%rsp),%eax
+ rol $30,%r12d
+ add %ebx,%ebp
+ rol $1,%eax
+ mov %eax,44(%rsp)
+ lea 0x6ed9eba1(%eax,%esi),%edi
+ mov 48(%rsp),%eax
+ mov %r12d,%ebx
+ mov %ebp,%esi
+ xor 56(%rsp),%eax
+ xor %r11d,%ebx
+ rol $5,%esi
+ xor 16(%rsp),%eax
+ xor %edx,%ebx
+ add %esi,%edi
+ xor 36(%rsp),%eax
+ rol $30,%r11d
+ add %ebx,%edi
+ rol $1,%eax
+ mov %eax,48(%rsp)
+ lea 0x6ed9eba1(%eax,%edx),%esi
+ mov 52(%rsp),%eax
+ mov %r11d,%ebx
+ mov %edi,%edx
+ xor 60(%rsp),%eax
+ xor %ebp,%ebx
+ rol $5,%edx
+ xor 20(%rsp),%eax
+ xor %r12d,%ebx
+ add %edx,%esi
+ xor 40(%rsp),%eax
+ rol $30,%ebp
+ add %ebx,%esi
+ rol $1,%eax
+ mov %eax,52(%rsp)
+ lea 0x6ed9eba1(%eax,%r12d),%edx
+ mov 56(%rsp),%eax
+ mov %ebp,%ebx
+ mov %esi,%r12d
+ xor 0(%rsp),%eax
+ xor %edi,%ebx
+ rol $5,%r12d
+ xor 24(%rsp),%eax
+ xor %r11d,%ebx
+ add %r12d,%edx
+ xor 44(%rsp),%eax
+ rol $30,%edi
+ add %ebx,%edx
+ rol $1,%eax
+ mov %eax,56(%rsp)
+ lea 0x6ed9eba1(%eax,%r11d),%r12d
+ mov 60(%rsp),%eax
+ mov %edi,%ebx
+ mov %edx,%r11d
+ xor 4(%rsp),%eax
+ xor %esi,%ebx
+ rol $5,%r11d
+ xor 28(%rsp),%eax
+ xor %ebp,%ebx
+ add %r11d,%r12d
+ xor 48(%rsp),%eax
+ rol $30,%esi
+ add %ebx,%r12d
+ rol $1,%eax
+ mov %eax,60(%rsp)
+ lea 0x6ed9eba1(%eax,%ebp),%r11d
+ mov 0(%rsp),%eax
+ mov %esi,%ebx
+ mov %r12d,%ebp
+ xor 8(%rsp),%eax
+ xor %edx,%ebx
+ rol $5,%ebp
+ xor 32(%rsp),%eax
+ xor %edi,%ebx
+ add %ebp,%r11d
+ xor 52(%rsp),%eax
+ rol $30,%edx
+ add %ebx,%r11d
+ rol $1,%eax
+ mov %eax,0(%rsp)
+ lea 0x6ed9eba1(%eax,%edi),%ebp
+ mov 4(%rsp),%eax
+ mov %edx,%ebx
+ mov %r11d,%edi
+ xor 12(%rsp),%eax
+ xor %r12d,%ebx
+ rol $5,%edi
+ xor 36(%rsp),%eax
+ xor %esi,%ebx
+ add %edi,%ebp
+ xor 56(%rsp),%eax
+ rol $30,%r12d
+ add %ebx,%ebp
+ rol $1,%eax
+ mov %eax,4(%rsp)
+ lea 0x6ed9eba1(%eax,%esi),%edi
+ mov 8(%rsp),%eax
+ mov %r12d,%ebx
+ mov %ebp,%esi
+ xor 16(%rsp),%eax
+ xor %r11d,%ebx
+ rol $5,%esi
+ xor 40(%rsp),%eax
+ xor %edx,%ebx
+ add %esi,%edi
+ xor 60(%rsp),%eax
+ rol $30,%r11d
+ add %ebx,%edi
+ rol $1,%eax
+ mov %eax,8(%rsp)
+ lea 0x6ed9eba1(%eax,%edx),%esi
+ mov 12(%rsp),%eax
+ mov %r11d,%ebx
+ mov %edi,%edx
+ xor 20(%rsp),%eax
+ xor %ebp,%ebx
+ rol $5,%edx
+ xor 44(%rsp),%eax
+ xor %r12d,%ebx
+ add %edx,%esi
+ xor 0(%rsp),%eax
+ rol $30,%ebp
+ add %ebx,%esi
+ rol $1,%eax
+ mov %eax,12(%rsp)
+ lea 0x6ed9eba1(%eax,%r12d),%edx
+ mov 16(%rsp),%eax
+ mov %ebp,%ebx
+ mov %esi,%r12d
+ xor 24(%rsp),%eax
+ xor %edi,%ebx
+ rol $5,%r12d
+ xor 48(%rsp),%eax
+ xor %r11d,%ebx
+ add %r12d,%edx
+ xor 4(%rsp),%eax
+ rol $30,%edi
+ add %ebx,%edx
+ rol $1,%eax
+ mov %eax,16(%rsp)
+ lea 0x6ed9eba1(%eax,%r11d),%r12d
+ mov 20(%rsp),%eax
+ mov %edi,%ebx
+ mov %edx,%r11d
+ xor 28(%rsp),%eax
+ xor %esi,%ebx
+ rol $5,%r11d
+ xor 52(%rsp),%eax
+ xor %ebp,%ebx
+ add %r11d,%r12d
+ xor 8(%rsp),%eax
+ rol $30,%esi
+ add %ebx,%r12d
+ rol $1,%eax
+ mov %eax,20(%rsp)
+ lea 0x6ed9eba1(%eax,%ebp),%r11d
+ mov 24(%rsp),%eax
+ mov %esi,%ebx
+ mov %r12d,%ebp
+ xor 32(%rsp),%eax
+ xor %edx,%ebx
+ rol $5,%ebp
+ xor 56(%rsp),%eax
+ xor %edi,%ebx
+ add %ebp,%r11d
+ xor 12(%rsp),%eax
+ rol $30,%edx
+ add %ebx,%r11d
+ rol $1,%eax
+ mov %eax,24(%rsp)
+ lea 0x6ed9eba1(%eax,%edi),%ebp
+ mov 28(%rsp),%eax
+ mov %edx,%ebx
+ mov %r11d,%edi
+ xor 36(%rsp),%eax
+ xor %r12d,%ebx
+ rol $5,%edi
+ xor 60(%rsp),%eax
+ xor %esi,%ebx
+ add %edi,%ebp
+ xor 16(%rsp),%eax
+ rol $30,%r12d
+ add %ebx,%ebp
+ rol $1,%eax
+ mov %eax,28(%rsp)
+ lea 0x6ed9eba1(%eax,%esi),%edi
+ mov 32(%rsp),%eax
+ mov %r12d,%ebx
+ mov %ebp,%esi
+ xor 40(%rsp),%eax
+ xor %r11d,%ebx
+ rol $5,%esi
+ xor 0(%rsp),%eax
+ xor %edx,%ebx
+ add %esi,%edi
+ xor 20(%rsp),%eax
+ rol $30,%r11d
+ add %ebx,%edi
+ rol $1,%eax
+ mov %eax,32(%rsp)
+ lea -0x70e44324(%eax,%edx),%esi
+ mov 36(%rsp),%eax
+ mov %ebp,%ebx
+ mov %ebp,%ecx
+ xor 44(%rsp),%eax
+ mov %edi,%edx
+ and %r11d,%ebx
+ xor 4(%rsp),%eax
+ or %r11d,%ecx
+ rol $5,%edx
+ xor 24(%rsp),%eax
+ and %r12d,%ecx
+ add %edx,%esi
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%ebp
+ mov %eax,36(%rsp)
+ add %ebx,%esi
+ lea -0x70e44324(%eax,%r12d),%edx
+ mov 40(%rsp),%eax
+ mov %edi,%ebx
+ mov %edi,%ecx
+ xor 48(%rsp),%eax
+ mov %esi,%r12d
+ and %ebp,%ebx
+ xor 8(%rsp),%eax
+ or %ebp,%ecx
+ rol $5,%r12d
+ xor 28(%rsp),%eax
+ and %r11d,%ecx
+ add %r12d,%edx
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%edi
+ mov %eax,40(%rsp)
+ add %ebx,%edx
+ lea -0x70e44324(%eax,%r11d),%r12d
+ mov 44(%rsp),%eax
+ mov %esi,%ebx
+ mov %esi,%ecx
+ xor 52(%rsp),%eax
+ mov %edx,%r11d
+ and %edi,%ebx
+ xor 12(%rsp),%eax
+ or %edi,%ecx
+ rol $5,%r11d
+ xor 32(%rsp),%eax
+ and %ebp,%ecx
+ add %r11d,%r12d
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%esi
+ mov %eax,44(%rsp)
+ add %ebx,%r12d
+ lea -0x70e44324(%eax,%ebp),%r11d
+ mov 48(%rsp),%eax
+ mov %edx,%ebx
+ mov %edx,%ecx
+ xor 56(%rsp),%eax
+ mov %r12d,%ebp
+ and %esi,%ebx
+ xor 16(%rsp),%eax
+ or %esi,%ecx
+ rol $5,%ebp
+ xor 36(%rsp),%eax
+ and %edi,%ecx
+ add %ebp,%r11d
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%edx
+ mov %eax,48(%rsp)
+ add %ebx,%r11d
+ lea -0x70e44324(%eax,%edi),%ebp
+ mov 52(%rsp),%eax
+ mov %r12d,%ebx
+ mov %r12d,%ecx
+ xor 60(%rsp),%eax
+ mov %r11d,%edi
+ and %edx,%ebx
+ xor 20(%rsp),%eax
+ or %edx,%ecx
+ rol $5,%edi
+ xor 40(%rsp),%eax
+ and %esi,%ecx
+ add %edi,%ebp
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%r12d
+ mov %eax,52(%rsp)
+ add %ebx,%ebp
+ lea -0x70e44324(%eax,%esi),%edi
+ mov 56(%rsp),%eax
+ mov %r11d,%ebx
+ mov %r11d,%ecx
+ xor 0(%rsp),%eax
+ mov %ebp,%esi
+ and %r12d,%ebx
+ xor 24(%rsp),%eax
+ or %r12d,%ecx
+ rol $5,%esi
+ xor 44(%rsp),%eax
+ and %edx,%ecx
+ add %esi,%edi
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%r11d
+ mov %eax,56(%rsp)
+ add %ebx,%edi
+ lea -0x70e44324(%eax,%edx),%esi
+ mov 60(%rsp),%eax
+ mov %ebp,%ebx
+ mov %ebp,%ecx
+ xor 4(%rsp),%eax
+ mov %edi,%edx
+ and %r11d,%ebx
+ xor 28(%rsp),%eax
+ or %r11d,%ecx
+ rol $5,%edx
+ xor 48(%rsp),%eax
+ and %r12d,%ecx
+ add %edx,%esi
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%ebp
+ mov %eax,60(%rsp)
+ add %ebx,%esi
+ lea -0x70e44324(%eax,%r12d),%edx
+ mov 0(%rsp),%eax
+ mov %edi,%ebx
+ mov %edi,%ecx
+ xor 8(%rsp),%eax
+ mov %esi,%r12d
+ and %ebp,%ebx
+ xor 32(%rsp),%eax
+ or %ebp,%ecx
+ rol $5,%r12d
+ xor 52(%rsp),%eax
+ and %r11d,%ecx
+ add %r12d,%edx
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%edi
+ mov %eax,0(%rsp)
+ add %ebx,%edx
+ lea -0x70e44324(%eax,%r11d),%r12d
+ mov 4(%rsp),%eax
+ mov %esi,%ebx
+ mov %esi,%ecx
+ xor 12(%rsp),%eax
+ mov %edx,%r11d
+ and %edi,%ebx
+ xor 36(%rsp),%eax
+ or %edi,%ecx
+ rol $5,%r11d
+ xor 56(%rsp),%eax
+ and %ebp,%ecx
+ add %r11d,%r12d
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%esi
+ mov %eax,4(%rsp)
+ add %ebx,%r12d
+ lea -0x70e44324(%eax,%ebp),%r11d
+ mov 8(%rsp),%eax
+ mov %edx,%ebx
+ mov %edx,%ecx
+ xor 16(%rsp),%eax
+ mov %r12d,%ebp
+ and %esi,%ebx
+ xor 40(%rsp),%eax
+ or %esi,%ecx
+ rol $5,%ebp
+ xor 60(%rsp),%eax
+ and %edi,%ecx
+ add %ebp,%r11d
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%edx
+ mov %eax,8(%rsp)
+ add %ebx,%r11d
+ lea -0x70e44324(%eax,%edi),%ebp
+ mov 12(%rsp),%eax
+ mov %r12d,%ebx
+ mov %r12d,%ecx
+ xor 20(%rsp),%eax
+ mov %r11d,%edi
+ and %edx,%ebx
+ xor 44(%rsp),%eax
+ or %edx,%ecx
+ rol $5,%edi
+ xor 0(%rsp),%eax
+ and %esi,%ecx
+ add %edi,%ebp
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%r12d
+ mov %eax,12(%rsp)
+ add %ebx,%ebp
+ lea -0x70e44324(%eax,%esi),%edi
+ mov 16(%rsp),%eax
+ mov %r11d,%ebx
+ mov %r11d,%ecx
+ xor 24(%rsp),%eax
+ mov %ebp,%esi
+ and %r12d,%ebx
+ xor 48(%rsp),%eax
+ or %r12d,%ecx
+ rol $5,%esi
+ xor 4(%rsp),%eax
+ and %edx,%ecx
+ add %esi,%edi
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%r11d
+ mov %eax,16(%rsp)
+ add %ebx,%edi
+ lea -0x70e44324(%eax,%edx),%esi
+ mov 20(%rsp),%eax
+ mov %ebp,%ebx
+ mov %ebp,%ecx
+ xor 28(%rsp),%eax
+ mov %edi,%edx
+ and %r11d,%ebx
+ xor 52(%rsp),%eax
+ or %r11d,%ecx
+ rol $5,%edx
+ xor 8(%rsp),%eax
+ and %r12d,%ecx
+ add %edx,%esi
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%ebp
+ mov %eax,20(%rsp)
+ add %ebx,%esi
+ lea -0x70e44324(%eax,%r12d),%edx
+ mov 24(%rsp),%eax
+ mov %edi,%ebx
+ mov %edi,%ecx
+ xor 32(%rsp),%eax
+ mov %esi,%r12d
+ and %ebp,%ebx
+ xor 56(%rsp),%eax
+ or %ebp,%ecx
+ rol $5,%r12d
+ xor 12(%rsp),%eax
+ and %r11d,%ecx
+ add %r12d,%edx
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%edi
+ mov %eax,24(%rsp)
+ add %ebx,%edx
+ lea -0x70e44324(%eax,%r11d),%r12d
+ mov 28(%rsp),%eax
+ mov %esi,%ebx
+ mov %esi,%ecx
+ xor 36(%rsp),%eax
+ mov %edx,%r11d
+ and %edi,%ebx
+ xor 60(%rsp),%eax
+ or %edi,%ecx
+ rol $5,%r11d
+ xor 16(%rsp),%eax
+ and %ebp,%ecx
+ add %r11d,%r12d
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%esi
+ mov %eax,28(%rsp)
+ add %ebx,%r12d
+ lea -0x70e44324(%eax,%ebp),%r11d
+ mov 32(%rsp),%eax
+ mov %edx,%ebx
+ mov %edx,%ecx
+ xor 40(%rsp),%eax
+ mov %r12d,%ebp
+ and %esi,%ebx
+ xor 0(%rsp),%eax
+ or %esi,%ecx
+ rol $5,%ebp
+ xor 20(%rsp),%eax
+ and %edi,%ecx
+ add %ebp,%r11d
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%edx
+ mov %eax,32(%rsp)
+ add %ebx,%r11d
+ lea -0x70e44324(%eax,%edi),%ebp
+ mov 36(%rsp),%eax
+ mov %r12d,%ebx
+ mov %r12d,%ecx
+ xor 44(%rsp),%eax
+ mov %r11d,%edi
+ and %edx,%ebx
+ xor 4(%rsp),%eax
+ or %edx,%ecx
+ rol $5,%edi
+ xor 24(%rsp),%eax
+ and %esi,%ecx
+ add %edi,%ebp
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%r12d
+ mov %eax,36(%rsp)
+ add %ebx,%ebp
+ lea -0x70e44324(%eax,%esi),%edi
+ mov 40(%rsp),%eax
+ mov %r11d,%ebx
+ mov %r11d,%ecx
+ xor 48(%rsp),%eax
+ mov %ebp,%esi
+ and %r12d,%ebx
+ xor 8(%rsp),%eax
+ or %r12d,%ecx
+ rol $5,%esi
+ xor 28(%rsp),%eax
+ and %edx,%ecx
+ add %esi,%edi
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%r11d
+ mov %eax,40(%rsp)
+ add %ebx,%edi
+ lea -0x70e44324(%eax,%edx),%esi
+ mov 44(%rsp),%eax
+ mov %ebp,%ebx
+ mov %ebp,%ecx
+ xor 52(%rsp),%eax
+ mov %edi,%edx
+ and %r11d,%ebx
+ xor 12(%rsp),%eax
+ or %r11d,%ecx
+ rol $5,%edx
+ xor 32(%rsp),%eax
+ and %r12d,%ecx
+ add %edx,%esi
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%ebp
+ mov %eax,44(%rsp)
+ add %ebx,%esi
+ lea -0x70e44324(%eax,%r12d),%edx
+ mov 48(%rsp),%eax
+ mov %edi,%ebx
+ mov %edi,%ecx
+ xor 56(%rsp),%eax
+ mov %esi,%r12d
+ and %ebp,%ebx
+ xor 16(%rsp),%eax
+ or %ebp,%ecx
+ rol $5,%r12d
+ xor 36(%rsp),%eax
+ and %r11d,%ecx
+ add %r12d,%edx
+ rol $1,%eax
+ or %ecx,%ebx
+ rol $30,%edi
+ mov %eax,48(%rsp)
+ add %ebx,%edx
+ lea -0x359d3e2a(%eax,%r11d),%r12d
+ mov 52(%rsp),%eax
+ mov %edi,%ebx
+ mov %edx,%r11d
+ xor 60(%rsp),%eax
+ xor %esi,%ebx
+ rol $5,%r11d
+ xor 20(%rsp),%eax
+ xor %ebp,%ebx
+ add %r11d,%r12d
+ xor 40(%rsp),%eax
+ rol $30,%esi
+ add %ebx,%r12d
+ rol $1,%eax
+ mov %eax,52(%rsp)
+ lea -0x359d3e2a(%eax,%ebp),%r11d
+ mov 56(%rsp),%eax
+ mov %esi,%ebx
+ mov %r12d,%ebp
+ xor 0(%rsp),%eax
+ xor %edx,%ebx
+ rol $5,%ebp
+ xor 24(%rsp),%eax
+ xor %edi,%ebx
+ add %ebp,%r11d
+ xor 44(%rsp),%eax
+ rol $30,%edx
+ add %ebx,%r11d
+ rol $1,%eax
+ mov %eax,56(%rsp)
+ lea -0x359d3e2a(%eax,%edi),%ebp
+ mov 60(%rsp),%eax
+ mov %edx,%ebx
+ mov %r11d,%edi
+ xor 4(%rsp),%eax
+ xor %r12d,%ebx
+ rol $5,%edi
+ xor 28(%rsp),%eax
+ xor %esi,%ebx
+ add %edi,%ebp
+ xor 48(%rsp),%eax
+ rol $30,%r12d
+ add %ebx,%ebp
+ rol $1,%eax
+ mov %eax,60(%rsp)
+ lea -0x359d3e2a(%eax,%esi),%edi
+ mov 0(%rsp),%eax
+ mov %r12d,%ebx
+ mov %ebp,%esi
+ xor 8(%rsp),%eax
+ xor %r11d,%ebx
+ rol $5,%esi
+ xor 32(%rsp),%eax
+ xor %edx,%ebx
+ add %esi,%edi
+ xor 52(%rsp),%eax
+ rol $30,%r11d
+ add %ebx,%edi
+ rol $1,%eax
+ mov %eax,0(%rsp)
+ lea -0x359d3e2a(%eax,%edx),%esi
+ mov 4(%rsp),%eax
+ mov %r11d,%ebx
+ mov %edi,%edx
+ xor 12(%rsp),%eax
+ xor %ebp,%ebx
+ rol $5,%edx
+ xor 36(%rsp),%eax
+ xor %r12d,%ebx
+ add %edx,%esi
+ xor 56(%rsp),%eax
+ rol $30,%ebp
+ add %ebx,%esi
+ rol $1,%eax
+ mov %eax,4(%rsp)
+ lea -0x359d3e2a(%eax,%r12d),%edx
+ mov 8(%rsp),%eax
+ mov %ebp,%ebx
+ mov %esi,%r12d
+ xor 16(%rsp),%eax
+ xor %edi,%ebx
+ rol $5,%r12d
+ xor 40(%rsp),%eax
+ xor %r11d,%ebx
+ add %r12d,%edx
+ xor 60(%rsp),%eax
+ rol $30,%edi
+ add %ebx,%edx
+ rol $1,%eax
+ mov %eax,8(%rsp)
+ lea -0x359d3e2a(%eax,%r11d),%r12d
+ mov 12(%rsp),%eax
+ mov %edi,%ebx
+ mov %edx,%r11d
+ xor 20(%rsp),%eax
+ xor %esi,%ebx
+ rol $5,%r11d
+ xor 44(%rsp),%eax
+ xor %ebp,%ebx
+ add %r11d,%r12d
+ xor 0(%rsp),%eax
+ rol $30,%esi
+ add %ebx,%r12d
+ rol $1,%eax
+ mov %eax,12(%rsp)
+ lea -0x359d3e2a(%eax,%ebp),%r11d
+ mov 16(%rsp),%eax
+ mov %esi,%ebx
+ mov %r12d,%ebp
+ xor 24(%rsp),%eax
+ xor %edx,%ebx
+ rol $5,%ebp
+ xor 48(%rsp),%eax
+ xor %edi,%ebx
+ add %ebp,%r11d
+ xor 4(%rsp),%eax
+ rol $30,%edx
+ add %ebx,%r11d
+ rol $1,%eax
+ mov %eax,16(%rsp)
+ lea -0x359d3e2a(%eax,%edi),%ebp
+ mov 20(%rsp),%eax
+ mov %edx,%ebx
+ mov %r11d,%edi
+ xor 28(%rsp),%eax
+ xor %r12d,%ebx
+ rol $5,%edi
+ xor 52(%rsp),%eax
+ xor %esi,%ebx
+ add %edi,%ebp
+ xor 8(%rsp),%eax
+ rol $30,%r12d
+ add %ebx,%ebp
+ rol $1,%eax
+ mov %eax,20(%rsp)
+ lea -0x359d3e2a(%eax,%esi),%edi
+ mov 24(%rsp),%eax
+ mov %r12d,%ebx
+ mov %ebp,%esi
+ xor 32(%rsp),%eax
+ xor %r11d,%ebx
+ rol $5,%esi
+ xor 56(%rsp),%eax
+ xor %edx,%ebx
+ add %esi,%edi
+ xor 12(%rsp),%eax
+ rol $30,%r11d
+ add %ebx,%edi
+ rol $1,%eax
+ mov %eax,24(%rsp)
+ lea -0x359d3e2a(%eax,%edx),%esi
+ mov 28(%rsp),%eax
+ mov %r11d,%ebx
+ mov %edi,%edx
+ xor 36(%rsp),%eax
+ xor %ebp,%ebx
+ rol $5,%edx
+ xor 60(%rsp),%eax
+ xor %r12d,%ebx
+ add %edx,%esi
+ xor 16(%rsp),%eax
+ rol $30,%ebp
+ add %ebx,%esi
+ rol $1,%eax
+ mov %eax,28(%rsp)
+ lea -0x359d3e2a(%eax,%r12d),%edx
+ mov 32(%rsp),%eax
+ mov %ebp,%ebx
+ mov %esi,%r12d
+ xor 40(%rsp),%eax
+ xor %edi,%ebx
+ rol $5,%r12d
+ xor 0(%rsp),%eax
+ xor %r11d,%ebx
+ add %r12d,%edx
+ xor 20(%rsp),%eax
+ rol $30,%edi
+ add %ebx,%edx
+ rol $1,%eax
+ mov %eax,32(%rsp)
+ lea -0x359d3e2a(%eax,%r11d),%r12d
+ mov 36(%rsp),%eax
+ mov %edi,%ebx
+ mov %edx,%r11d
+ xor 44(%rsp),%eax
+ xor %esi,%ebx
+ rol $5,%r11d
+ xor 4(%rsp),%eax
+ xor %ebp,%ebx
+ add %r11d,%r12d
+ xor 24(%rsp),%eax
+ rol $30,%esi
+ add %ebx,%r12d
+ rol $1,%eax
+ mov %eax,36(%rsp)
+ lea -0x359d3e2a(%eax,%ebp),%r11d
+ mov 40(%rsp),%eax
+ mov %esi,%ebx
+ mov %r12d,%ebp
+ xor 48(%rsp),%eax
+ xor %edx,%ebx
+ rol $5,%ebp
+ xor 8(%rsp),%eax
+ xor %edi,%ebx
+ add %ebp,%r11d
+ xor 28(%rsp),%eax
+ rol $30,%edx
+ add %ebx,%r11d
+ rol $1,%eax
+ mov %eax,40(%rsp)
+ lea -0x359d3e2a(%eax,%edi),%ebp
+ mov 44(%rsp),%eax
+ mov %edx,%ebx
+ mov %r11d,%edi
+ xor 52(%rsp),%eax
+ xor %r12d,%ebx
+ rol $5,%edi
+ xor 12(%rsp),%eax
+ xor %esi,%ebx
+ add %edi,%ebp
+ xor 32(%rsp),%eax
+ rol $30,%r12d
+ add %ebx,%ebp
+ rol $1,%eax
+ mov %eax,44(%rsp)
+ lea -0x359d3e2a(%eax,%esi),%edi
+ mov 48(%rsp),%eax
+ mov %r12d,%ebx
+ mov %ebp,%esi
+ xor 56(%rsp),%eax
+ xor %r11d,%ebx
+ rol $5,%esi
+ xor 16(%rsp),%eax
+ xor %edx,%ebx
+ add %esi,%edi
+ xor 36(%rsp),%eax
+ rol $30,%r11d
+ add %ebx,%edi
+ rol $1,%eax
+ mov %eax,48(%rsp)
+ lea -0x359d3e2a(%eax,%edx),%esi
+ mov 52(%rsp),%eax
+ mov %r11d,%ebx
+ mov %edi,%edx
+ xor 60(%rsp),%eax
+ xor %ebp,%ebx
+ rol $5,%edx
+ xor 20(%rsp),%eax
+ xor %r12d,%ebx
+ add %edx,%esi
+ xor 40(%rsp),%eax
+ rol $30,%ebp
+ add %ebx,%esi
+ rol $1,%eax
+ lea -0x359d3e2a(%eax,%r12d),%edx
+ mov 56(%rsp),%eax
+ mov %ebp,%ebx
+ mov %esi,%r12d
+ xor 0(%rsp),%eax
+ xor %edi,%ebx
+ rol $5,%r12d
+ xor 24(%rsp),%eax
+ xor %r11d,%ebx
+ add %r12d,%edx
+ xor 44(%rsp),%eax
+ rol $30,%edi
+ add %ebx,%edx
+ rol $1,%eax
+ lea -0x359d3e2a(%eax,%r11d),%r12d
+ mov 60(%rsp),%eax
+ mov %edi,%ebx
+ mov %edx,%r11d
+ xor 4(%rsp),%eax
+ xor %esi,%ebx
+ rol $5,%r11d
+ xor 28(%rsp),%eax
+ xor %ebp,%ebx
+ add %r11d,%r12d
+ xor 48(%rsp),%eax
+ rol $30,%esi
+ add %ebx,%r12d
+ rol $1,%eax
+ lea -0x359d3e2a(%eax,%ebp),%r11d
+ mov %esi,%ebx
+ mov %r12d,%ebp
+ xor %edx,%ebx
+ rol $5,%ebp
+ xor %edi,%ebx
+ add %ebp,%r11d
+ rol $30,%edx
+ add %ebx,%r11d
+ // Update and save state information in SHA-1 context
+ add 0(%r8),%r11d
+ add 4(%r8),%r12d
+ add 8(%r8),%edx
+ add 12(%r8),%esi
+ add 16(%r8),%edi
+ mov %r11d,0(%r8)
+ mov %r12d,4(%r8)
+ mov %edx,8(%r8)
+ mov %esi,12(%r8)
+ mov %edi,16(%r8)
+
+ xchg %r11d,%edx # mov %r11d,%edx
+ xchg %r12d,%esi # mov %r12d,%esi
+ xchg %r11d,%edi # mov %edx,%edi
+ xchg %r12d,%ebp # mov %esi,%ebp
+ # mov %edi,%r11d
+ lea 64(%r9),%r9
+ sub $1,%r10
+ jnz .Lloop
+ mov 64(%rsp),%rsp
+ pop %r12
+ pop %rbp
+ pop %rbx
+ ret
+SET_SIZE(sha1_block_data_order)
+.asciz "SHA1 block transform for x86_64, CRYPTOGAMS by <appro@openssl.org>"
+
+#endif /* lint || __lint */
--- /dev/null
+/*
+ * ====================================================================
+ * Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
+ * project. Rights for redistribution and usage in source and binary
+ * forms are granted according to the OpenSSL license.
+ * ====================================================================
+ *
+ * sha256/512_block procedure for x86_64.
+ *
+ * 40% improvement over compiler-generated code on Opteron. On EM64T
+ * sha256 was observed to run >80% faster and sha512 - >40%. No magical
+ * tricks, just straight implementation... I really wonder why gcc
+ * [being armed with inline assembler] fails to generate as fast code.
+ * The only thing which is cool about this module is that it's very
+ * same instruction sequence used for both SHA-256 and SHA-512. In
+ * former case the instructions operate on 32-bit operands, while in
+ * latter - on 64-bit ones. All I had to do is to get one flavor right,
+ * the other one passed the test right away:-)
+ *
+ * sha256_block runs in ~1005 cycles on Opteron, which gives you
+ * asymptotic performance of 64*1000/1005=63.7MBps times CPU clock
+ * frequency in GHz. sha512_block runs in ~1275 cycles, which results
+ * in 128*1000/1275=100MBps per GHz. Is there room for improvement?
+ * Well, if you compare it to IA-64 implementation, which maintains
+ * X[16] in register bank[!], tends to 4 instructions per CPU clock
+ * cycle and runs in 1003 cycles, 1275 is very good result for 3-way
+ * issue Opteron pipeline and X[16] maintained in memory. So that *if*
+ * there is a way to improve it, *then* the only way would be to try to
+ * offload X[16] updates to SSE unit, but that would require "deeper"
+ * loop unroll, which in turn would naturally cause size blow-up, not
+ * to mention increased complexity! And once again, only *if* it's
+ * actually possible to noticeably improve overall ILP, instruction
+ * level parallelism, on a given CPU implementation in this case.
+ *
+ * Special note on Intel EM64T. While Opteron CPU exhibits perfect
+ * perfromance ratio of 1.5 between 64- and 32-bit flavors [see above],
+ * [currently available] EM64T CPUs apparently are far from it. On the
+ * contrary, 64-bit version, sha512_block, is ~30% *slower* than 32-bit
+ * sha256_block:-( This is presumably because 64-bit shifts/rotates
+ * apparently are not atomic instructions, but implemented in microcode.
+ */
+
+/*
+ * OpenSolaris OS modifications
+ *
+ * Sun elects to use this software under the BSD license.
+ *
+ * This source originates from OpenSSL file sha512-x86_64.pl at
+ * ftp://ftp.openssl.org/snapshot/openssl-0.9.8-stable-SNAP-20080131.tar.gz
+ * (presumably for future OpenSSL release 0.9.8h), with these changes:
+ *
+ * 1. Added perl "use strict" and declared variables.
+ *
+ * 2. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
+ * /usr/include/sys/asm_linkage.h, .ident keywords, and lint(1B) guards.
+ *
+ * 3. Removed x86_64-xlate.pl script (not needed for as(1) or gas(1)
+ * assemblers). Replaced the .picmeup macro with assembler code.
+ *
+ * 4. Added 8 to $ctx, as OpenSolaris OS has an extra 4-byte field, "algotype",
+ * at the beginning of SHA2_CTX (the next field is 8-byte aligned).
+ */
+
+/*
+ * This file was generated by a perl script (sha512-x86_64.pl) that could
+ * be used to generate sha256 and sha512 variants from the same code base.
+ * For our purposes, we only need sha256 and so getting the perl script to
+ * run as part of the build process seemed superfluous. The comments from
+ * the original file have been pasted above.
+ */
+
+#if defined(lint) || defined(__lint)
+#include <sys/stdint.h>
+#include <sha2/sha2.h>
+
+/* ARGSUSED */
+void
+SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num)
+{
+}
+
+
+#else
+#define _ASM
+#include <sys/asm_linkage.h>
+
+ENTRY_NP(SHA256TransformBlocks)
+ push %rbx
+ push %rbp
+ push %r12
+ push %r13
+ push %r14
+ push %r15
+ mov %rsp,%rbp # copy %rsp
+ shl $4,%rdx # num*16
+ sub $16*4+4*8,%rsp
+ lea (%rsi,%rdx,4),%rdx # inp+num*16*4
+ and $-64,%rsp # align stack frame
+ add $8,%rdi # Skip OpenSolaris field, "algotype"
+ mov %rdi,16*4+0*8(%rsp) # save ctx, 1st arg
+ mov %rsi,16*4+1*8(%rsp) # save inp, 2nd arg
+ mov %rdx,16*4+2*8(%rsp) # save end pointer, "3rd" arg
+ mov %rbp,16*4+3*8(%rsp) # save copy of %rsp
+
+ /.picmeup %rbp
+ / The .picmeup pseudo-directive, from perlasm/x86_64_xlate.pl, puts
+ / the address of the "next" instruction into the target register
+ / (%rbp). This generates these 2 instructions:
+ lea .Llea(%rip),%rbp
+ /nop / .picmeup generates a nop for mod 8 alignment--not needed here
+
+.Llea:
+ lea K256-.(%rbp),%rbp
+
+ mov 4*0(%rdi),%eax
+ mov 4*1(%rdi),%ebx
+ mov 4*2(%rdi),%ecx
+ mov 4*3(%rdi),%edx
+ mov 4*4(%rdi),%r8d
+ mov 4*5(%rdi),%r9d
+ mov 4*6(%rdi),%r10d
+ mov 4*7(%rdi),%r11d
+ jmp .Lloop
+
+.align 16
+.Lloop:
+ xor %rdi,%rdi
+ mov 4*0(%rsi),%r12d
+ bswap %r12d
+ mov %r8d,%r13d
+ mov %r8d,%r14d
+ mov %r9d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r10d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r8d,%r15d # (f^g)&e
+ mov %r12d,0(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r10d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r11d,%r12d # T1+=h
+
+ mov %eax,%r11d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %eax,%r13d
+ mov %eax,%r14d
+
+ ror $2,%r11d
+ ror $13,%r13d
+ mov %eax,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r11d
+ ror $9,%r13d
+ or %ecx,%r14d # a|c
+
+ xor %r13d,%r11d # h=Sigma0(a)
+ and %ecx,%r15d # a&c
+ add %r12d,%edx # d+=T1
+
+ and %ebx,%r14d # (a|c)&b
+ add %r12d,%r11d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r11d # h+=Maj(a,b,c)
+ mov 4*1(%rsi),%r12d
+ bswap %r12d
+ mov %edx,%r13d
+ mov %edx,%r14d
+ mov %r8d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r9d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %edx,%r15d # (f^g)&e
+ mov %r12d,4(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r9d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r10d,%r12d # T1+=h
+
+ mov %r11d,%r10d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r11d,%r13d
+ mov %r11d,%r14d
+
+ ror $2,%r10d
+ ror $13,%r13d
+ mov %r11d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r10d
+ ror $9,%r13d
+ or %ebx,%r14d # a|c
+
+ xor %r13d,%r10d # h=Sigma0(a)
+ and %ebx,%r15d # a&c
+ add %r12d,%ecx # d+=T1
+
+ and %eax,%r14d # (a|c)&b
+ add %r12d,%r10d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r10d # h+=Maj(a,b,c)
+ mov 4*2(%rsi),%r12d
+ bswap %r12d
+ mov %ecx,%r13d
+ mov %ecx,%r14d
+ mov %edx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r8d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %ecx,%r15d # (f^g)&e
+ mov %r12d,8(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r8d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r9d,%r12d # T1+=h
+
+ mov %r10d,%r9d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r10d,%r13d
+ mov %r10d,%r14d
+
+ ror $2,%r9d
+ ror $13,%r13d
+ mov %r10d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r9d
+ ror $9,%r13d
+ or %eax,%r14d # a|c
+
+ xor %r13d,%r9d # h=Sigma0(a)
+ and %eax,%r15d # a&c
+ add %r12d,%ebx # d+=T1
+
+ and %r11d,%r14d # (a|c)&b
+ add %r12d,%r9d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r9d # h+=Maj(a,b,c)
+ mov 4*3(%rsi),%r12d
+ bswap %r12d
+ mov %ebx,%r13d
+ mov %ebx,%r14d
+ mov %ecx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %edx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %ebx,%r15d # (f^g)&e
+ mov %r12d,12(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %edx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r8d,%r12d # T1+=h
+
+ mov %r9d,%r8d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r9d,%r13d
+ mov %r9d,%r14d
+
+ ror $2,%r8d
+ ror $13,%r13d
+ mov %r9d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r8d
+ ror $9,%r13d
+ or %r11d,%r14d # a|c
+
+ xor %r13d,%r8d # h=Sigma0(a)
+ and %r11d,%r15d # a&c
+ add %r12d,%eax # d+=T1
+
+ and %r10d,%r14d # (a|c)&b
+ add %r12d,%r8d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r8d # h+=Maj(a,b,c)
+ mov 4*4(%rsi),%r12d
+ bswap %r12d
+ mov %eax,%r13d
+ mov %eax,%r14d
+ mov %ebx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %ecx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %eax,%r15d # (f^g)&e
+ mov %r12d,16(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %ecx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %edx,%r12d # T1+=h
+
+ mov %r8d,%edx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r8d,%r13d
+ mov %r8d,%r14d
+
+ ror $2,%edx
+ ror $13,%r13d
+ mov %r8d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%edx
+ ror $9,%r13d
+ or %r10d,%r14d # a|c
+
+ xor %r13d,%edx # h=Sigma0(a)
+ and %r10d,%r15d # a&c
+ add %r12d,%r11d # d+=T1
+
+ and %r9d,%r14d # (a|c)&b
+ add %r12d,%edx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%edx # h+=Maj(a,b,c)
+ mov 4*5(%rsi),%r12d
+ bswap %r12d
+ mov %r11d,%r13d
+ mov %r11d,%r14d
+ mov %eax,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %ebx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r11d,%r15d # (f^g)&e
+ mov %r12d,20(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %ebx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %ecx,%r12d # T1+=h
+
+ mov %edx,%ecx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %edx,%r13d
+ mov %edx,%r14d
+
+ ror $2,%ecx
+ ror $13,%r13d
+ mov %edx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%ecx
+ ror $9,%r13d
+ or %r9d,%r14d # a|c
+
+ xor %r13d,%ecx # h=Sigma0(a)
+ and %r9d,%r15d # a&c
+ add %r12d,%r10d # d+=T1
+
+ and %r8d,%r14d # (a|c)&b
+ add %r12d,%ecx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%ecx # h+=Maj(a,b,c)
+ mov 4*6(%rsi),%r12d
+ bswap %r12d
+ mov %r10d,%r13d
+ mov %r10d,%r14d
+ mov %r11d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %eax,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r10d,%r15d # (f^g)&e
+ mov %r12d,24(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %eax,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %ebx,%r12d # T1+=h
+
+ mov %ecx,%ebx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %ecx,%r13d
+ mov %ecx,%r14d
+
+ ror $2,%ebx
+ ror $13,%r13d
+ mov %ecx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%ebx
+ ror $9,%r13d
+ or %r8d,%r14d # a|c
+
+ xor %r13d,%ebx # h=Sigma0(a)
+ and %r8d,%r15d # a&c
+ add %r12d,%r9d # d+=T1
+
+ and %edx,%r14d # (a|c)&b
+ add %r12d,%ebx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%ebx # h+=Maj(a,b,c)
+ mov 4*7(%rsi),%r12d
+ bswap %r12d
+ mov %r9d,%r13d
+ mov %r9d,%r14d
+ mov %r10d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r11d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r9d,%r15d # (f^g)&e
+ mov %r12d,28(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r11d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %eax,%r12d # T1+=h
+
+ mov %ebx,%eax
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %ebx,%r13d
+ mov %ebx,%r14d
+
+ ror $2,%eax
+ ror $13,%r13d
+ mov %ebx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%eax
+ ror $9,%r13d
+ or %edx,%r14d # a|c
+
+ xor %r13d,%eax # h=Sigma0(a)
+ and %edx,%r15d # a&c
+ add %r12d,%r8d # d+=T1
+
+ and %ecx,%r14d # (a|c)&b
+ add %r12d,%eax # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%eax # h+=Maj(a,b,c)
+ mov 4*8(%rsi),%r12d
+ bswap %r12d
+ mov %r8d,%r13d
+ mov %r8d,%r14d
+ mov %r9d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r10d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r8d,%r15d # (f^g)&e
+ mov %r12d,32(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r10d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r11d,%r12d # T1+=h
+
+ mov %eax,%r11d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %eax,%r13d
+ mov %eax,%r14d
+
+ ror $2,%r11d
+ ror $13,%r13d
+ mov %eax,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r11d
+ ror $9,%r13d
+ or %ecx,%r14d # a|c
+
+ xor %r13d,%r11d # h=Sigma0(a)
+ and %ecx,%r15d # a&c
+ add %r12d,%edx # d+=T1
+
+ and %ebx,%r14d # (a|c)&b
+ add %r12d,%r11d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r11d # h+=Maj(a,b,c)
+ mov 4*9(%rsi),%r12d
+ bswap %r12d
+ mov %edx,%r13d
+ mov %edx,%r14d
+ mov %r8d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r9d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %edx,%r15d # (f^g)&e
+ mov %r12d,36(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r9d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r10d,%r12d # T1+=h
+
+ mov %r11d,%r10d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r11d,%r13d
+ mov %r11d,%r14d
+
+ ror $2,%r10d
+ ror $13,%r13d
+ mov %r11d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r10d
+ ror $9,%r13d
+ or %ebx,%r14d # a|c
+
+ xor %r13d,%r10d # h=Sigma0(a)
+ and %ebx,%r15d # a&c
+ add %r12d,%ecx # d+=T1
+
+ and %eax,%r14d # (a|c)&b
+ add %r12d,%r10d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r10d # h+=Maj(a,b,c)
+ mov 4*10(%rsi),%r12d
+ bswap %r12d
+ mov %ecx,%r13d
+ mov %ecx,%r14d
+ mov %edx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r8d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %ecx,%r15d # (f^g)&e
+ mov %r12d,40(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r8d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r9d,%r12d # T1+=h
+
+ mov %r10d,%r9d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r10d,%r13d
+ mov %r10d,%r14d
+
+ ror $2,%r9d
+ ror $13,%r13d
+ mov %r10d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r9d
+ ror $9,%r13d
+ or %eax,%r14d # a|c
+
+ xor %r13d,%r9d # h=Sigma0(a)
+ and %eax,%r15d # a&c
+ add %r12d,%ebx # d+=T1
+
+ and %r11d,%r14d # (a|c)&b
+ add %r12d,%r9d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r9d # h+=Maj(a,b,c)
+ mov 4*11(%rsi),%r12d
+ bswap %r12d
+ mov %ebx,%r13d
+ mov %ebx,%r14d
+ mov %ecx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %edx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %ebx,%r15d # (f^g)&e
+ mov %r12d,44(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %edx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r8d,%r12d # T1+=h
+
+ mov %r9d,%r8d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r9d,%r13d
+ mov %r9d,%r14d
+
+ ror $2,%r8d
+ ror $13,%r13d
+ mov %r9d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r8d
+ ror $9,%r13d
+ or %r11d,%r14d # a|c
+
+ xor %r13d,%r8d # h=Sigma0(a)
+ and %r11d,%r15d # a&c
+ add %r12d,%eax # d+=T1
+
+ and %r10d,%r14d # (a|c)&b
+ add %r12d,%r8d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r8d # h+=Maj(a,b,c)
+ mov 4*12(%rsi),%r12d
+ bswap %r12d
+ mov %eax,%r13d
+ mov %eax,%r14d
+ mov %ebx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %ecx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %eax,%r15d # (f^g)&e
+ mov %r12d,48(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %ecx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %edx,%r12d # T1+=h
+
+ mov %r8d,%edx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r8d,%r13d
+ mov %r8d,%r14d
+
+ ror $2,%edx
+ ror $13,%r13d
+ mov %r8d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%edx
+ ror $9,%r13d
+ or %r10d,%r14d # a|c
+
+ xor %r13d,%edx # h=Sigma0(a)
+ and %r10d,%r15d # a&c
+ add %r12d,%r11d # d+=T1
+
+ and %r9d,%r14d # (a|c)&b
+ add %r12d,%edx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%edx # h+=Maj(a,b,c)
+ mov 4*13(%rsi),%r12d
+ bswap %r12d
+ mov %r11d,%r13d
+ mov %r11d,%r14d
+ mov %eax,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %ebx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r11d,%r15d # (f^g)&e
+ mov %r12d,52(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %ebx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %ecx,%r12d # T1+=h
+
+ mov %edx,%ecx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %edx,%r13d
+ mov %edx,%r14d
+
+ ror $2,%ecx
+ ror $13,%r13d
+ mov %edx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%ecx
+ ror $9,%r13d
+ or %r9d,%r14d # a|c
+
+ xor %r13d,%ecx # h=Sigma0(a)
+ and %r9d,%r15d # a&c
+ add %r12d,%r10d # d+=T1
+
+ and %r8d,%r14d # (a|c)&b
+ add %r12d,%ecx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%ecx # h+=Maj(a,b,c)
+ mov 4*14(%rsi),%r12d
+ bswap %r12d
+ mov %r10d,%r13d
+ mov %r10d,%r14d
+ mov %r11d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %eax,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r10d,%r15d # (f^g)&e
+ mov %r12d,56(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %eax,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %ebx,%r12d # T1+=h
+
+ mov %ecx,%ebx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %ecx,%r13d
+ mov %ecx,%r14d
+
+ ror $2,%ebx
+ ror $13,%r13d
+ mov %ecx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%ebx
+ ror $9,%r13d
+ or %r8d,%r14d # a|c
+
+ xor %r13d,%ebx # h=Sigma0(a)
+ and %r8d,%r15d # a&c
+ add %r12d,%r9d # d+=T1
+
+ and %edx,%r14d # (a|c)&b
+ add %r12d,%ebx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%ebx # h+=Maj(a,b,c)
+ mov 4*15(%rsi),%r12d
+ bswap %r12d
+ mov %r9d,%r13d
+ mov %r9d,%r14d
+ mov %r10d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r11d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r9d,%r15d # (f^g)&e
+ mov %r12d,60(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r11d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %eax,%r12d # T1+=h
+
+ mov %ebx,%eax
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %ebx,%r13d
+ mov %ebx,%r14d
+
+ ror $2,%eax
+ ror $13,%r13d
+ mov %ebx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%eax
+ ror $9,%r13d
+ or %edx,%r14d # a|c
+
+ xor %r13d,%eax # h=Sigma0(a)
+ and %edx,%r15d # a&c
+ add %r12d,%r8d # d+=T1
+
+ and %ecx,%r14d # (a|c)&b
+ add %r12d,%eax # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%eax # h+=Maj(a,b,c)
+ jmp .Lrounds_16_xx
+.align 16
+.Lrounds_16_xx:
+ mov 4(%rsp),%r13d
+ mov 56(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 36(%rsp),%r12d
+
+ add 0(%rsp),%r12d
+ mov %r8d,%r13d
+ mov %r8d,%r14d
+ mov %r9d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r10d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r8d,%r15d # (f^g)&e
+ mov %r12d,0(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r10d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r11d,%r12d # T1+=h
+
+ mov %eax,%r11d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %eax,%r13d
+ mov %eax,%r14d
+
+ ror $2,%r11d
+ ror $13,%r13d
+ mov %eax,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r11d
+ ror $9,%r13d
+ or %ecx,%r14d # a|c
+
+ xor %r13d,%r11d # h=Sigma0(a)
+ and %ecx,%r15d # a&c
+ add %r12d,%edx # d+=T1
+
+ and %ebx,%r14d # (a|c)&b
+ add %r12d,%r11d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r11d # h+=Maj(a,b,c)
+ mov 8(%rsp),%r13d
+ mov 60(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 40(%rsp),%r12d
+
+ add 4(%rsp),%r12d
+ mov %edx,%r13d
+ mov %edx,%r14d
+ mov %r8d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r9d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %edx,%r15d # (f^g)&e
+ mov %r12d,4(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r9d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r10d,%r12d # T1+=h
+
+ mov %r11d,%r10d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r11d,%r13d
+ mov %r11d,%r14d
+
+ ror $2,%r10d
+ ror $13,%r13d
+ mov %r11d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r10d
+ ror $9,%r13d
+ or %ebx,%r14d # a|c
+
+ xor %r13d,%r10d # h=Sigma0(a)
+ and %ebx,%r15d # a&c
+ add %r12d,%ecx # d+=T1
+
+ and %eax,%r14d # (a|c)&b
+ add %r12d,%r10d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r10d # h+=Maj(a,b,c)
+ mov 12(%rsp),%r13d
+ mov 0(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 44(%rsp),%r12d
+
+ add 8(%rsp),%r12d
+ mov %ecx,%r13d
+ mov %ecx,%r14d
+ mov %edx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r8d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %ecx,%r15d # (f^g)&e
+ mov %r12d,8(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r8d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r9d,%r12d # T1+=h
+
+ mov %r10d,%r9d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r10d,%r13d
+ mov %r10d,%r14d
+
+ ror $2,%r9d
+ ror $13,%r13d
+ mov %r10d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r9d
+ ror $9,%r13d
+ or %eax,%r14d # a|c
+
+ xor %r13d,%r9d # h=Sigma0(a)
+ and %eax,%r15d # a&c
+ add %r12d,%ebx # d+=T1
+
+ and %r11d,%r14d # (a|c)&b
+ add %r12d,%r9d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r9d # h+=Maj(a,b,c)
+ mov 16(%rsp),%r13d
+ mov 4(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 48(%rsp),%r12d
+
+ add 12(%rsp),%r12d
+ mov %ebx,%r13d
+ mov %ebx,%r14d
+ mov %ecx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %edx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %ebx,%r15d # (f^g)&e
+ mov %r12d,12(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %edx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r8d,%r12d # T1+=h
+
+ mov %r9d,%r8d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r9d,%r13d
+ mov %r9d,%r14d
+
+ ror $2,%r8d
+ ror $13,%r13d
+ mov %r9d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r8d
+ ror $9,%r13d
+ or %r11d,%r14d # a|c
+
+ xor %r13d,%r8d # h=Sigma0(a)
+ and %r11d,%r15d # a&c
+ add %r12d,%eax # d+=T1
+
+ and %r10d,%r14d # (a|c)&b
+ add %r12d,%r8d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r8d # h+=Maj(a,b,c)
+ mov 20(%rsp),%r13d
+ mov 8(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 52(%rsp),%r12d
+
+ add 16(%rsp),%r12d
+ mov %eax,%r13d
+ mov %eax,%r14d
+ mov %ebx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %ecx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %eax,%r15d # (f^g)&e
+ mov %r12d,16(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %ecx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %edx,%r12d # T1+=h
+
+ mov %r8d,%edx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r8d,%r13d
+ mov %r8d,%r14d
+
+ ror $2,%edx
+ ror $13,%r13d
+ mov %r8d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%edx
+ ror $9,%r13d
+ or %r10d,%r14d # a|c
+
+ xor %r13d,%edx # h=Sigma0(a)
+ and %r10d,%r15d # a&c
+ add %r12d,%r11d # d+=T1
+
+ and %r9d,%r14d # (a|c)&b
+ add %r12d,%edx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%edx # h+=Maj(a,b,c)
+ mov 24(%rsp),%r13d
+ mov 12(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 56(%rsp),%r12d
+
+ add 20(%rsp),%r12d
+ mov %r11d,%r13d
+ mov %r11d,%r14d
+ mov %eax,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %ebx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r11d,%r15d # (f^g)&e
+ mov %r12d,20(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %ebx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %ecx,%r12d # T1+=h
+
+ mov %edx,%ecx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %edx,%r13d
+ mov %edx,%r14d
+
+ ror $2,%ecx
+ ror $13,%r13d
+ mov %edx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%ecx
+ ror $9,%r13d
+ or %r9d,%r14d # a|c
+
+ xor %r13d,%ecx # h=Sigma0(a)
+ and %r9d,%r15d # a&c
+ add %r12d,%r10d # d+=T1
+
+ and %r8d,%r14d # (a|c)&b
+ add %r12d,%ecx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%ecx # h+=Maj(a,b,c)
+ mov 28(%rsp),%r13d
+ mov 16(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 60(%rsp),%r12d
+
+ add 24(%rsp),%r12d
+ mov %r10d,%r13d
+ mov %r10d,%r14d
+ mov %r11d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %eax,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r10d,%r15d # (f^g)&e
+ mov %r12d,24(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %eax,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %ebx,%r12d # T1+=h
+
+ mov %ecx,%ebx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %ecx,%r13d
+ mov %ecx,%r14d
+
+ ror $2,%ebx
+ ror $13,%r13d
+ mov %ecx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%ebx
+ ror $9,%r13d
+ or %r8d,%r14d # a|c
+
+ xor %r13d,%ebx # h=Sigma0(a)
+ and %r8d,%r15d # a&c
+ add %r12d,%r9d # d+=T1
+
+ and %edx,%r14d # (a|c)&b
+ add %r12d,%ebx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%ebx # h+=Maj(a,b,c)
+ mov 32(%rsp),%r13d
+ mov 20(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 0(%rsp),%r12d
+
+ add 28(%rsp),%r12d
+ mov %r9d,%r13d
+ mov %r9d,%r14d
+ mov %r10d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r11d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r9d,%r15d # (f^g)&e
+ mov %r12d,28(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r11d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %eax,%r12d # T1+=h
+
+ mov %ebx,%eax
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %ebx,%r13d
+ mov %ebx,%r14d
+
+ ror $2,%eax
+ ror $13,%r13d
+ mov %ebx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%eax
+ ror $9,%r13d
+ or %edx,%r14d # a|c
+
+ xor %r13d,%eax # h=Sigma0(a)
+ and %edx,%r15d # a&c
+ add %r12d,%r8d # d+=T1
+
+ and %ecx,%r14d # (a|c)&b
+ add %r12d,%eax # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%eax # h+=Maj(a,b,c)
+ mov 36(%rsp),%r13d
+ mov 24(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 4(%rsp),%r12d
+
+ add 32(%rsp),%r12d
+ mov %r8d,%r13d
+ mov %r8d,%r14d
+ mov %r9d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r10d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r8d,%r15d # (f^g)&e
+ mov %r12d,32(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r10d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r11d,%r12d # T1+=h
+
+ mov %eax,%r11d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %eax,%r13d
+ mov %eax,%r14d
+
+ ror $2,%r11d
+ ror $13,%r13d
+ mov %eax,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r11d
+ ror $9,%r13d
+ or %ecx,%r14d # a|c
+
+ xor %r13d,%r11d # h=Sigma0(a)
+ and %ecx,%r15d # a&c
+ add %r12d,%edx # d+=T1
+
+ and %ebx,%r14d # (a|c)&b
+ add %r12d,%r11d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r11d # h+=Maj(a,b,c)
+ mov 40(%rsp),%r13d
+ mov 28(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 8(%rsp),%r12d
+
+ add 36(%rsp),%r12d
+ mov %edx,%r13d
+ mov %edx,%r14d
+ mov %r8d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r9d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %edx,%r15d # (f^g)&e
+ mov %r12d,36(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r9d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r10d,%r12d # T1+=h
+
+ mov %r11d,%r10d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r11d,%r13d
+ mov %r11d,%r14d
+
+ ror $2,%r10d
+ ror $13,%r13d
+ mov %r11d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r10d
+ ror $9,%r13d
+ or %ebx,%r14d # a|c
+
+ xor %r13d,%r10d # h=Sigma0(a)
+ and %ebx,%r15d # a&c
+ add %r12d,%ecx # d+=T1
+
+ and %eax,%r14d # (a|c)&b
+ add %r12d,%r10d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r10d # h+=Maj(a,b,c)
+ mov 44(%rsp),%r13d
+ mov 32(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 12(%rsp),%r12d
+
+ add 40(%rsp),%r12d
+ mov %ecx,%r13d
+ mov %ecx,%r14d
+ mov %edx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r8d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %ecx,%r15d # (f^g)&e
+ mov %r12d,40(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r8d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r9d,%r12d # T1+=h
+
+ mov %r10d,%r9d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r10d,%r13d
+ mov %r10d,%r14d
+
+ ror $2,%r9d
+ ror $13,%r13d
+ mov %r10d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r9d
+ ror $9,%r13d
+ or %eax,%r14d # a|c
+
+ xor %r13d,%r9d # h=Sigma0(a)
+ and %eax,%r15d # a&c
+ add %r12d,%ebx # d+=T1
+
+ and %r11d,%r14d # (a|c)&b
+ add %r12d,%r9d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r9d # h+=Maj(a,b,c)
+ mov 48(%rsp),%r13d
+ mov 36(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 16(%rsp),%r12d
+
+ add 44(%rsp),%r12d
+ mov %ebx,%r13d
+ mov %ebx,%r14d
+ mov %ecx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %edx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %ebx,%r15d # (f^g)&e
+ mov %r12d,44(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %edx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %r8d,%r12d # T1+=h
+
+ mov %r9d,%r8d
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r9d,%r13d
+ mov %r9d,%r14d
+
+ ror $2,%r8d
+ ror $13,%r13d
+ mov %r9d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%r8d
+ ror $9,%r13d
+ or %r11d,%r14d # a|c
+
+ xor %r13d,%r8d # h=Sigma0(a)
+ and %r11d,%r15d # a&c
+ add %r12d,%eax # d+=T1
+
+ and %r10d,%r14d # (a|c)&b
+ add %r12d,%r8d # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%r8d # h+=Maj(a,b,c)
+ mov 52(%rsp),%r13d
+ mov 40(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 20(%rsp),%r12d
+
+ add 48(%rsp),%r12d
+ mov %eax,%r13d
+ mov %eax,%r14d
+ mov %ebx,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %ecx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %eax,%r15d # (f^g)&e
+ mov %r12d,48(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %ecx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %edx,%r12d # T1+=h
+
+ mov %r8d,%edx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %r8d,%r13d
+ mov %r8d,%r14d
+
+ ror $2,%edx
+ ror $13,%r13d
+ mov %r8d,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%edx
+ ror $9,%r13d
+ or %r10d,%r14d # a|c
+
+ xor %r13d,%edx # h=Sigma0(a)
+ and %r10d,%r15d # a&c
+ add %r12d,%r11d # d+=T1
+
+ and %r9d,%r14d # (a|c)&b
+ add %r12d,%edx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%edx # h+=Maj(a,b,c)
+ mov 56(%rsp),%r13d
+ mov 44(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 24(%rsp),%r12d
+
+ add 52(%rsp),%r12d
+ mov %r11d,%r13d
+ mov %r11d,%r14d
+ mov %eax,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %ebx,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r11d,%r15d # (f^g)&e
+ mov %r12d,52(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %ebx,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %ecx,%r12d # T1+=h
+
+ mov %edx,%ecx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %edx,%r13d
+ mov %edx,%r14d
+
+ ror $2,%ecx
+ ror $13,%r13d
+ mov %edx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%ecx
+ ror $9,%r13d
+ or %r9d,%r14d # a|c
+
+ xor %r13d,%ecx # h=Sigma0(a)
+ and %r9d,%r15d # a&c
+ add %r12d,%r10d # d+=T1
+
+ and %r8d,%r14d # (a|c)&b
+ add %r12d,%ecx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%ecx # h+=Maj(a,b,c)
+ mov 60(%rsp),%r13d
+ mov 48(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 28(%rsp),%r12d
+
+ add 56(%rsp),%r12d
+ mov %r10d,%r13d
+ mov %r10d,%r14d
+ mov %r11d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %eax,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r10d,%r15d # (f^g)&e
+ mov %r12d,56(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %eax,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %ebx,%r12d # T1+=h
+
+ mov %ecx,%ebx
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %ecx,%r13d
+ mov %ecx,%r14d
+
+ ror $2,%ebx
+ ror $13,%r13d
+ mov %ecx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%ebx
+ ror $9,%r13d
+ or %r8d,%r14d # a|c
+
+ xor %r13d,%ebx # h=Sigma0(a)
+ and %r8d,%r15d # a&c
+ add %r12d,%r9d # d+=T1
+
+ and %edx,%r14d # (a|c)&b
+ add %r12d,%ebx # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%ebx # h+=Maj(a,b,c)
+ mov 0(%rsp),%r13d
+ mov 52(%rsp),%r12d
+
+ mov %r13d,%r15d
+
+ shr $3,%r13d
+ ror $7,%r15d
+
+ xor %r15d,%r13d
+ ror $11,%r15d
+
+ xor %r15d,%r13d # sigma0(X[(i+1)&0xf])
+ mov %r12d,%r14d
+
+ shr $10,%r12d
+ ror $17,%r14d
+
+ xor %r14d,%r12d
+ ror $2,%r14d
+
+ xor %r14d,%r12d # sigma1(X[(i+14)&0xf])
+
+ add %r13d,%r12d
+
+ add 32(%rsp),%r12d
+
+ add 60(%rsp),%r12d
+ mov %r9d,%r13d
+ mov %r9d,%r14d
+ mov %r10d,%r15d
+
+ ror $6,%r13d
+ ror $11,%r14d
+ xor %r11d,%r15d # f^g
+
+ xor %r14d,%r13d
+ ror $14,%r14d
+ and %r9d,%r15d # (f^g)&e
+ mov %r12d,60(%rsp)
+
+ xor %r14d,%r13d # Sigma1(e)
+ xor %r11d,%r15d # Ch(e,f,g)=((f^g)&e)^g
+ add %eax,%r12d # T1+=h
+
+ mov %ebx,%eax
+ add %r13d,%r12d # T1+=Sigma1(e)
+
+ add %r15d,%r12d # T1+=Ch(e,f,g)
+ mov %ebx,%r13d
+ mov %ebx,%r14d
+
+ ror $2,%eax
+ ror $13,%r13d
+ mov %ebx,%r15d
+ add (%rbp,%rdi,4),%r12d # T1+=K[round]
+
+ xor %r13d,%eax
+ ror $9,%r13d
+ or %edx,%r14d # a|c
+
+ xor %r13d,%eax # h=Sigma0(a)
+ and %edx,%r15d # a&c
+ add %r12d,%r8d # d+=T1
+
+ and %ecx,%r14d # (a|c)&b
+ add %r12d,%eax # h+=T1
+
+ or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c)
+ lea 1(%rdi),%rdi # round++
+
+ add %r14d,%eax # h+=Maj(a,b,c)
+ cmp $64,%rdi
+ jb .Lrounds_16_xx
+
+ mov 16*4+0*8(%rsp),%rdi
+ lea 16*4(%rsi),%rsi
+
+ add 4*0(%rdi),%eax
+ add 4*1(%rdi),%ebx
+ add 4*2(%rdi),%ecx
+ add 4*3(%rdi),%edx
+ add 4*4(%rdi),%r8d
+ add 4*5(%rdi),%r9d
+ add 4*6(%rdi),%r10d
+ add 4*7(%rdi),%r11d
+
+ cmp 16*4+2*8(%rsp),%rsi
+
+ mov %eax,4*0(%rdi)
+ mov %ebx,4*1(%rdi)
+ mov %ecx,4*2(%rdi)
+ mov %edx,4*3(%rdi)
+ mov %r8d,4*4(%rdi)
+ mov %r9d,4*5(%rdi)
+ mov %r10d,4*6(%rdi)
+ mov %r11d,4*7(%rdi)
+ jb .Lloop
+
+ mov 16*4+3*8(%rsp),%rsp
+ pop %r15
+ pop %r14
+ pop %r13
+ pop %r12
+ pop %rbp
+ pop %rbx
+
+ ret
+SET_SIZE(SHA256TransformBlocks)
+
+.align 64
+.type K256,@object
+K256:
+ .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
+ .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
+ .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
+ .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
+ .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
+ .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
+ .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
+ .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
+ .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
+ .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
+ .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
+ .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
+ .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
+ .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
+ .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
+ .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
+#endif /* !lint && !__lint */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/sched_impl.h>
+
+static int kcf_emulate_dual(kcf_provider_desc_t *, crypto_ctx_t *,
+ kcf_req_params_t *);
+
+void
+kcf_free_triedlist(kcf_prov_tried_t *list)
+{
+ kcf_prov_tried_t *l;
+
+ while ((l = list) != NULL) {
+ list = list->pt_next;
+ KCF_PROV_REFRELE(l->pt_pd);
+ kmem_free(l, sizeof (kcf_prov_tried_t));
+ }
+}
+
+kcf_prov_tried_t *
+kcf_insert_triedlist(kcf_prov_tried_t **list, kcf_provider_desc_t *pd,
+ int kmflag)
+{
+ kcf_prov_tried_t *l;
+
+ l = kmem_alloc(sizeof (kcf_prov_tried_t), kmflag);
+ if (l == NULL)
+ return (NULL);
+
+ l->pt_pd = pd;
+ l->pt_next = *list;
+ *list = l;
+
+ return (l);
+}
+
+static boolean_t
+is_in_triedlist(kcf_provider_desc_t *pd, kcf_prov_tried_t *triedl)
+{
+ while (triedl != NULL) {
+ if (triedl->pt_pd == pd)
+ return (B_TRUE);
+ triedl = triedl->pt_next;
+ };
+
+ return (B_FALSE);
+}
+
+/*
+ * Search a mech entry's hardware provider list for the specified
+ * provider. Return true if found.
+ */
+static boolean_t
+is_valid_provider_for_mech(kcf_provider_desc_t *pd, kcf_mech_entry_t *me,
+ crypto_func_group_t fg)
+{
+ kcf_prov_mech_desc_t *prov_chain;
+
+ prov_chain = me->me_hw_prov_chain;
+ if (prov_chain != NULL) {
+ ASSERT(me->me_num_hwprov > 0);
+ for (; prov_chain != NULL; prov_chain = prov_chain->pm_next) {
+ if (prov_chain->pm_prov_desc == pd &&
+ IS_FG_SUPPORTED(prov_chain, fg)) {
+ return (B_TRUE);
+ }
+ }
+ }
+ return (B_FALSE);
+}
+
+/*
+ * This routine, given a logical provider, returns the least loaded
+ * provider belonging to the logical provider. The provider must be
+ * able to do the specified mechanism, i.e. check that the mechanism
+ * hasn't been disabled. In addition, just in case providers are not
+ * entirely equivalent, the provider's entry point is checked for
+ * non-nullness. This is accomplished by having the caller pass, as
+ * arguments, the offset of the function group (offset_1), and the
+ * offset of the function within the function group (offset_2).
+ * Returns NULL if no provider can be found.
+ */
+int
+kcf_get_hardware_provider(crypto_mech_type_t mech_type_1,
+ crypto_mech_type_t mech_type_2, boolean_t call_restrict,
+ kcf_provider_desc_t *old, kcf_provider_desc_t **new, crypto_func_group_t fg)
+{
+ kcf_provider_desc_t *provider, *real_pd = old;
+ kcf_provider_desc_t *gpd = NULL; /* good provider */
+ kcf_provider_desc_t *bpd = NULL; /* busy provider */
+ kcf_provider_list_t *p;
+ kcf_ops_class_t class;
+ kcf_mech_entry_t *me;
+ kcf_mech_entry_tab_t *me_tab;
+ int index, len, gqlen = INT_MAX, rv = CRYPTO_SUCCESS;
+
+ /* get the mech entry for the specified mechanism */
+ class = KCF_MECH2CLASS(mech_type_1);
+ if ((class < KCF_FIRST_OPSCLASS) || (class > KCF_LAST_OPSCLASS)) {
+ return (CRYPTO_MECHANISM_INVALID);
+ }
+
+ me_tab = &kcf_mech_tabs_tab[class];
+ index = KCF_MECH2INDEX(mech_type_1);
+ if ((index < 0) || (index >= me_tab->met_size)) {
+ return (CRYPTO_MECHANISM_INVALID);
+ }
+
+ me = &((me_tab->met_tab)[index]);
+ mutex_enter(&me->me_mutex);
+
+ /*
+ * We assume the provider descriptor will not go away because
+ * it is being held somewhere, i.e. its reference count has been
+ * incremented. In the case of the crypto module, the provider
+ * descriptor is held by the session structure.
+ */
+ if (old->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ if (old->pd_provider_list == NULL) {
+ real_pd = NULL;
+ rv = CRYPTO_DEVICE_ERROR;
+ goto out;
+ }
+ /*
+ * Find the least loaded real provider. KCF_PROV_LOAD gives
+ * the load (number of pending requests) of the provider.
+ */
+ mutex_enter(&old->pd_lock);
+ p = old->pd_provider_list;
+ while (p != NULL) {
+ provider = p->pl_provider;
+
+ ASSERT(provider->pd_prov_type !=
+ CRYPTO_LOGICAL_PROVIDER);
+
+ if (call_restrict &&
+ (provider->pd_flags & KCF_PROV_RESTRICTED)) {
+ p = p->pl_next;
+ continue;
+ }
+
+ if (!is_valid_provider_for_mech(provider, me, fg)) {
+ p = p->pl_next;
+ continue;
+ }
+
+ /* provider does second mech */
+ if (mech_type_2 != CRYPTO_MECH_INVALID) {
+ int i;
+
+ i = KCF_TO_PROV_MECH_INDX(provider,
+ mech_type_2);
+ if (i == KCF_INVALID_INDX) {
+ p = p->pl_next;
+ continue;
+ }
+ }
+
+ if (provider->pd_state != KCF_PROV_READY) {
+ /* choose BUSY if no READY providers */
+ if (provider->pd_state == KCF_PROV_BUSY)
+ bpd = provider;
+ p = p->pl_next;
+ continue;
+ }
+
+ len = KCF_PROV_LOAD(provider);
+ if (len < gqlen) {
+ gqlen = len;
+ gpd = provider;
+ }
+
+ p = p->pl_next;
+ }
+
+ if (gpd != NULL) {
+ real_pd = gpd;
+ KCF_PROV_REFHOLD(real_pd);
+ } else if (bpd != NULL) {
+ real_pd = bpd;
+ KCF_PROV_REFHOLD(real_pd);
+ } else {
+ /* can't find provider */
+ real_pd = NULL;
+ rv = CRYPTO_MECHANISM_INVALID;
+ }
+ mutex_exit(&old->pd_lock);
+
+ } else {
+ if (!KCF_IS_PROV_USABLE(old) ||
+ (call_restrict && (old->pd_flags & KCF_PROV_RESTRICTED))) {
+ real_pd = NULL;
+ rv = CRYPTO_DEVICE_ERROR;
+ goto out;
+ }
+
+ if (!is_valid_provider_for_mech(old, me, fg)) {
+ real_pd = NULL;
+ rv = CRYPTO_MECHANISM_INVALID;
+ goto out;
+ }
+
+ KCF_PROV_REFHOLD(real_pd);
+ }
+out:
+ mutex_exit(&me->me_mutex);
+ *new = real_pd;
+ return (rv);
+}
+
+/*
+ * Return the best provider for the specified mechanism. The provider
+ * is held and it is the caller's responsibility to release it when done.
+ * The fg input argument is used as a search criterion to pick a provider.
+ * A provider has to support this function group to be picked.
+ *
+ * Find the least loaded provider in the list of providers. We do a linear
+ * search to find one. This is fine as we assume there are only a few
+ * number of providers in this list. If this assumption ever changes,
+ * we should revisit this.
+ *
+ * call_restrict represents if the caller should not be allowed to
+ * use restricted providers.
+ */
+kcf_provider_desc_t *
+kcf_get_mech_provider(crypto_mech_type_t mech_type, kcf_mech_entry_t **mepp,
+ int *error, kcf_prov_tried_t *triedl, crypto_func_group_t fg,
+ boolean_t call_restrict, size_t data_size)
+{
+ kcf_provider_desc_t *pd = NULL, *gpd = NULL;
+ kcf_prov_mech_desc_t *prov_chain, *mdesc;
+ int len, gqlen = INT_MAX;
+ kcf_ops_class_t class;
+ int index;
+ kcf_mech_entry_t *me;
+ kcf_mech_entry_tab_t *me_tab;
+
+ class = KCF_MECH2CLASS(mech_type);
+ if ((class < KCF_FIRST_OPSCLASS) || (class > KCF_LAST_OPSCLASS)) {
+ *error = CRYPTO_MECHANISM_INVALID;
+ return (NULL);
+ }
+
+ me_tab = &kcf_mech_tabs_tab[class];
+ index = KCF_MECH2INDEX(mech_type);
+ if ((index < 0) || (index >= me_tab->met_size)) {
+ *error = CRYPTO_MECHANISM_INVALID;
+ return (NULL);
+ }
+
+ me = &((me_tab->met_tab)[index]);
+ if (mepp != NULL)
+ *mepp = me;
+
+ mutex_enter(&me->me_mutex);
+
+ prov_chain = me->me_hw_prov_chain;
+
+ /*
+ * We check for the threshhold for using a hardware provider for
+ * this amount of data. If there is no software provider available
+ * for the mechanism, then the threshold is ignored.
+ */
+ if ((prov_chain != NULL) &&
+ ((data_size == 0) || (me->me_threshold == 0) ||
+ (data_size >= me->me_threshold) ||
+ ((mdesc = me->me_sw_prov) == NULL) ||
+ (!IS_FG_SUPPORTED(mdesc, fg)) ||
+ (!KCF_IS_PROV_USABLE(mdesc->pm_prov_desc)))) {
+ ASSERT(me->me_num_hwprov > 0);
+ /* there is at least one provider */
+
+ /*
+ * Find the least loaded real provider. KCF_PROV_LOAD gives
+ * the load (number of pending requests) of the provider.
+ */
+ while (prov_chain != NULL) {
+ pd = prov_chain->pm_prov_desc;
+
+ if (!IS_FG_SUPPORTED(prov_chain, fg) ||
+ !KCF_IS_PROV_USABLE(pd) ||
+ IS_PROVIDER_TRIED(pd, triedl) ||
+ (call_restrict &&
+ (pd->pd_flags & KCF_PROV_RESTRICTED))) {
+ prov_chain = prov_chain->pm_next;
+ continue;
+ }
+
+ if ((len = KCF_PROV_LOAD(pd)) < gqlen) {
+ gqlen = len;
+ gpd = pd;
+ }
+
+ prov_chain = prov_chain->pm_next;
+ }
+
+ pd = gpd;
+ }
+
+ /* No HW provider for this mech, is there a SW provider? */
+ if (pd == NULL && (mdesc = me->me_sw_prov) != NULL) {
+ pd = mdesc->pm_prov_desc;
+ if (!IS_FG_SUPPORTED(mdesc, fg) ||
+ !KCF_IS_PROV_USABLE(pd) ||
+ IS_PROVIDER_TRIED(pd, triedl) ||
+ (call_restrict && (pd->pd_flags & KCF_PROV_RESTRICTED)))
+ pd = NULL;
+ }
+
+ if (pd == NULL) {
+ /*
+ * We do not want to report CRYPTO_MECH_NOT_SUPPORTED, when
+ * we are in the "fallback to the next provider" case. Rather
+ * we preserve the error, so that the client gets the right
+ * error code.
+ */
+ if (triedl == NULL)
+ *error = CRYPTO_MECH_NOT_SUPPORTED;
+ } else
+ KCF_PROV_REFHOLD(pd);
+
+ mutex_exit(&me->me_mutex);
+ return (pd);
+}
+
+/*
+ * Very similar to kcf_get_mech_provider(). Finds the best provider capable of
+ * a dual operation with both me1 and me2.
+ * When no dual-ops capable providers are available, return the best provider
+ * for me1 only, and sets *prov_mt2 to CRYPTO_INVALID_MECHID;
+ * We assume/expect that a slower HW capable of the dual is still
+ * faster than the 2 fastest providers capable of the individual ops
+ * separately.
+ */
+kcf_provider_desc_t *
+kcf_get_dual_provider(crypto_mechanism_t *mech1, crypto_mechanism_t *mech2,
+ kcf_mech_entry_t **mepp, crypto_mech_type_t *prov_mt1,
+ crypto_mech_type_t *prov_mt2, int *error, kcf_prov_tried_t *triedl,
+ crypto_func_group_t fg1, crypto_func_group_t fg2, boolean_t call_restrict,
+ size_t data_size)
+{
+ kcf_provider_desc_t *pd = NULL, *pdm1 = NULL, *pdm1m2 = NULL;
+ kcf_prov_mech_desc_t *prov_chain, *mdesc;
+ int len, gqlen = INT_MAX, dgqlen = INT_MAX;
+ crypto_mech_info_list_t *mil;
+ crypto_mech_type_t m2id = mech2->cm_type;
+ kcf_mech_entry_t *me;
+
+ /* when mech is a valid mechanism, me will be its mech_entry */
+ if (kcf_get_mech_entry(mech1->cm_type, &me) != KCF_SUCCESS) {
+ *error = CRYPTO_MECHANISM_INVALID;
+ return (NULL);
+ }
+
+ *prov_mt2 = CRYPTO_MECH_INVALID;
+
+ if (mepp != NULL)
+ *mepp = me;
+ mutex_enter(&me->me_mutex);
+
+ prov_chain = me->me_hw_prov_chain;
+ /*
+ * We check the threshold for using a hardware provider for
+ * this amount of data. If there is no software provider available
+ * for the first mechanism, then the threshold is ignored.
+ */
+ if ((prov_chain != NULL) &&
+ ((data_size == 0) || (me->me_threshold == 0) ||
+ (data_size >= me->me_threshold) ||
+ ((mdesc = me->me_sw_prov) == NULL) ||
+ (!IS_FG_SUPPORTED(mdesc, fg1)) ||
+ (!KCF_IS_PROV_USABLE(mdesc->pm_prov_desc)))) {
+ /* there is at least one provider */
+ ASSERT(me->me_num_hwprov > 0);
+
+ /*
+ * Find the least loaded provider capable of the combo
+ * me1 + me2, and save a pointer to the least loaded
+ * provider capable of me1 only.
+ */
+ while (prov_chain != NULL) {
+ pd = prov_chain->pm_prov_desc;
+ len = KCF_PROV_LOAD(pd);
+
+ if (!IS_FG_SUPPORTED(prov_chain, fg1) ||
+ !KCF_IS_PROV_USABLE(pd) ||
+ IS_PROVIDER_TRIED(pd, triedl) ||
+ (call_restrict &&
+ (pd->pd_flags & KCF_PROV_RESTRICTED))) {
+ prov_chain = prov_chain->pm_next;
+ continue;
+ }
+
+ /* Save the best provider capable of m1 */
+ if (len < gqlen) {
+ *prov_mt1 =
+ prov_chain->pm_mech_info.cm_mech_number;
+ gqlen = len;
+ pdm1 = pd;
+ }
+
+ /* See if pd can do me2 too */
+ for (mil = prov_chain->pm_mi_list;
+ mil != NULL; mil = mil->ml_next) {
+ if ((mil->ml_mech_info.cm_func_group_mask &
+ fg2) == 0)
+ continue;
+
+ if ((mil->ml_kcf_mechid == m2id) &&
+ (len < dgqlen)) {
+ /* Bingo! */
+ dgqlen = len;
+ pdm1m2 = pd;
+ *prov_mt2 =
+ mil->ml_mech_info.cm_mech_number;
+ *prov_mt1 = prov_chain->
+ pm_mech_info.cm_mech_number;
+ break;
+ }
+ }
+
+ prov_chain = prov_chain->pm_next;
+ }
+
+ pd = (pdm1m2 != NULL) ? pdm1m2 : pdm1;
+ }
+
+ /* no HW provider for this mech, is there a SW provider? */
+ if (pd == NULL && (mdesc = me->me_sw_prov) != NULL) {
+ pd = mdesc->pm_prov_desc;
+ if (!IS_FG_SUPPORTED(mdesc, fg1) ||
+ !KCF_IS_PROV_USABLE(pd) ||
+ IS_PROVIDER_TRIED(pd, triedl) ||
+ (call_restrict && (pd->pd_flags & KCF_PROV_RESTRICTED)))
+ pd = NULL;
+ else {
+ /* See if pd can do me2 too */
+ for (mil = me->me_sw_prov->pm_mi_list;
+ mil != NULL; mil = mil->ml_next) {
+ if ((mil->ml_mech_info.cm_func_group_mask &
+ fg2) == 0)
+ continue;
+
+ if (mil->ml_kcf_mechid == m2id) {
+ /* Bingo! */
+ *prov_mt2 =
+ mil->ml_mech_info.cm_mech_number;
+ break;
+ }
+ }
+ *prov_mt1 = me->me_sw_prov->pm_mech_info.cm_mech_number;
+ }
+ }
+
+ if (pd == NULL)
+ *error = CRYPTO_MECH_NOT_SUPPORTED;
+ else
+ KCF_PROV_REFHOLD(pd);
+
+ mutex_exit(&me->me_mutex);
+ return (pd);
+}
+
+/*
+ * Do the actual work of calling the provider routines.
+ *
+ * pd - Provider structure
+ * ctx - Context for this operation
+ * params - Parameters for this operation
+ * rhndl - Request handle to use for notification
+ *
+ * The return values are the same as that of the respective SPI.
+ */
+int
+common_submit_request(kcf_provider_desc_t *pd, crypto_ctx_t *ctx,
+ kcf_req_params_t *params, crypto_req_handle_t rhndl)
+{
+ int err = CRYPTO_ARGUMENTS_BAD;
+ kcf_op_type_t optype;
+
+ optype = params->rp_optype;
+
+ switch (params->rp_opgrp) {
+ case KCF_OG_DIGEST: {
+ kcf_digest_ops_params_t *dops = ¶ms->rp_u.digest_params;
+
+ switch (optype) {
+ case KCF_OP_INIT:
+ /*
+ * We should do this only here and not in KCF_WRAP_*
+ * macros. This is because we may want to try other
+ * providers, in case we recover from a failure.
+ */
+ KCF_SET_PROVIDER_MECHNUM(dops->do_framework_mechtype,
+ pd, &dops->do_mech);
+
+ err = KCF_PROV_DIGEST_INIT(pd, ctx, &dops->do_mech,
+ rhndl);
+ break;
+
+ case KCF_OP_SINGLE:
+ err = KCF_PROV_DIGEST(pd, ctx, dops->do_data,
+ dops->do_digest, rhndl);
+ break;
+
+ case KCF_OP_UPDATE:
+ err = KCF_PROV_DIGEST_UPDATE(pd, ctx,
+ dops->do_data, rhndl);
+ break;
+
+ case KCF_OP_FINAL:
+ err = KCF_PROV_DIGEST_FINAL(pd, ctx,
+ dops->do_digest, rhndl);
+ break;
+
+ case KCF_OP_ATOMIC:
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(dops->do_framework_mechtype,
+ pd, &dops->do_mech);
+ err = KCF_PROV_DIGEST_ATOMIC(pd, dops->do_sid,
+ &dops->do_mech, dops->do_data, dops->do_digest,
+ rhndl);
+ break;
+
+ case KCF_OP_DIGEST_KEY:
+ err = KCF_PROV_DIGEST_KEY(pd, ctx, dops->do_digest_key,
+ rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_MAC: {
+ kcf_mac_ops_params_t *mops = ¶ms->rp_u.mac_params;
+
+ switch (optype) {
+ case KCF_OP_INIT:
+ KCF_SET_PROVIDER_MECHNUM(mops->mo_framework_mechtype,
+ pd, &mops->mo_mech);
+
+ err = KCF_PROV_MAC_INIT(pd, ctx, &mops->mo_mech,
+ mops->mo_key, mops->mo_templ, rhndl);
+ break;
+
+ case KCF_OP_SINGLE:
+ err = KCF_PROV_MAC(pd, ctx, mops->mo_data,
+ mops->mo_mac, rhndl);
+ break;
+
+ case KCF_OP_UPDATE:
+ err = KCF_PROV_MAC_UPDATE(pd, ctx, mops->mo_data,
+ rhndl);
+ break;
+
+ case KCF_OP_FINAL:
+ err = KCF_PROV_MAC_FINAL(pd, ctx, mops->mo_mac, rhndl);
+ break;
+
+ case KCF_OP_ATOMIC:
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(mops->mo_framework_mechtype,
+ pd, &mops->mo_mech);
+
+ err = KCF_PROV_MAC_ATOMIC(pd, mops->mo_sid,
+ &mops->mo_mech, mops->mo_key, mops->mo_data,
+ mops->mo_mac, mops->mo_templ, rhndl);
+ break;
+
+ case KCF_OP_MAC_VERIFY_ATOMIC:
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(mops->mo_framework_mechtype,
+ pd, &mops->mo_mech);
+
+ err = KCF_PROV_MAC_VERIFY_ATOMIC(pd, mops->mo_sid,
+ &mops->mo_mech, mops->mo_key, mops->mo_data,
+ mops->mo_mac, mops->mo_templ, rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_ENCRYPT: {
+ kcf_encrypt_ops_params_t *eops = ¶ms->rp_u.encrypt_params;
+
+ switch (optype) {
+ case KCF_OP_INIT:
+ KCF_SET_PROVIDER_MECHNUM(eops->eo_framework_mechtype,
+ pd, &eops->eo_mech);
+
+ err = KCF_PROV_ENCRYPT_INIT(pd, ctx, &eops->eo_mech,
+ eops->eo_key, eops->eo_templ, rhndl);
+ break;
+
+ case KCF_OP_SINGLE:
+ err = KCF_PROV_ENCRYPT(pd, ctx, eops->eo_plaintext,
+ eops->eo_ciphertext, rhndl);
+ break;
+
+ case KCF_OP_UPDATE:
+ err = KCF_PROV_ENCRYPT_UPDATE(pd, ctx,
+ eops->eo_plaintext, eops->eo_ciphertext, rhndl);
+ break;
+
+ case KCF_OP_FINAL:
+ err = KCF_PROV_ENCRYPT_FINAL(pd, ctx,
+ eops->eo_ciphertext, rhndl);
+ break;
+
+ case KCF_OP_ATOMIC:
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(eops->eo_framework_mechtype,
+ pd, &eops->eo_mech);
+
+ err = KCF_PROV_ENCRYPT_ATOMIC(pd, eops->eo_sid,
+ &eops->eo_mech, eops->eo_key, eops->eo_plaintext,
+ eops->eo_ciphertext, eops->eo_templ, rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_DECRYPT: {
+ kcf_decrypt_ops_params_t *dcrops = ¶ms->rp_u.decrypt_params;
+
+ switch (optype) {
+ case KCF_OP_INIT:
+ KCF_SET_PROVIDER_MECHNUM(dcrops->dop_framework_mechtype,
+ pd, &dcrops->dop_mech);
+
+ err = KCF_PROV_DECRYPT_INIT(pd, ctx, &dcrops->dop_mech,
+ dcrops->dop_key, dcrops->dop_templ, rhndl);
+ break;
+
+ case KCF_OP_SINGLE:
+ err = KCF_PROV_DECRYPT(pd, ctx, dcrops->dop_ciphertext,
+ dcrops->dop_plaintext, rhndl);
+ break;
+
+ case KCF_OP_UPDATE:
+ err = KCF_PROV_DECRYPT_UPDATE(pd, ctx,
+ dcrops->dop_ciphertext, dcrops->dop_plaintext,
+ rhndl);
+ break;
+
+ case KCF_OP_FINAL:
+ err = KCF_PROV_DECRYPT_FINAL(pd, ctx,
+ dcrops->dop_plaintext, rhndl);
+ break;
+
+ case KCF_OP_ATOMIC:
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(dcrops->dop_framework_mechtype,
+ pd, &dcrops->dop_mech);
+
+ err = KCF_PROV_DECRYPT_ATOMIC(pd, dcrops->dop_sid,
+ &dcrops->dop_mech, dcrops->dop_key,
+ dcrops->dop_ciphertext, dcrops->dop_plaintext,
+ dcrops->dop_templ, rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_SIGN: {
+ kcf_sign_ops_params_t *sops = ¶ms->rp_u.sign_params;
+
+ switch (optype) {
+ case KCF_OP_INIT:
+ KCF_SET_PROVIDER_MECHNUM(sops->so_framework_mechtype,
+ pd, &sops->so_mech);
+
+ err = KCF_PROV_SIGN_INIT(pd, ctx, &sops->so_mech,
+ sops->so_key, sops->so_templ, rhndl);
+ break;
+
+ case KCF_OP_SIGN_RECOVER_INIT:
+ KCF_SET_PROVIDER_MECHNUM(sops->so_framework_mechtype,
+ pd, &sops->so_mech);
+
+ err = KCF_PROV_SIGN_RECOVER_INIT(pd, ctx,
+ &sops->so_mech, sops->so_key, sops->so_templ,
+ rhndl);
+ break;
+
+ case KCF_OP_SINGLE:
+ err = KCF_PROV_SIGN(pd, ctx, sops->so_data,
+ sops->so_signature, rhndl);
+ break;
+
+ case KCF_OP_SIGN_RECOVER:
+ err = KCF_PROV_SIGN_RECOVER(pd, ctx,
+ sops->so_data, sops->so_signature, rhndl);
+ break;
+
+ case KCF_OP_UPDATE:
+ err = KCF_PROV_SIGN_UPDATE(pd, ctx, sops->so_data,
+ rhndl);
+ break;
+
+ case KCF_OP_FINAL:
+ err = KCF_PROV_SIGN_FINAL(pd, ctx, sops->so_signature,
+ rhndl);
+ break;
+
+ case KCF_OP_ATOMIC:
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(sops->so_framework_mechtype,
+ pd, &sops->so_mech);
+
+ err = KCF_PROV_SIGN_ATOMIC(pd, sops->so_sid,
+ &sops->so_mech, sops->so_key, sops->so_data,
+ sops->so_templ, sops->so_signature, rhndl);
+ break;
+
+ case KCF_OP_SIGN_RECOVER_ATOMIC:
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(sops->so_framework_mechtype,
+ pd, &sops->so_mech);
+
+ err = KCF_PROV_SIGN_RECOVER_ATOMIC(pd, sops->so_sid,
+ &sops->so_mech, sops->so_key, sops->so_data,
+ sops->so_templ, sops->so_signature, rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_VERIFY: {
+ kcf_verify_ops_params_t *vops = ¶ms->rp_u.verify_params;
+
+ switch (optype) {
+ case KCF_OP_INIT:
+ KCF_SET_PROVIDER_MECHNUM(vops->vo_framework_mechtype,
+ pd, &vops->vo_mech);
+
+ err = KCF_PROV_VERIFY_INIT(pd, ctx, &vops->vo_mech,
+ vops->vo_key, vops->vo_templ, rhndl);
+ break;
+
+ case KCF_OP_VERIFY_RECOVER_INIT:
+ KCF_SET_PROVIDER_MECHNUM(vops->vo_framework_mechtype,
+ pd, &vops->vo_mech);
+
+ err = KCF_PROV_VERIFY_RECOVER_INIT(pd, ctx,
+ &vops->vo_mech, vops->vo_key, vops->vo_templ,
+ rhndl);
+ break;
+
+ case KCF_OP_SINGLE:
+ err = KCF_PROV_VERIFY(pd, ctx, vops->vo_data,
+ vops->vo_signature, rhndl);
+ break;
+
+ case KCF_OP_VERIFY_RECOVER:
+ err = KCF_PROV_VERIFY_RECOVER(pd, ctx,
+ vops->vo_signature, vops->vo_data, rhndl);
+ break;
+
+ case KCF_OP_UPDATE:
+ err = KCF_PROV_VERIFY_UPDATE(pd, ctx, vops->vo_data,
+ rhndl);
+ break;
+
+ case KCF_OP_FINAL:
+ err = KCF_PROV_VERIFY_FINAL(pd, ctx, vops->vo_signature,
+ rhndl);
+ break;
+
+ case KCF_OP_ATOMIC:
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(vops->vo_framework_mechtype,
+ pd, &vops->vo_mech);
+
+ err = KCF_PROV_VERIFY_ATOMIC(pd, vops->vo_sid,
+ &vops->vo_mech, vops->vo_key, vops->vo_data,
+ vops->vo_templ, vops->vo_signature, rhndl);
+ break;
+
+ case KCF_OP_VERIFY_RECOVER_ATOMIC:
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(vops->vo_framework_mechtype,
+ pd, &vops->vo_mech);
+
+ err = KCF_PROV_VERIFY_RECOVER_ATOMIC(pd, vops->vo_sid,
+ &vops->vo_mech, vops->vo_key, vops->vo_signature,
+ vops->vo_templ, vops->vo_data, rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_ENCRYPT_MAC: {
+ kcf_encrypt_mac_ops_params_t *eops =
+ ¶ms->rp_u.encrypt_mac_params;
+ kcf_context_t *kcf_secondctx;
+
+ switch (optype) {
+ case KCF_OP_INIT:
+ kcf_secondctx = ((kcf_context_t *)
+ (ctx->cc_framework_private))->kc_secondctx;
+
+ if (kcf_secondctx != NULL) {
+ err = kcf_emulate_dual(pd, ctx, params);
+ break;
+ }
+ KCF_SET_PROVIDER_MECHNUM(
+ eops->em_framework_encr_mechtype,
+ pd, &eops->em_encr_mech);
+
+ KCF_SET_PROVIDER_MECHNUM(
+ eops->em_framework_mac_mechtype,
+ pd, &eops->em_mac_mech);
+
+ err = KCF_PROV_ENCRYPT_MAC_INIT(pd, ctx,
+ &eops->em_encr_mech, eops->em_encr_key,
+ &eops->em_mac_mech, eops->em_mac_key,
+ eops->em_encr_templ, eops->em_mac_templ,
+ rhndl);
+
+ break;
+
+ case KCF_OP_SINGLE:
+ err = KCF_PROV_ENCRYPT_MAC(pd, ctx,
+ eops->em_plaintext, eops->em_ciphertext,
+ eops->em_mac, rhndl);
+ break;
+
+ case KCF_OP_UPDATE:
+ kcf_secondctx = ((kcf_context_t *)
+ (ctx->cc_framework_private))->kc_secondctx;
+ if (kcf_secondctx != NULL) {
+ err = kcf_emulate_dual(pd, ctx, params);
+ break;
+ }
+ err = KCF_PROV_ENCRYPT_MAC_UPDATE(pd, ctx,
+ eops->em_plaintext, eops->em_ciphertext, rhndl);
+ break;
+
+ case KCF_OP_FINAL:
+ kcf_secondctx = ((kcf_context_t *)
+ (ctx->cc_framework_private))->kc_secondctx;
+ if (kcf_secondctx != NULL) {
+ err = kcf_emulate_dual(pd, ctx, params);
+ break;
+ }
+ err = KCF_PROV_ENCRYPT_MAC_FINAL(pd, ctx,
+ eops->em_ciphertext, eops->em_mac, rhndl);
+ break;
+
+ case KCF_OP_ATOMIC:
+ ASSERT(ctx == NULL);
+
+ KCF_SET_PROVIDER_MECHNUM(
+ eops->em_framework_encr_mechtype,
+ pd, &eops->em_encr_mech);
+
+ KCF_SET_PROVIDER_MECHNUM(
+ eops->em_framework_mac_mechtype,
+ pd, &eops->em_mac_mech);
+
+ err = KCF_PROV_ENCRYPT_MAC_ATOMIC(pd, eops->em_sid,
+ &eops->em_encr_mech, eops->em_encr_key,
+ &eops->em_mac_mech, eops->em_mac_key,
+ eops->em_plaintext, eops->em_ciphertext,
+ eops->em_mac,
+ eops->em_encr_templ, eops->em_mac_templ,
+ rhndl);
+
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_MAC_DECRYPT: {
+ kcf_mac_decrypt_ops_params_t *dops =
+ ¶ms->rp_u.mac_decrypt_params;
+ kcf_context_t *kcf_secondctx;
+
+ switch (optype) {
+ case KCF_OP_INIT:
+ kcf_secondctx = ((kcf_context_t *)
+ (ctx->cc_framework_private))->kc_secondctx;
+
+ if (kcf_secondctx != NULL) {
+ err = kcf_emulate_dual(pd, ctx, params);
+ break;
+ }
+ KCF_SET_PROVIDER_MECHNUM(
+ dops->md_framework_mac_mechtype,
+ pd, &dops->md_mac_mech);
+
+ KCF_SET_PROVIDER_MECHNUM(
+ dops->md_framework_decr_mechtype,
+ pd, &dops->md_decr_mech);
+
+ err = KCF_PROV_MAC_DECRYPT_INIT(pd, ctx,
+ &dops->md_mac_mech, dops->md_mac_key,
+ &dops->md_decr_mech, dops->md_decr_key,
+ dops->md_mac_templ, dops->md_decr_templ,
+ rhndl);
+
+ break;
+
+ case KCF_OP_SINGLE:
+ err = KCF_PROV_MAC_DECRYPT(pd, ctx,
+ dops->md_ciphertext, dops->md_mac,
+ dops->md_plaintext, rhndl);
+ break;
+
+ case KCF_OP_UPDATE:
+ kcf_secondctx = ((kcf_context_t *)
+ (ctx->cc_framework_private))->kc_secondctx;
+ if (kcf_secondctx != NULL) {
+ err = kcf_emulate_dual(pd, ctx, params);
+ break;
+ }
+ err = KCF_PROV_MAC_DECRYPT_UPDATE(pd, ctx,
+ dops->md_ciphertext, dops->md_plaintext, rhndl);
+ break;
+
+ case KCF_OP_FINAL:
+ kcf_secondctx = ((kcf_context_t *)
+ (ctx->cc_framework_private))->kc_secondctx;
+ if (kcf_secondctx != NULL) {
+ err = kcf_emulate_dual(pd, ctx, params);
+ break;
+ }
+ err = KCF_PROV_MAC_DECRYPT_FINAL(pd, ctx,
+ dops->md_mac, dops->md_plaintext, rhndl);
+ break;
+
+ case KCF_OP_ATOMIC:
+ ASSERT(ctx == NULL);
+
+ KCF_SET_PROVIDER_MECHNUM(
+ dops->md_framework_mac_mechtype,
+ pd, &dops->md_mac_mech);
+
+ KCF_SET_PROVIDER_MECHNUM(
+ dops->md_framework_decr_mechtype,
+ pd, &dops->md_decr_mech);
+
+ err = KCF_PROV_MAC_DECRYPT_ATOMIC(pd, dops->md_sid,
+ &dops->md_mac_mech, dops->md_mac_key,
+ &dops->md_decr_mech, dops->md_decr_key,
+ dops->md_ciphertext, dops->md_mac,
+ dops->md_plaintext,
+ dops->md_mac_templ, dops->md_decr_templ,
+ rhndl);
+
+ break;
+
+ case KCF_OP_MAC_VERIFY_DECRYPT_ATOMIC:
+ ASSERT(ctx == NULL);
+
+ KCF_SET_PROVIDER_MECHNUM(
+ dops->md_framework_mac_mechtype,
+ pd, &dops->md_mac_mech);
+
+ KCF_SET_PROVIDER_MECHNUM(
+ dops->md_framework_decr_mechtype,
+ pd, &dops->md_decr_mech);
+
+ err = KCF_PROV_MAC_VERIFY_DECRYPT_ATOMIC(pd,
+ dops->md_sid, &dops->md_mac_mech, dops->md_mac_key,
+ &dops->md_decr_mech, dops->md_decr_key,
+ dops->md_ciphertext, dops->md_mac,
+ dops->md_plaintext,
+ dops->md_mac_templ, dops->md_decr_templ,
+ rhndl);
+
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_KEY: {
+ kcf_key_ops_params_t *kops = ¶ms->rp_u.key_params;
+
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(kops->ko_framework_mechtype, pd,
+ &kops->ko_mech);
+
+ switch (optype) {
+ case KCF_OP_KEY_GENERATE:
+ err = KCF_PROV_KEY_GENERATE(pd, kops->ko_sid,
+ &kops->ko_mech,
+ kops->ko_key_template, kops->ko_key_attribute_count,
+ kops->ko_key_object_id_ptr, rhndl);
+ break;
+
+ case KCF_OP_KEY_GENERATE_PAIR:
+ err = KCF_PROV_KEY_GENERATE_PAIR(pd, kops->ko_sid,
+ &kops->ko_mech,
+ kops->ko_key_template, kops->ko_key_attribute_count,
+ kops->ko_private_key_template,
+ kops->ko_private_key_attribute_count,
+ kops->ko_key_object_id_ptr,
+ kops->ko_private_key_object_id_ptr, rhndl);
+ break;
+
+ case KCF_OP_KEY_WRAP:
+ err = KCF_PROV_KEY_WRAP(pd, kops->ko_sid,
+ &kops->ko_mech,
+ kops->ko_key, kops->ko_key_object_id_ptr,
+ kops->ko_wrapped_key, kops->ko_wrapped_key_len_ptr,
+ rhndl);
+ break;
+
+ case KCF_OP_KEY_UNWRAP:
+ err = KCF_PROV_KEY_UNWRAP(pd, kops->ko_sid,
+ &kops->ko_mech,
+ kops->ko_key, kops->ko_wrapped_key,
+ kops->ko_wrapped_key_len_ptr,
+ kops->ko_key_template, kops->ko_key_attribute_count,
+ kops->ko_key_object_id_ptr, rhndl);
+ break;
+
+ case KCF_OP_KEY_DERIVE:
+ err = KCF_PROV_KEY_DERIVE(pd, kops->ko_sid,
+ &kops->ko_mech,
+ kops->ko_key, kops->ko_key_template,
+ kops->ko_key_attribute_count,
+ kops->ko_key_object_id_ptr, rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_RANDOM: {
+ kcf_random_number_ops_params_t *rops =
+ ¶ms->rp_u.random_number_params;
+
+ ASSERT(ctx == NULL);
+
+ switch (optype) {
+ case KCF_OP_RANDOM_SEED:
+ err = KCF_PROV_SEED_RANDOM(pd, rops->rn_sid,
+ rops->rn_buf, rops->rn_buflen, rops->rn_entropy_est,
+ rops->rn_flags, rhndl);
+ break;
+
+ case KCF_OP_RANDOM_GENERATE:
+ err = KCF_PROV_GENERATE_RANDOM(pd, rops->rn_sid,
+ rops->rn_buf, rops->rn_buflen, rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_SESSION: {
+ kcf_session_ops_params_t *sops = ¶ms->rp_u.session_params;
+
+ ASSERT(ctx == NULL);
+ switch (optype) {
+ case KCF_OP_SESSION_OPEN:
+ /*
+ * so_pd may be a logical provider, in which case
+ * we need to check whether it has been removed.
+ */
+ if (KCF_IS_PROV_REMOVED(sops->so_pd)) {
+ err = CRYPTO_DEVICE_ERROR;
+ break;
+ }
+ err = KCF_PROV_SESSION_OPEN(pd, sops->so_sid_ptr,
+ rhndl, sops->so_pd);
+ break;
+
+ case KCF_OP_SESSION_CLOSE:
+ /*
+ * so_pd may be a logical provider, in which case
+ * we need to check whether it has been removed.
+ */
+ if (KCF_IS_PROV_REMOVED(sops->so_pd)) {
+ err = CRYPTO_DEVICE_ERROR;
+ break;
+ }
+ err = KCF_PROV_SESSION_CLOSE(pd, sops->so_sid,
+ rhndl, sops->so_pd);
+ break;
+
+ case KCF_OP_SESSION_LOGIN:
+ err = KCF_PROV_SESSION_LOGIN(pd, sops->so_sid,
+ sops->so_user_type, sops->so_pin,
+ sops->so_pin_len, rhndl);
+ break;
+
+ case KCF_OP_SESSION_LOGOUT:
+ err = KCF_PROV_SESSION_LOGOUT(pd, sops->so_sid, rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_OBJECT: {
+ kcf_object_ops_params_t *jops = ¶ms->rp_u.object_params;
+
+ ASSERT(ctx == NULL);
+ switch (optype) {
+ case KCF_OP_OBJECT_CREATE:
+ err = KCF_PROV_OBJECT_CREATE(pd, jops->oo_sid,
+ jops->oo_template, jops->oo_attribute_count,
+ jops->oo_object_id_ptr, rhndl);
+ break;
+
+ case KCF_OP_OBJECT_COPY:
+ err = KCF_PROV_OBJECT_COPY(pd, jops->oo_sid,
+ jops->oo_object_id,
+ jops->oo_template, jops->oo_attribute_count,
+ jops->oo_object_id_ptr, rhndl);
+ break;
+
+ case KCF_OP_OBJECT_DESTROY:
+ err = KCF_PROV_OBJECT_DESTROY(pd, jops->oo_sid,
+ jops->oo_object_id, rhndl);
+ break;
+
+ case KCF_OP_OBJECT_GET_SIZE:
+ err = KCF_PROV_OBJECT_GET_SIZE(pd, jops->oo_sid,
+ jops->oo_object_id, jops->oo_object_size, rhndl);
+ break;
+
+ case KCF_OP_OBJECT_GET_ATTRIBUTE_VALUE:
+ err = KCF_PROV_OBJECT_GET_ATTRIBUTE_VALUE(pd,
+ jops->oo_sid, jops->oo_object_id,
+ jops->oo_template, jops->oo_attribute_count, rhndl);
+ break;
+
+ case KCF_OP_OBJECT_SET_ATTRIBUTE_VALUE:
+ err = KCF_PROV_OBJECT_SET_ATTRIBUTE_VALUE(pd,
+ jops->oo_sid, jops->oo_object_id,
+ jops->oo_template, jops->oo_attribute_count, rhndl);
+ break;
+
+ case KCF_OP_OBJECT_FIND_INIT:
+ err = KCF_PROV_OBJECT_FIND_INIT(pd, jops->oo_sid,
+ jops->oo_template, jops->oo_attribute_count,
+ jops->oo_find_init_pp_ptr, rhndl);
+ break;
+
+ case KCF_OP_OBJECT_FIND:
+ err = KCF_PROV_OBJECT_FIND(pd, jops->oo_find_pp,
+ jops->oo_object_id_ptr, jops->oo_max_object_count,
+ jops->oo_object_count_ptr, rhndl);
+ break;
+
+ case KCF_OP_OBJECT_FIND_FINAL:
+ err = KCF_PROV_OBJECT_FIND_FINAL(pd, jops->oo_find_pp,
+ rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_PROVMGMT: {
+ kcf_provmgmt_ops_params_t *pops = ¶ms->rp_u.provmgmt_params;
+
+ ASSERT(ctx == NULL);
+ switch (optype) {
+ case KCF_OP_MGMT_EXTINFO:
+ /*
+ * po_pd may be a logical provider, in which case
+ * we need to check whether it has been removed.
+ */
+ if (KCF_IS_PROV_REMOVED(pops->po_pd)) {
+ err = CRYPTO_DEVICE_ERROR;
+ break;
+ }
+ err = KCF_PROV_EXT_INFO(pd, pops->po_ext_info, rhndl,
+ pops->po_pd);
+ break;
+
+ case KCF_OP_MGMT_INITTOKEN:
+ err = KCF_PROV_INIT_TOKEN(pd, pops->po_pin,
+ pops->po_pin_len, pops->po_label, rhndl);
+ break;
+
+ case KCF_OP_MGMT_INITPIN:
+ err = KCF_PROV_INIT_PIN(pd, pops->po_sid, pops->po_pin,
+ pops->po_pin_len, rhndl);
+ break;
+
+ case KCF_OP_MGMT_SETPIN:
+ err = KCF_PROV_SET_PIN(pd, pops->po_sid,
+ pops->po_old_pin, pops->po_old_pin_len,
+ pops->po_pin, pops->po_pin_len, rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+
+ case KCF_OG_NOSTORE_KEY: {
+ kcf_key_ops_params_t *kops = ¶ms->rp_u.key_params;
+
+ ASSERT(ctx == NULL);
+ KCF_SET_PROVIDER_MECHNUM(kops->ko_framework_mechtype, pd,
+ &kops->ko_mech);
+
+ switch (optype) {
+ case KCF_OP_KEY_GENERATE:
+ err = KCF_PROV_NOSTORE_KEY_GENERATE(pd, kops->ko_sid,
+ &kops->ko_mech, kops->ko_key_template,
+ kops->ko_key_attribute_count,
+ kops->ko_out_template1,
+ kops->ko_out_attribute_count1, rhndl);
+ break;
+
+ case KCF_OP_KEY_GENERATE_PAIR:
+ err = KCF_PROV_NOSTORE_KEY_GENERATE_PAIR(pd,
+ kops->ko_sid, &kops->ko_mech,
+ kops->ko_key_template, kops->ko_key_attribute_count,
+ kops->ko_private_key_template,
+ kops->ko_private_key_attribute_count,
+ kops->ko_out_template1,
+ kops->ko_out_attribute_count1,
+ kops->ko_out_template2,
+ kops->ko_out_attribute_count2,
+ rhndl);
+ break;
+
+ case KCF_OP_KEY_DERIVE:
+ err = KCF_PROV_NOSTORE_KEY_DERIVE(pd, kops->ko_sid,
+ &kops->ko_mech, kops->ko_key,
+ kops->ko_key_template,
+ kops->ko_key_attribute_count,
+ kops->ko_out_template1,
+ kops->ko_out_attribute_count1, rhndl);
+ break;
+
+ default:
+ break;
+ }
+ break;
+ }
+ default:
+ break;
+ } /* end of switch(params->rp_opgrp) */
+
+ KCF_PROV_INCRSTATS(pd, err);
+ return (err);
+}
+
+
+/*
+ * Emulate the call for a multipart dual ops with 2 single steps.
+ * This routine is always called in the context of a working thread
+ * running kcf_svc_do_run().
+ * The single steps are submitted in a pure synchronous way (blocking).
+ * When this routine returns, kcf_svc_do_run() will call kcf_aop_done()
+ * so the originating consumer's callback gets invoked. kcf_aop_done()
+ * takes care of freeing the operation context. So, this routine does
+ * not free the operation context.
+ *
+ * The provider descriptor is assumed held by the callers.
+ */
+static int
+kcf_emulate_dual(kcf_provider_desc_t *pd, crypto_ctx_t *ctx,
+ kcf_req_params_t *params)
+{
+ int err = CRYPTO_ARGUMENTS_BAD;
+ kcf_op_type_t optype;
+ size_t save_len;
+ off_t save_offset;
+
+ optype = params->rp_optype;
+
+ switch (params->rp_opgrp) {
+ case KCF_OG_ENCRYPT_MAC: {
+ kcf_encrypt_mac_ops_params_t *cmops =
+ ¶ms->rp_u.encrypt_mac_params;
+ kcf_context_t *encr_kcf_ctx;
+ crypto_ctx_t *mac_ctx;
+ kcf_req_params_t encr_params;
+
+ encr_kcf_ctx = (kcf_context_t *)(ctx->cc_framework_private);
+
+ switch (optype) {
+ case KCF_OP_INIT: {
+ encr_kcf_ctx->kc_secondctx = NULL;
+
+ KCF_WRAP_ENCRYPT_OPS_PARAMS(&encr_params, KCF_OP_INIT,
+ pd->pd_sid, &cmops->em_encr_mech,
+ cmops->em_encr_key, NULL, NULL,
+ cmops->em_encr_templ);
+
+ err = kcf_submit_request(pd, ctx, NULL, &encr_params,
+ B_FALSE);
+
+ /* It can't be CRYPTO_QUEUED */
+ if (err != CRYPTO_SUCCESS) {
+ break;
+ }
+
+ err = crypto_mac_init(&cmops->em_mac_mech,
+ cmops->em_mac_key, cmops->em_mac_templ,
+ (crypto_context_t *)&mac_ctx, NULL);
+
+ if (err == CRYPTO_SUCCESS) {
+ encr_kcf_ctx->kc_secondctx = (kcf_context_t *)
+ mac_ctx->cc_framework_private;
+ KCF_CONTEXT_REFHOLD((kcf_context_t *)
+ mac_ctx->cc_framework_private);
+ }
+
+ break;
+
+ }
+ case KCF_OP_UPDATE: {
+ crypto_dual_data_t *ct = cmops->em_ciphertext;
+ crypto_data_t *pt = cmops->em_plaintext;
+ kcf_context_t *mac_kcf_ctx = encr_kcf_ctx->kc_secondctx;
+ crypto_ctx_t *mac_ctx = &mac_kcf_ctx->kc_glbl_ctx;
+
+ KCF_WRAP_ENCRYPT_OPS_PARAMS(&encr_params, KCF_OP_UPDATE,
+ pd->pd_sid, NULL, NULL, pt, (crypto_data_t *)ct,
+ NULL);
+
+ err = kcf_submit_request(pd, ctx, NULL, &encr_params,
+ B_FALSE);
+
+ /* It can't be CRYPTO_QUEUED */
+ if (err != CRYPTO_SUCCESS) {
+ break;
+ }
+
+ save_offset = ct->dd_offset1;
+ save_len = ct->dd_len1;
+ if (ct->dd_len2 == 0) {
+ /*
+ * The previous encrypt step was an
+ * accumulation only and didn't produce any
+ * partial output
+ */
+ if (ct->dd_len1 == 0)
+ break;
+
+ } else {
+ ct->dd_offset1 = ct->dd_offset2;
+ ct->dd_len1 = ct->dd_len2;
+ }
+ err = crypto_mac_update((crypto_context_t)mac_ctx,
+ (crypto_data_t *)ct, NULL);
+
+ ct->dd_offset1 = save_offset;
+ ct->dd_len1 = save_len;
+
+ break;
+ }
+ case KCF_OP_FINAL: {
+ crypto_dual_data_t *ct = cmops->em_ciphertext;
+ crypto_data_t *mac = cmops->em_mac;
+ kcf_context_t *mac_kcf_ctx = encr_kcf_ctx->kc_secondctx;
+ crypto_ctx_t *mac_ctx = &mac_kcf_ctx->kc_glbl_ctx;
+ crypto_context_t mac_context = mac_ctx;
+
+ KCF_WRAP_ENCRYPT_OPS_PARAMS(&encr_params, KCF_OP_FINAL,
+ pd->pd_sid, NULL, NULL, NULL, (crypto_data_t *)ct,
+ NULL);
+
+ err = kcf_submit_request(pd, ctx, NULL, &encr_params,
+ B_FALSE);
+
+ /* It can't be CRYPTO_QUEUED */
+ if (err != CRYPTO_SUCCESS) {
+ crypto_cancel_ctx(mac_context);
+ break;
+ }
+
+ if (ct->dd_len2 > 0) {
+ save_offset = ct->dd_offset1;
+ save_len = ct->dd_len1;
+ ct->dd_offset1 = ct->dd_offset2;
+ ct->dd_len1 = ct->dd_len2;
+
+ err = crypto_mac_update(mac_context,
+ (crypto_data_t *)ct, NULL);
+
+ ct->dd_offset1 = save_offset;
+ ct->dd_len1 = save_len;
+
+ if (err != CRYPTO_SUCCESS) {
+ crypto_cancel_ctx(mac_context);
+ return (err);
+ }
+ }
+
+ /* and finally, collect the MAC */
+ err = crypto_mac_final(mac_context, mac, NULL);
+ break;
+ }
+
+ default:
+ break;
+ }
+ KCF_PROV_INCRSTATS(pd, err);
+ break;
+ }
+ case KCF_OG_MAC_DECRYPT: {
+ kcf_mac_decrypt_ops_params_t *mdops =
+ ¶ms->rp_u.mac_decrypt_params;
+ kcf_context_t *decr_kcf_ctx;
+ crypto_ctx_t *mac_ctx;
+ kcf_req_params_t decr_params;
+
+ decr_kcf_ctx = (kcf_context_t *)(ctx->cc_framework_private);
+
+ switch (optype) {
+ case KCF_OP_INIT: {
+ decr_kcf_ctx->kc_secondctx = NULL;
+
+ err = crypto_mac_init(&mdops->md_mac_mech,
+ mdops->md_mac_key, mdops->md_mac_templ,
+ (crypto_context_t *)&mac_ctx, NULL);
+
+ /* It can't be CRYPTO_QUEUED */
+ if (err != CRYPTO_SUCCESS) {
+ break;
+ }
+
+ KCF_WRAP_DECRYPT_OPS_PARAMS(&decr_params, KCF_OP_INIT,
+ pd->pd_sid, &mdops->md_decr_mech,
+ mdops->md_decr_key, NULL, NULL,
+ mdops->md_decr_templ);
+
+ err = kcf_submit_request(pd, ctx, NULL, &decr_params,
+ B_FALSE);
+
+ /* It can't be CRYPTO_QUEUED */
+ if (err != CRYPTO_SUCCESS) {
+ crypto_cancel_ctx((crypto_context_t)mac_ctx);
+ break;
+ }
+
+ decr_kcf_ctx->kc_secondctx = (kcf_context_t *)
+ mac_ctx->cc_framework_private;
+ KCF_CONTEXT_REFHOLD((kcf_context_t *)
+ mac_ctx->cc_framework_private);
+
+ break;
+ default:
+ break;
+
+ }
+ case KCF_OP_UPDATE: {
+ crypto_dual_data_t *ct = mdops->md_ciphertext;
+ crypto_data_t *pt = mdops->md_plaintext;
+ kcf_context_t *mac_kcf_ctx = decr_kcf_ctx->kc_secondctx;
+ crypto_ctx_t *mac_ctx = &mac_kcf_ctx->kc_glbl_ctx;
+
+ err = crypto_mac_update((crypto_context_t)mac_ctx,
+ (crypto_data_t *)ct, NULL);
+
+ if (err != CRYPTO_SUCCESS)
+ break;
+
+ save_offset = ct->dd_offset1;
+ save_len = ct->dd_len1;
+
+ /* zero ct->dd_len2 means decrypt everything */
+ if (ct->dd_len2 > 0) {
+ ct->dd_offset1 = ct->dd_offset2;
+ ct->dd_len1 = ct->dd_len2;
+ }
+
+ err = crypto_decrypt_update((crypto_context_t)ctx,
+ (crypto_data_t *)ct, pt, NULL);
+
+ ct->dd_offset1 = save_offset;
+ ct->dd_len1 = save_len;
+
+ break;
+ }
+ case KCF_OP_FINAL: {
+ crypto_data_t *pt = mdops->md_plaintext;
+ crypto_data_t *mac = mdops->md_mac;
+ kcf_context_t *mac_kcf_ctx = decr_kcf_ctx->kc_secondctx;
+ crypto_ctx_t *mac_ctx = &mac_kcf_ctx->kc_glbl_ctx;
+
+ err = crypto_mac_final((crypto_context_t)mac_ctx,
+ mac, NULL);
+
+ if (err != CRYPTO_SUCCESS) {
+ crypto_cancel_ctx(ctx);
+ break;
+ }
+
+ /* Get the last chunk of plaintext */
+ KCF_CONTEXT_REFHOLD(decr_kcf_ctx);
+ err = crypto_decrypt_final((crypto_context_t)ctx, pt,
+ NULL);
+
+ break;
+ }
+ }
+ break;
+ }
+ default:
+
+ break;
+ } /* end of switch(params->rp_opgrp) */
+
+ return (err);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/impl.h>
+#include <sys/modhash.h>
+
+/* Cryptographic mechanisms tables and their access functions */
+
+/*
+ * Internal numbers assigned to mechanisms are coded as follows:
+ *
+ * +----------------+----------------+
+ * | mech. class | mech. index |
+ * <--- 32-bits --->+<--- 32-bits --->
+ *
+ * the mech_class identifies the table the mechanism belongs to.
+ * mech_index is the index for that mechanism in the table.
+ * A mechanism belongs to exactly 1 table.
+ * The tables are:
+ * . digest_mechs_tab[] for the msg digest mechs.
+ * . cipher_mechs_tab[] for encrypt/decrypt and wrap/unwrap mechs.
+ * . mac_mechs_tab[] for MAC mechs.
+ * . sign_mechs_tab[] for sign & verify mechs.
+ * . keyops_mechs_tab[] for key/key pair generation, and key derivation.
+ * . misc_mechs_tab[] for mechs that don't belong to any of the above.
+ *
+ * There are no holes in the tables.
+ */
+
+/*
+ * Locking conventions:
+ * --------------------
+ * A global mutex, kcf_mech_tabs_lock, serializes writes to the
+ * mechanism table via kcf_create_mech_entry().
+ *
+ * A mutex is associated with every entry of the tables.
+ * The mutex is acquired whenever the entry is accessed for
+ * 1) retrieving the mech_id (comparing the mech name)
+ * 2) finding a provider for an xxx_init() or atomic operation.
+ * 3) altering the mechs entry to add or remove a provider.
+ *
+ * In 2), after a provider is chosen, its prov_desc is held and the
+ * entry's mutex must be dropped. The provider's working function (SPI) is
+ * called outside the mech_entry's mutex.
+ *
+ * The number of providers for a particular mechanism is not expected to be
+ * long enough to justify the cost of using rwlocks, so the per-mechanism
+ * entry mutex won't be very *hot*.
+ *
+ * When both kcf_mech_tabs_lock and a mech_entry mutex need to be held,
+ * kcf_mech_tabs_lock must always be acquired first.
+ *
+ */
+
+ /* Mechanisms tables */
+
+
+/* RFE 4687834 Will deal with the extensibility of these tables later */
+
+kcf_mech_entry_t kcf_digest_mechs_tab[KCF_MAXDIGEST];
+kcf_mech_entry_t kcf_cipher_mechs_tab[KCF_MAXCIPHER];
+kcf_mech_entry_t kcf_mac_mechs_tab[KCF_MAXMAC];
+kcf_mech_entry_t kcf_sign_mechs_tab[KCF_MAXSIGN];
+kcf_mech_entry_t kcf_keyops_mechs_tab[KCF_MAXKEYOPS];
+kcf_mech_entry_t kcf_misc_mechs_tab[KCF_MAXMISC];
+
+kcf_mech_entry_tab_t kcf_mech_tabs_tab[KCF_LAST_OPSCLASS + 1] = {
+ {0, NULL}, /* No class zero */
+ {KCF_MAXDIGEST, kcf_digest_mechs_tab},
+ {KCF_MAXCIPHER, kcf_cipher_mechs_tab},
+ {KCF_MAXMAC, kcf_mac_mechs_tab},
+ {KCF_MAXSIGN, kcf_sign_mechs_tab},
+ {KCF_MAXKEYOPS, kcf_keyops_mechs_tab},
+ {KCF_MAXMISC, kcf_misc_mechs_tab}
+};
+
+/*
+ * Per-algorithm internal threasholds for the minimum input size of before
+ * offloading to hardware provider.
+ * Dispatching a crypto operation to a hardware provider entails paying the
+ * cost of an additional context switch. Measurments with Sun Accelerator 4000
+ * shows that 512-byte jobs or smaller are better handled in software.
+ * There is room for refinement here.
+ *
+ */
+int kcf_md5_threshold = 512;
+int kcf_sha1_threshold = 512;
+int kcf_des_threshold = 512;
+int kcf_des3_threshold = 512;
+int kcf_aes_threshold = 512;
+int kcf_bf_threshold = 512;
+int kcf_rc4_threshold = 512;
+
+kmutex_t kcf_mech_tabs_lock;
+static uint32_t kcf_gen_swprov = 0;
+
+int kcf_mech_hash_size = 256;
+mod_hash_t *kcf_mech_hash; /* mech name to id hash */
+
+static crypto_mech_type_t
+kcf_mech_hash_find(char *mechname)
+{
+ mod_hash_val_t hv;
+ crypto_mech_type_t mt;
+
+ mt = CRYPTO_MECH_INVALID;
+ if (mod_hash_find(kcf_mech_hash, (mod_hash_key_t)mechname, &hv) == 0) {
+ mt = *(crypto_mech_type_t *)hv;
+ ASSERT(mt != CRYPTO_MECH_INVALID);
+ }
+
+ return (mt);
+}
+
+void
+kcf_destroy_mech_tabs(void)
+{
+ if (kcf_mech_hash) mod_hash_destroy_hash(kcf_mech_hash);
+}
+
+/*
+ * kcf_init_mech_tabs()
+ *
+ * Called by the misc/kcf's _init() routine to initialize the tables
+ * of mech_entry's.
+ */
+void
+kcf_init_mech_tabs(void)
+{
+ int i, max;
+ kcf_ops_class_t class;
+ kcf_mech_entry_t *me_tab;
+
+ /* Initializes the mutex locks. */
+
+ mutex_init(&kcf_mech_tabs_lock, NULL, MUTEX_DEFAULT, NULL);
+
+ /* Then the pre-defined mechanism entries */
+
+ /* Two digests */
+ (void) strncpy(kcf_digest_mechs_tab[0].me_name, SUN_CKM_MD5,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_digest_mechs_tab[0].me_threshold = kcf_md5_threshold;
+
+ (void) strncpy(kcf_digest_mechs_tab[1].me_name, SUN_CKM_SHA1,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_digest_mechs_tab[1].me_threshold = kcf_sha1_threshold;
+
+ /* The symmetric ciphers in various modes */
+ (void) strncpy(kcf_cipher_mechs_tab[0].me_name, SUN_CKM_DES_CBC,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_cipher_mechs_tab[0].me_threshold = kcf_des_threshold;
+
+ (void) strncpy(kcf_cipher_mechs_tab[1].me_name, SUN_CKM_DES3_CBC,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_cipher_mechs_tab[1].me_threshold = kcf_des3_threshold;
+
+ (void) strncpy(kcf_cipher_mechs_tab[2].me_name, SUN_CKM_DES_ECB,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_cipher_mechs_tab[2].me_threshold = kcf_des_threshold;
+
+ (void) strncpy(kcf_cipher_mechs_tab[3].me_name, SUN_CKM_DES3_ECB,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_cipher_mechs_tab[3].me_threshold = kcf_des3_threshold;
+
+ (void) strncpy(kcf_cipher_mechs_tab[4].me_name, SUN_CKM_BLOWFISH_CBC,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_cipher_mechs_tab[4].me_threshold = kcf_bf_threshold;
+
+ (void) strncpy(kcf_cipher_mechs_tab[5].me_name, SUN_CKM_BLOWFISH_ECB,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_cipher_mechs_tab[5].me_threshold = kcf_bf_threshold;
+
+ (void) strncpy(kcf_cipher_mechs_tab[6].me_name, SUN_CKM_AES_CBC,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_cipher_mechs_tab[6].me_threshold = kcf_aes_threshold;
+
+ (void) strncpy(kcf_cipher_mechs_tab[7].me_name, SUN_CKM_AES_ECB,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_cipher_mechs_tab[7].me_threshold = kcf_aes_threshold;
+
+ (void) strncpy(kcf_cipher_mechs_tab[8].me_name, SUN_CKM_RC4,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_cipher_mechs_tab[8].me_threshold = kcf_rc4_threshold;
+
+
+ /* 4 HMACs */
+ (void) strncpy(kcf_mac_mechs_tab[0].me_name, SUN_CKM_MD5_HMAC,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_mac_mechs_tab[0].me_threshold = kcf_md5_threshold;
+
+ (void) strncpy(kcf_mac_mechs_tab[1].me_name, SUN_CKM_MD5_HMAC_GENERAL,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_mac_mechs_tab[1].me_threshold = kcf_md5_threshold;
+
+ (void) strncpy(kcf_mac_mechs_tab[2].me_name, SUN_CKM_SHA1_HMAC,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_mac_mechs_tab[2].me_threshold = kcf_sha1_threshold;
+
+ (void) strncpy(kcf_mac_mechs_tab[3].me_name, SUN_CKM_SHA1_HMAC_GENERAL,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_mac_mechs_tab[3].me_threshold = kcf_sha1_threshold;
+
+
+ /* 1 random number generation pseudo mechanism */
+ (void) strncpy(kcf_misc_mechs_tab[0].me_name, SUN_RANDOM,
+ CRYPTO_MAX_MECH_NAME);
+
+ kcf_mech_hash = mod_hash_create_strhash_nodtr("kcf mech2id hash",
+ kcf_mech_hash_size, mod_hash_null_valdtor);
+
+ for (class = KCF_FIRST_OPSCLASS; class <= KCF_LAST_OPSCLASS; class++) {
+ max = kcf_mech_tabs_tab[class].met_size;
+ me_tab = kcf_mech_tabs_tab[class].met_tab;
+ for (i = 0; i < max; i++) {
+ mutex_init(&(me_tab[i].me_mutex), NULL,
+ MUTEX_DEFAULT, NULL);
+ if (me_tab[i].me_name[0] != 0) {
+ me_tab[i].me_mechid = KCF_MECHID(class, i);
+ (void) mod_hash_insert(kcf_mech_hash,
+ (mod_hash_key_t)me_tab[i].me_name,
+ (mod_hash_val_t)&(me_tab[i].me_mechid));
+ }
+ }
+ }
+}
+
+/*
+ * kcf_create_mech_entry()
+ *
+ * Arguments:
+ * . The class of mechanism.
+ * . the name of the new mechanism.
+ *
+ * Description:
+ * Creates a new mech_entry for a mechanism not yet known to the
+ * framework.
+ * This routine is called by kcf_add_mech_provider, which is
+ * in turn invoked for each mechanism supported by a provider.
+ * The'class' argument depends on the crypto_func_group_t bitmask
+ * in the registering provider's mech_info struct for this mechanism.
+ * When there is ambiguity in the mapping between the crypto_func_group_t
+ * and a class (dual ops, ...) the KCF_MISC_CLASS should be used.
+ *
+ * Context:
+ * User context only.
+ *
+ * Returns:
+ * KCF_INVALID_MECH_CLASS or KCF_INVALID_MECH_NAME if the class or
+ * the mechname is bogus.
+ * KCF_MECH_TAB_FULL when there is no room left in the mech. tabs.
+ * KCF_SUCCESS otherwise.
+ */
+static int
+kcf_create_mech_entry(kcf_ops_class_t class, char *mechname)
+{
+ crypto_mech_type_t mt;
+ kcf_mech_entry_t *me_tab;
+ int i = 0, size;
+
+ if ((class < KCF_FIRST_OPSCLASS) || (class > KCF_LAST_OPSCLASS))
+ return (KCF_INVALID_MECH_CLASS);
+
+ if ((mechname == NULL) || (mechname[0] == 0))
+ return (KCF_INVALID_MECH_NAME);
+ /*
+ * First check if the mechanism is already in one of the tables.
+ * The mech_entry could be in another class.
+ */
+ mutex_enter(&kcf_mech_tabs_lock);
+ mt = kcf_mech_hash_find(mechname);
+ if (mt != CRYPTO_MECH_INVALID) {
+ /* Nothing to do, regardless the suggested class. */
+ mutex_exit(&kcf_mech_tabs_lock);
+ return (KCF_SUCCESS);
+ }
+ /* Now take the next unused mech entry in the class's tab */
+ me_tab = kcf_mech_tabs_tab[class].met_tab;
+ size = kcf_mech_tabs_tab[class].met_size;
+
+ while (i < size) {
+ mutex_enter(&(me_tab[i].me_mutex));
+ if (me_tab[i].me_name[0] == 0) {
+ /* Found an empty spot */
+ (void) strncpy(me_tab[i].me_name, mechname,
+ CRYPTO_MAX_MECH_NAME);
+ me_tab[i].me_name[CRYPTO_MAX_MECH_NAME-1] = '\0';
+ me_tab[i].me_mechid = KCF_MECHID(class, i);
+ /*
+ * No a-priori information about the new mechanism, so
+ * the threshold is set to zero.
+ */
+ me_tab[i].me_threshold = 0;
+
+ mutex_exit(&(me_tab[i].me_mutex));
+ /* Add the new mechanism to the hash table */
+ (void) mod_hash_insert(kcf_mech_hash,
+ (mod_hash_key_t)me_tab[i].me_name,
+ (mod_hash_val_t)&(me_tab[i].me_mechid));
+ break;
+ }
+ mutex_exit(&(me_tab[i].me_mutex));
+ i++;
+ }
+
+ mutex_exit(&kcf_mech_tabs_lock);
+
+ if (i == size) {
+ return (KCF_MECH_TAB_FULL);
+ }
+
+ return (KCF_SUCCESS);
+}
+
+/*
+ * kcf_add_mech_provider()
+ *
+ * Arguments:
+ * . An index in to the provider mechanism array
+ * . A pointer to the provider descriptor
+ * . A storage for the kcf_prov_mech_desc_t the entry was added at.
+ *
+ * Description:
+ * Adds a new provider of a mechanism to the mechanism's mech_entry
+ * chain.
+ *
+ * Context:
+ * User context only.
+ *
+ * Returns
+ * KCF_SUCCESS on success
+ * KCF_MECH_TAB_FULL otherwise.
+ */
+int
+kcf_add_mech_provider(short mech_indx,
+ kcf_provider_desc_t *prov_desc, kcf_prov_mech_desc_t **pmdpp)
+{
+ int error;
+ kcf_mech_entry_t *mech_entry = NULL;
+ crypto_mech_info_t *mech_info;
+ crypto_mech_type_t kcf_mech_type, mt;
+ kcf_prov_mech_desc_t *prov_mech, *prov_mech2;
+ crypto_func_group_t simple_fg_mask, dual_fg_mask;
+ crypto_mech_info_t *dmi;
+ crypto_mech_info_list_t *mil, *mil2;
+ kcf_mech_entry_t *me;
+ int i;
+
+ ASSERT(prov_desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ mech_info = &prov_desc->pd_mechanisms[mech_indx];
+
+ /*
+ * A mechanism belongs to exactly one mechanism table.
+ * Find the class corresponding to the function group flag of
+ * the mechanism.
+ */
+ kcf_mech_type = kcf_mech_hash_find(mech_info->cm_mech_name);
+ if (kcf_mech_type == CRYPTO_MECH_INVALID) {
+ crypto_func_group_t fg = mech_info->cm_func_group_mask;
+ kcf_ops_class_t class;
+
+ if (fg & CRYPTO_FG_DIGEST || fg & CRYPTO_FG_DIGEST_ATOMIC)
+ class = KCF_DIGEST_CLASS;
+ else if (fg & CRYPTO_FG_ENCRYPT || fg & CRYPTO_FG_DECRYPT ||
+ fg & CRYPTO_FG_ENCRYPT_ATOMIC ||
+ fg & CRYPTO_FG_DECRYPT_ATOMIC)
+ class = KCF_CIPHER_CLASS;
+ else if (fg & CRYPTO_FG_MAC || fg & CRYPTO_FG_MAC_ATOMIC)
+ class = KCF_MAC_CLASS;
+ else if (fg & CRYPTO_FG_SIGN || fg & CRYPTO_FG_VERIFY ||
+ fg & CRYPTO_FG_SIGN_ATOMIC ||
+ fg & CRYPTO_FG_VERIFY_ATOMIC ||
+ fg & CRYPTO_FG_SIGN_RECOVER ||
+ fg & CRYPTO_FG_VERIFY_RECOVER)
+ class = KCF_SIGN_CLASS;
+ else if (fg & CRYPTO_FG_GENERATE ||
+ fg & CRYPTO_FG_GENERATE_KEY_PAIR ||
+ fg & CRYPTO_FG_WRAP || fg & CRYPTO_FG_UNWRAP ||
+ fg & CRYPTO_FG_DERIVE)
+ class = KCF_KEYOPS_CLASS;
+ else
+ class = KCF_MISC_CLASS;
+
+ /*
+ * Attempt to create a new mech_entry for the specified
+ * mechanism. kcf_create_mech_entry() can handle the case
+ * where such an entry already exists.
+ */
+ if ((error = kcf_create_mech_entry(class,
+ mech_info->cm_mech_name)) != KCF_SUCCESS) {
+ return (error);
+ }
+ /* get the KCF mech type that was assigned to the mechanism */
+ kcf_mech_type = kcf_mech_hash_find(mech_info->cm_mech_name);
+ ASSERT(kcf_mech_type != CRYPTO_MECH_INVALID);
+ }
+
+ error = kcf_get_mech_entry(kcf_mech_type, &mech_entry);
+ ASSERT(error == KCF_SUCCESS);
+
+ /* allocate and initialize new kcf_prov_mech_desc */
+ prov_mech = kmem_zalloc(sizeof (kcf_prov_mech_desc_t), KM_SLEEP);
+ bcopy(mech_info, &prov_mech->pm_mech_info, sizeof (crypto_mech_info_t));
+ prov_mech->pm_prov_desc = prov_desc;
+ prov_desc->pd_mech_indx[KCF_MECH2CLASS(kcf_mech_type)]
+ [KCF_MECH2INDEX(kcf_mech_type)] = mech_indx;
+
+ KCF_PROV_REFHOLD(prov_desc);
+ KCF_PROV_IREFHOLD(prov_desc);
+
+ dual_fg_mask = mech_info->cm_func_group_mask & CRYPTO_FG_DUAL_MASK;
+
+ if (dual_fg_mask == ((crypto_func_group_t)0))
+ goto add_entry;
+
+ simple_fg_mask = (mech_info->cm_func_group_mask &
+ CRYPTO_FG_SIMPLEOP_MASK) | CRYPTO_FG_RANDOM;
+
+ for (i = 0; i < prov_desc->pd_mech_list_count; i++) {
+ dmi = &prov_desc->pd_mechanisms[i];
+
+ /* skip self */
+ if (dmi->cm_mech_number == mech_info->cm_mech_number)
+ continue;
+
+ /* skip if not a dual operation mechanism */
+ if (!(dmi->cm_func_group_mask & dual_fg_mask) ||
+ (dmi->cm_func_group_mask & simple_fg_mask))
+ continue;
+
+ mt = kcf_mech_hash_find(dmi->cm_mech_name);
+ if (mt == CRYPTO_MECH_INVALID)
+ continue;
+
+ if (kcf_get_mech_entry(mt, &me) != KCF_SUCCESS)
+ continue;
+
+ mil = kmem_zalloc(sizeof (*mil), KM_SLEEP);
+ mil2 = kmem_zalloc(sizeof (*mil2), KM_SLEEP);
+
+ /*
+ * Ignore hard-coded entries in the mech table
+ * if the provider hasn't registered.
+ */
+ mutex_enter(&me->me_mutex);
+ if (me->me_hw_prov_chain == NULL && me->me_sw_prov == NULL) {
+ mutex_exit(&me->me_mutex);
+ kmem_free(mil, sizeof (*mil));
+ kmem_free(mil2, sizeof (*mil2));
+ continue;
+ }
+
+ /*
+ * Add other dual mechanisms that have registered
+ * with the framework to this mechanism's
+ * cross-reference list.
+ */
+ mil->ml_mech_info = *dmi; /* struct assignment */
+ mil->ml_kcf_mechid = mt;
+
+ /* add to head of list */
+ mil->ml_next = prov_mech->pm_mi_list;
+ prov_mech->pm_mi_list = mil;
+
+ if (prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER)
+ prov_mech2 = me->me_hw_prov_chain;
+ else
+ prov_mech2 = me->me_sw_prov;
+
+ if (prov_mech2 == NULL) {
+ kmem_free(mil2, sizeof (*mil2));
+ mutex_exit(&me->me_mutex);
+ continue;
+ }
+
+ /*
+ * Update all other cross-reference lists by
+ * adding this new mechanism.
+ */
+ while (prov_mech2 != NULL) {
+ if (prov_mech2->pm_prov_desc == prov_desc) {
+ /* struct assignment */
+ mil2->ml_mech_info = *mech_info;
+ mil2->ml_kcf_mechid = kcf_mech_type;
+
+ /* add to head of list */
+ mil2->ml_next = prov_mech2->pm_mi_list;
+ prov_mech2->pm_mi_list = mil2;
+ break;
+ }
+ prov_mech2 = prov_mech2->pm_next;
+ }
+ if (prov_mech2 == NULL)
+ kmem_free(mil2, sizeof (*mil2));
+
+ mutex_exit(&me->me_mutex);
+ }
+
+add_entry:
+ /*
+ * Add new kcf_prov_mech_desc at the front of HW providers
+ * chain.
+ */
+ switch (prov_desc->pd_prov_type) {
+
+ case CRYPTO_HW_PROVIDER:
+ mutex_enter(&mech_entry->me_mutex);
+ prov_mech->pm_me = mech_entry;
+ prov_mech->pm_next = mech_entry->me_hw_prov_chain;
+ mech_entry->me_hw_prov_chain = prov_mech;
+ mech_entry->me_num_hwprov++;
+ mutex_exit(&mech_entry->me_mutex);
+ break;
+
+ case CRYPTO_SW_PROVIDER:
+ mutex_enter(&mech_entry->me_mutex);
+ if (mech_entry->me_sw_prov != NULL) {
+ /*
+ * There is already a SW provider for this mechanism.
+ * Since we allow only one SW provider per mechanism,
+ * report this condition.
+ */
+ cmn_err(CE_WARN, "The cryptographic software provider "
+ "\"%s\" will not be used for %s. The provider "
+ "\"%s\" will be used for this mechanism "
+ "instead.", prov_desc->pd_description,
+ mech_info->cm_mech_name,
+ mech_entry->me_sw_prov->pm_prov_desc->
+ pd_description);
+ KCF_PROV_REFRELE(prov_desc);
+ kmem_free(prov_mech, sizeof (kcf_prov_mech_desc_t));
+ prov_mech = NULL;
+ } else {
+ /*
+ * Set the provider as the software provider for
+ * this mechanism.
+ */
+ mech_entry->me_sw_prov = prov_mech;
+
+ /* We'll wrap around after 4 billion registrations! */
+ mech_entry->me_gen_swprov = kcf_gen_swprov++;
+ }
+ mutex_exit(&mech_entry->me_mutex);
+ break;
+ default:
+ break;
+ }
+
+ *pmdpp = prov_mech;
+
+ return (KCF_SUCCESS);
+}
+
+/*
+ * kcf_remove_mech_provider()
+ *
+ * Arguments:
+ * . mech_name: the name of the mechanism.
+ * . prov_desc: The provider descriptor
+ *
+ * Description:
+ * Removes a provider from chain of provider descriptors.
+ * The provider is made unavailable to kernel consumers for the specified
+ * mechanism.
+ *
+ * Context:
+ * User context only.
+ */
+void
+kcf_remove_mech_provider(char *mech_name, kcf_provider_desc_t *prov_desc)
+{
+ crypto_mech_type_t mech_type;
+ kcf_prov_mech_desc_t *prov_mech = NULL, *prov_chain;
+ kcf_prov_mech_desc_t **prev_entry_next;
+ kcf_mech_entry_t *mech_entry;
+ crypto_mech_info_list_t *mil, *mil2, *next, **prev_next;
+
+ ASSERT(prov_desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER);
+
+ /* get the KCF mech type that was assigned to the mechanism */
+ if ((mech_type = kcf_mech_hash_find(mech_name)) ==
+ CRYPTO_MECH_INVALID) {
+ /*
+ * Provider was not allowed for this mech due to policy or
+ * configuration.
+ */
+ return;
+ }
+
+ /* get a ptr to the mech_entry that was created */
+ if (kcf_get_mech_entry(mech_type, &mech_entry) != KCF_SUCCESS) {
+ /*
+ * Provider was not allowed for this mech due to policy or
+ * configuration.
+ */
+ return;
+ }
+
+ mutex_enter(&mech_entry->me_mutex);
+
+ switch (prov_desc->pd_prov_type) {
+
+ case CRYPTO_HW_PROVIDER:
+ /* find the provider in the mech_entry chain */
+ prev_entry_next = &mech_entry->me_hw_prov_chain;
+ prov_mech = mech_entry->me_hw_prov_chain;
+ while (prov_mech != NULL &&
+ prov_mech->pm_prov_desc != prov_desc) {
+ prev_entry_next = &prov_mech->pm_next;
+ prov_mech = prov_mech->pm_next;
+ }
+
+ if (prov_mech == NULL) {
+ /* entry not found, simply return */
+ mutex_exit(&mech_entry->me_mutex);
+ return;
+ }
+
+ /* remove provider entry from mech_entry chain */
+ *prev_entry_next = prov_mech->pm_next;
+ ASSERT(mech_entry->me_num_hwprov > 0);
+ mech_entry->me_num_hwprov--;
+ break;
+
+ case CRYPTO_SW_PROVIDER:
+ if (mech_entry->me_sw_prov == NULL ||
+ mech_entry->me_sw_prov->pm_prov_desc != prov_desc) {
+ /* not the software provider for this mechanism */
+ mutex_exit(&mech_entry->me_mutex);
+ return;
+ }
+ prov_mech = mech_entry->me_sw_prov;
+ mech_entry->me_sw_prov = NULL;
+ break;
+ default:
+ break;
+ }
+
+ mutex_exit(&mech_entry->me_mutex);
+
+ /* Free the dual ops cross-reference lists */
+ mil = prov_mech->pm_mi_list;
+ while (mil != NULL) {
+ next = mil->ml_next;
+ if (kcf_get_mech_entry(mil->ml_kcf_mechid,
+ &mech_entry) != KCF_SUCCESS) {
+ mil = next;
+ continue;
+ }
+
+ mutex_enter(&mech_entry->me_mutex);
+ if (prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER)
+ prov_chain = mech_entry->me_hw_prov_chain;
+ else
+ prov_chain = mech_entry->me_sw_prov;
+
+ while (prov_chain != NULL) {
+ if (prov_chain->pm_prov_desc == prov_desc) {
+ prev_next = &prov_chain->pm_mi_list;
+ mil2 = prov_chain->pm_mi_list;
+ while (mil2 != NULL &&
+ mil2->ml_kcf_mechid != mech_type) {
+ prev_next = &mil2->ml_next;
+ mil2 = mil2->ml_next;
+ }
+ if (mil2 != NULL) {
+ *prev_next = mil2->ml_next;
+ kmem_free(mil2, sizeof (*mil2));
+ }
+ break;
+ }
+ prov_chain = prov_chain->pm_next;
+ }
+
+ mutex_exit(&mech_entry->me_mutex);
+ kmem_free(mil, sizeof (crypto_mech_info_list_t));
+ mil = next;
+ }
+
+ /* free entry */
+ KCF_PROV_REFRELE(prov_mech->pm_prov_desc);
+ KCF_PROV_IREFRELE(prov_mech->pm_prov_desc);
+ kmem_free(prov_mech, sizeof (kcf_prov_mech_desc_t));
+}
+
+/*
+ * kcf_get_mech_entry()
+ *
+ * Arguments:
+ * . The framework mechanism type
+ * . Storage for the mechanism entry
+ *
+ * Description:
+ * Retrieves the mechanism entry for the mech.
+ *
+ * Context:
+ * User and interrupt contexts.
+ *
+ * Returns:
+ * KCF_MECHANISM_XXX appropriate error code.
+ * KCF_SUCCESS otherwise.
+ */
+int
+kcf_get_mech_entry(crypto_mech_type_t mech_type, kcf_mech_entry_t **mep)
+{
+ kcf_ops_class_t class;
+ int index;
+ kcf_mech_entry_tab_t *me_tab;
+
+ ASSERT(mep != NULL);
+
+ class = KCF_MECH2CLASS(mech_type);
+
+ if ((class < KCF_FIRST_OPSCLASS) || (class > KCF_LAST_OPSCLASS)) {
+ /* the caller won't need to know it's an invalid class */
+ return (KCF_INVALID_MECH_NUMBER);
+ }
+
+ me_tab = &kcf_mech_tabs_tab[class];
+ index = KCF_MECH2INDEX(mech_type);
+
+ if ((index < 0) || (index >= me_tab->met_size)) {
+ return (KCF_INVALID_MECH_NUMBER);
+ }
+
+ *mep = &((me_tab->met_tab)[index]);
+
+ return (KCF_SUCCESS);
+}
+
+/* CURRENTLY UNSUPPORTED: attempting to load the module if it isn't found */
+/*
+ * Lookup the hash table for an entry that matches the mechname.
+ * If there are no hardware or software providers for the mechanism,
+ * but there is an unloaded software provider, this routine will attempt
+ * to load it.
+ *
+ * If the MOD_NOAUTOUNLOAD flag is not set, a software provider is
+ * in constant danger of being unloaded. For consumers that call
+ * crypto_mech2id() only once, the provider will not be reloaded
+ * if it becomes unloaded. If a provider gets loaded elsewhere
+ * without the MOD_NOAUTOUNLOAD flag being set, we set it now.
+ */
+crypto_mech_type_t
+crypto_mech2id_common(char *mechname, boolean_t load_module)
+{
+ crypto_mech_type_t mt = kcf_mech_hash_find(mechname);
+ return (mt);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+
+/*
+ * Utility routine to copy a buffer to a crypto_data structure.
+ */
+
+/*
+ * Utility routine to apply the command, 'cmd', to the
+ * data in the uio structure.
+ */
+int
+crypto_uio_data(crypto_data_t *data, uchar_t *buf, int len, cmd_type_t cmd,
+ void *digest_ctx, void (*update)(void))
+{
+ uio_t *uiop = data->cd_uio;
+ off_t offset = data->cd_offset;
+ size_t length = len;
+ uint_t vec_idx;
+ size_t cur_len;
+ uchar_t *datap;
+
+ ASSERT(data->cd_format == CRYPTO_DATA_UIO);
+ if (uiop->uio_segflg != UIO_SYSSPACE) {
+ return (CRYPTO_ARGUMENTS_BAD);
+ }
+
+ /*
+ * Jump to the first iovec containing data to be
+ * processed.
+ */
+ for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
+ offset >= uiop->uio_iov[vec_idx].iov_len;
+ offset -= uiop->uio_iov[vec_idx++].iov_len)
+ ;
+
+ if (vec_idx == uiop->uio_iovcnt) {
+ /*
+ * The caller specified an offset that is larger than
+ * the total size of the buffers it provided.
+ */
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ while (vec_idx < uiop->uio_iovcnt && length > 0) {
+ cur_len = MIN(uiop->uio_iov[vec_idx].iov_len -
+ offset, length);
+
+ datap = (uchar_t *)(uiop->uio_iov[vec_idx].iov_base +
+ offset);
+ switch (cmd) {
+ case COPY_FROM_DATA:
+ bcopy(datap, buf, cur_len);
+ buf += cur_len;
+ break;
+ case COPY_TO_DATA:
+ bcopy(buf, datap, cur_len);
+ buf += cur_len;
+ break;
+ case COMPARE_TO_DATA:
+ if (bcmp(datap, buf, cur_len))
+ return (CRYPTO_SIGNATURE_INVALID);
+ buf += cur_len;
+ break;
+ case MD5_DIGEST_DATA:
+ case SHA1_DIGEST_DATA:
+ case SHA2_DIGEST_DATA:
+ case GHASH_DATA:
+ return (CRYPTO_ARGUMENTS_BAD);
+ }
+
+ length -= cur_len;
+ vec_idx++;
+ offset = 0;
+ }
+
+ if (vec_idx == uiop->uio_iovcnt && length > 0) {
+ /*
+ * The end of the specified iovec's was reached but
+ * the length requested could not be processed.
+ */
+ switch (cmd) {
+ case COPY_TO_DATA:
+ data->cd_length = len;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ default:
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+ }
+
+ return (CRYPTO_SUCCESS);
+}
+
+int
+crypto_put_output_data(uchar_t *buf, crypto_data_t *output, int len)
+{
+ switch (output->cd_format) {
+ case CRYPTO_DATA_RAW:
+ if (output->cd_raw.iov_len < len) {
+ output->cd_length = len;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+ bcopy(buf, (uchar_t *)(output->cd_raw.iov_base +
+ output->cd_offset), len);
+ break;
+
+ case CRYPTO_DATA_UIO:
+ return (crypto_uio_data(output, buf, len,
+ COPY_TO_DATA, NULL, NULL));
+ default:
+ return (CRYPTO_ARGUMENTS_BAD);
+ }
+
+ return (CRYPTO_SUCCESS);
+}
+
+int
+crypto_update_iov(void *ctx, crypto_data_t *input, crypto_data_t *output,
+ int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
+ void (*copy_block)(uint8_t *, uint64_t *))
+{
+ common_ctx_t *common_ctx = ctx;
+ int rv;
+
+ if (input->cd_miscdata != NULL) {
+ copy_block((uint8_t *)input->cd_miscdata,
+ &common_ctx->cc_iv[0]);
+ }
+
+ if (input->cd_raw.iov_len < input->cd_length)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ rv = (cipher)(ctx, input->cd_raw.iov_base + input->cd_offset,
+ input->cd_length, (input == output) ? NULL : output);
+
+ return (rv);
+}
+
+int
+crypto_update_uio(void *ctx, crypto_data_t *input, crypto_data_t *output,
+ int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
+ void (*copy_block)(uint8_t *, uint64_t *))
+{
+ common_ctx_t *common_ctx = ctx;
+ uio_t *uiop = input->cd_uio;
+ off_t offset = input->cd_offset;
+ size_t length = input->cd_length;
+ uint_t vec_idx;
+ size_t cur_len;
+
+ if (input->cd_miscdata != NULL) {
+ copy_block((uint8_t *)input->cd_miscdata,
+ &common_ctx->cc_iv[0]);
+ }
+
+ if (input->cd_uio->uio_segflg != UIO_SYSSPACE) {
+ return (CRYPTO_ARGUMENTS_BAD);
+ }
+
+ /*
+ * Jump to the first iovec containing data to be
+ * processed.
+ */
+ for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
+ offset >= uiop->uio_iov[vec_idx].iov_len;
+ offset -= uiop->uio_iov[vec_idx++].iov_len)
+ ;
+ if (vec_idx == uiop->uio_iovcnt) {
+ /*
+ * The caller specified an offset that is larger than the
+ * total size of the buffers it provided.
+ */
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ /*
+ * Now process the iovecs.
+ */
+ while (vec_idx < uiop->uio_iovcnt && length > 0) {
+ cur_len = MIN(uiop->uio_iov[vec_idx].iov_len -
+ offset, length);
+
+ (cipher)(ctx, uiop->uio_iov[vec_idx].iov_base + offset,
+ cur_len, (input == output) ? NULL : output);
+
+ length -= cur_len;
+ vec_idx++;
+ offset = 0;
+ }
+
+ if (vec_idx == uiop->uio_iovcnt && length > 0) {
+ /*
+ * The end of the specified iovec's was reached but
+ * the length requested could not be processed, i.e.
+ * The caller requested to digest more data than it provided.
+ */
+
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ return (CRYPTO_SUCCESS);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * This file is part of the core Kernel Cryptographic Framework.
+ * It implements the management of tables of Providers. Entries to
+ * added and removed when cryptographic providers register with
+ * and unregister from the framework, respectively. The KCF scheduler
+ * and ioctl pseudo driver call this function to obtain the list
+ * of available providers.
+ *
+ * The provider table is indexed by crypto_provider_id_t. Each
+ * element of the table contains a pointer to a provider descriptor,
+ * or NULL if the entry is free.
+ *
+ * This file also implements helper functions to allocate and free
+ * provider descriptors.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/sched_impl.h>
+#include <sys/crypto/spi.h>
+
+#define KCF_MAX_PROVIDERS 512 /* max number of providers */
+
+/*
+ * Prov_tab is an array of providers which is updated when
+ * a crypto provider registers with kcf. The provider calls the
+ * SPI routine, crypto_register_provider(), which in turn calls
+ * kcf_prov_tab_add_provider().
+ *
+ * A provider unregisters by calling crypto_unregister_provider()
+ * which triggers the removal of the prov_tab entry.
+ * It also calls kcf_remove_mech_provider().
+ *
+ * prov_tab entries are not updated from kcf.conf or by cryptoadm(1M).
+ */
+static kcf_provider_desc_t **prov_tab = NULL;
+static kmutex_t prov_tab_mutex; /* ensure exclusive access to the table */
+static uint_t prov_tab_num = 0; /* number of providers in table */
+static uint_t prov_tab_max = KCF_MAX_PROVIDERS;
+
+void
+kcf_prov_tab_destroy(void)
+{
+ if (prov_tab) kmem_free(prov_tab, prov_tab_max *
+ sizeof (kcf_provider_desc_t *));
+}
+
+/*
+ * Initialize a mutex and the KCF providers table, prov_tab.
+ * The providers table is dynamically allocated with prov_tab_max entries.
+ * Called from kcf module _init().
+ */
+void
+kcf_prov_tab_init(void)
+{
+ mutex_init(&prov_tab_mutex, NULL, MUTEX_DEFAULT, NULL);
+
+ prov_tab = kmem_zalloc(prov_tab_max * sizeof (kcf_provider_desc_t *),
+ KM_SLEEP);
+}
+
+/*
+ * Add a provider to the provider table. If no free entry can be found
+ * for the new provider, returns CRYPTO_HOST_MEMORY. Otherwise, add
+ * the provider to the table, initialize the pd_prov_id field
+ * of the specified provider descriptor to the index in that table,
+ * and return CRYPTO_SUCCESS. Note that a REFHOLD is done on the
+ * provider when pointed to by a table entry.
+ */
+int
+kcf_prov_tab_add_provider(kcf_provider_desc_t *prov_desc)
+{
+ uint_t i;
+
+ ASSERT(prov_tab != NULL);
+
+ mutex_enter(&prov_tab_mutex);
+
+ /* find free slot in providers table */
+ for (i = 1; i < KCF_MAX_PROVIDERS && prov_tab[i] != NULL; i++)
+ ;
+ if (i == KCF_MAX_PROVIDERS) {
+ /* ran out of providers entries */
+ mutex_exit(&prov_tab_mutex);
+ cmn_err(CE_WARN, "out of providers entries");
+ return (CRYPTO_HOST_MEMORY);
+ }
+
+ /* initialize entry */
+ prov_tab[i] = prov_desc;
+ KCF_PROV_REFHOLD(prov_desc);
+ KCF_PROV_IREFHOLD(prov_desc);
+ prov_tab_num++;
+
+ mutex_exit(&prov_tab_mutex);
+
+ /* update provider descriptor */
+ prov_desc->pd_prov_id = i;
+
+ /*
+ * The KCF-private provider handle is defined as the internal
+ * provider id.
+ */
+ prov_desc->pd_kcf_prov_handle =
+ (crypto_kcf_provider_handle_t)prov_desc->pd_prov_id;
+
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Remove the provider specified by its id. A REFRELE is done on the
+ * corresponding provider descriptor before this function returns.
+ * Returns CRYPTO_UNKNOWN_PROVIDER if the provider id is not valid.
+ */
+int
+kcf_prov_tab_rem_provider(crypto_provider_id_t prov_id)
+{
+ kcf_provider_desc_t *prov_desc;
+
+ ASSERT(prov_tab != NULL);
+ ASSERT(prov_tab_num >= 0);
+
+ /*
+ * Validate provider id, since it can be specified by a 3rd-party
+ * provider.
+ */
+
+ mutex_enter(&prov_tab_mutex);
+ if (prov_id >= KCF_MAX_PROVIDERS ||
+ ((prov_desc = prov_tab[prov_id]) == NULL)) {
+ mutex_exit(&prov_tab_mutex);
+ return (CRYPTO_INVALID_PROVIDER_ID);
+ }
+ mutex_exit(&prov_tab_mutex);
+
+ /*
+ * The provider id must remain valid until the associated provider
+ * descriptor is freed. For this reason, we simply release our
+ * reference to the descriptor here. When the reference count
+ * reaches zero, kcf_free_provider_desc() will be invoked and
+ * the associated entry in the providers table will be released
+ * at that time.
+ */
+
+ KCF_PROV_REFRELE(prov_desc);
+ KCF_PROV_IREFRELE(prov_desc);
+
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Returns the provider descriptor corresponding to the specified
+ * provider id. A REFHOLD is done on the descriptor before it is
+ * returned to the caller. It is the responsibility of the caller
+ * to do a REFRELE once it is done with the provider descriptor.
+ */
+kcf_provider_desc_t *
+kcf_prov_tab_lookup(crypto_provider_id_t prov_id)
+{
+ kcf_provider_desc_t *prov_desc;
+
+ mutex_enter(&prov_tab_mutex);
+
+ prov_desc = prov_tab[prov_id];
+
+ if (prov_desc == NULL) {
+ mutex_exit(&prov_tab_mutex);
+ return (NULL);
+ }
+
+ KCF_PROV_REFHOLD(prov_desc);
+
+ mutex_exit(&prov_tab_mutex);
+
+ return (prov_desc);
+}
+
+static void
+allocate_ops_v1(crypto_ops_t *src, crypto_ops_t *dst, uint_t *mech_list_count)
+{
+ if (src->co_control_ops != NULL)
+ dst->co_control_ops = kmem_alloc(sizeof (crypto_control_ops_t),
+ KM_SLEEP);
+
+ if (src->co_digest_ops != NULL)
+ dst->co_digest_ops = kmem_alloc(sizeof (crypto_digest_ops_t),
+ KM_SLEEP);
+
+ if (src->co_cipher_ops != NULL)
+ dst->co_cipher_ops = kmem_alloc(sizeof (crypto_cipher_ops_t),
+ KM_SLEEP);
+
+ if (src->co_mac_ops != NULL)
+ dst->co_mac_ops = kmem_alloc(sizeof (crypto_mac_ops_t),
+ KM_SLEEP);
+
+ if (src->co_sign_ops != NULL)
+ dst->co_sign_ops = kmem_alloc(sizeof (crypto_sign_ops_t),
+ KM_SLEEP);
+
+ if (src->co_verify_ops != NULL)
+ dst->co_verify_ops = kmem_alloc(sizeof (crypto_verify_ops_t),
+ KM_SLEEP);
+
+ if (src->co_dual_ops != NULL)
+ dst->co_dual_ops = kmem_alloc(sizeof (crypto_dual_ops_t),
+ KM_SLEEP);
+
+ if (src->co_dual_cipher_mac_ops != NULL)
+ dst->co_dual_cipher_mac_ops = kmem_alloc(
+ sizeof (crypto_dual_cipher_mac_ops_t), KM_SLEEP);
+
+ if (src->co_random_ops != NULL) {
+ dst->co_random_ops = kmem_alloc(
+ sizeof (crypto_random_number_ops_t), KM_SLEEP);
+
+ /*
+ * Allocate storage to store the array of supported mechanisms
+ * specified by provider. We allocate extra mechanism storage
+ * if the provider has random_ops since we keep an internal
+ * mechanism, SUN_RANDOM, in this case.
+ */
+ (*mech_list_count)++;
+ }
+
+ if (src->co_session_ops != NULL)
+ dst->co_session_ops = kmem_alloc(sizeof (crypto_session_ops_t),
+ KM_SLEEP);
+
+ if (src->co_object_ops != NULL)
+ dst->co_object_ops = kmem_alloc(sizeof (crypto_object_ops_t),
+ KM_SLEEP);
+
+ if (src->co_key_ops != NULL)
+ dst->co_key_ops = kmem_alloc(sizeof (crypto_key_ops_t),
+ KM_SLEEP);
+
+ if (src->co_provider_ops != NULL)
+ dst->co_provider_ops = kmem_alloc(
+ sizeof (crypto_provider_management_ops_t), KM_SLEEP);
+
+ if (src->co_ctx_ops != NULL)
+ dst->co_ctx_ops = kmem_alloc(sizeof (crypto_ctx_ops_t),
+ KM_SLEEP);
+}
+
+static void
+allocate_ops_v2(crypto_ops_t *src, crypto_ops_t *dst)
+{
+ if (src->co_mech_ops != NULL)
+ dst->co_mech_ops = kmem_alloc(sizeof (crypto_mech_ops_t),
+ KM_SLEEP);
+}
+
+static void
+allocate_ops_v3(crypto_ops_t *src, crypto_ops_t *dst)
+{
+ if (src->co_nostore_key_ops != NULL)
+ dst->co_nostore_key_ops =
+ kmem_alloc(sizeof (crypto_nostore_key_ops_t), KM_SLEEP);
+}
+
+/*
+ * Allocate a provider descriptor. mech_list_count specifies the
+ * number of mechanisms supported by the providers, and is used
+ * to allocate storage for the mechanism table.
+ * This function may sleep while allocating memory, which is OK
+ * since it is invoked from user context during provider registration.
+ */
+kcf_provider_desc_t *
+kcf_alloc_provider_desc(crypto_provider_info_t *info)
+{
+ int i, j;
+ kcf_provider_desc_t *desc;
+ uint_t mech_list_count = info->pi_mech_list_count;
+ crypto_ops_t *src_ops = info->pi_ops_vector;
+
+ desc = kmem_zalloc(sizeof (kcf_provider_desc_t), KM_SLEEP);
+
+ /*
+ * pd_description serves two purposes
+ * - Appears as a blank padded PKCS#11 style string, that will be
+ * returned to applications in CK_SLOT_INFO.slotDescription.
+ * This means that we should not have a null character in the
+ * first CRYPTO_PROVIDER_DESCR_MAX_LEN bytes.
+ * - Appears as a null-terminated string that can be used by
+ * other kcf routines.
+ *
+ * So, we allocate enough room for one extra null terminator
+ * which keeps every one happy.
+ */
+ desc->pd_description = kmem_alloc(CRYPTO_PROVIDER_DESCR_MAX_LEN + 1,
+ KM_SLEEP);
+ (void) memset(desc->pd_description, ' ',
+ CRYPTO_PROVIDER_DESCR_MAX_LEN);
+ desc->pd_description[CRYPTO_PROVIDER_DESCR_MAX_LEN] = '\0';
+
+ /*
+ * Since the framework does not require the ops vector specified
+ * by the providers during registration to be persistent,
+ * KCF needs to allocate storage where copies of the ops
+ * vectors are copied.
+ */
+ desc->pd_ops_vector = kmem_zalloc(sizeof (crypto_ops_t), KM_SLEEP);
+
+ if (info->pi_provider_type != CRYPTO_LOGICAL_PROVIDER) {
+ allocate_ops_v1(src_ops, desc->pd_ops_vector, &mech_list_count);
+ if (info->pi_interface_version >= CRYPTO_SPI_VERSION_2)
+ allocate_ops_v2(src_ops, desc->pd_ops_vector);
+ if (info->pi_interface_version == CRYPTO_SPI_VERSION_3)
+ allocate_ops_v3(src_ops, desc->pd_ops_vector);
+ }
+
+ desc->pd_mech_list_count = mech_list_count;
+ desc->pd_mechanisms = kmem_zalloc(sizeof (crypto_mech_info_t) *
+ mech_list_count, KM_SLEEP);
+ for (i = 0; i < KCF_OPS_CLASSSIZE; i++)
+ for (j = 0; j < KCF_MAXMECHTAB; j++)
+ desc->pd_mech_indx[i][j] = KCF_INVALID_INDX;
+
+ desc->pd_prov_id = KCF_PROVID_INVALID;
+ desc->pd_state = KCF_PROV_ALLOCATED;
+
+ mutex_init(&desc->pd_lock, NULL, MUTEX_DEFAULT, NULL);
+ cv_init(&desc->pd_resume_cv, NULL, CV_DEFAULT, NULL);
+ cv_init(&desc->pd_remove_cv, NULL, CV_DEFAULT, NULL);
+
+ return (desc);
+}
+
+/*
+ * Called by KCF_PROV_REFRELE when a provider's reference count drops
+ * to zero. We free the descriptor when the last reference is released.
+ * However, for software providers, we do not free it when there is an
+ * unregister thread waiting. We signal that thread in this case and
+ * that thread is responsible for freeing the descriptor.
+ */
+void
+kcf_provider_zero_refcnt(kcf_provider_desc_t *desc)
+{
+ mutex_enter(&desc->pd_lock);
+ switch (desc->pd_prov_type) {
+ case CRYPTO_SW_PROVIDER:
+ if (desc->pd_state == KCF_PROV_REMOVED ||
+ desc->pd_state == KCF_PROV_DISABLED) {
+ desc->pd_state = KCF_PROV_FREED;
+ cv_broadcast(&desc->pd_remove_cv);
+ mutex_exit(&desc->pd_lock);
+ break;
+ }
+ /* FALLTHRU */
+
+ case CRYPTO_HW_PROVIDER:
+ case CRYPTO_LOGICAL_PROVIDER:
+ mutex_exit(&desc->pd_lock);
+ kcf_free_provider_desc(desc);
+ }
+}
+
+/*
+ * Free a provider descriptor.
+ */
+void
+kcf_free_provider_desc(kcf_provider_desc_t *desc)
+{
+ if (desc == NULL)
+ return;
+
+ mutex_enter(&prov_tab_mutex);
+ if (desc->pd_prov_id != KCF_PROVID_INVALID) {
+ /* release the associated providers table entry */
+ ASSERT(prov_tab[desc->pd_prov_id] != NULL);
+ prov_tab[desc->pd_prov_id] = NULL;
+ prov_tab_num--;
+ }
+ mutex_exit(&prov_tab_mutex);
+
+ /* free the kernel memory associated with the provider descriptor */
+
+ if (desc->pd_description != NULL)
+ kmem_free(desc->pd_description,
+ CRYPTO_PROVIDER_DESCR_MAX_LEN + 1);
+
+ if (desc->pd_ops_vector != NULL) {
+
+ if (desc->pd_ops_vector->co_control_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_control_ops,
+ sizeof (crypto_control_ops_t));
+
+ if (desc->pd_ops_vector->co_digest_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_digest_ops,
+ sizeof (crypto_digest_ops_t));
+
+ if (desc->pd_ops_vector->co_cipher_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_cipher_ops,
+ sizeof (crypto_cipher_ops_t));
+
+ if (desc->pd_ops_vector->co_mac_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_mac_ops,
+ sizeof (crypto_mac_ops_t));
+
+ if (desc->pd_ops_vector->co_sign_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_sign_ops,
+ sizeof (crypto_sign_ops_t));
+
+ if (desc->pd_ops_vector->co_verify_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_verify_ops,
+ sizeof (crypto_verify_ops_t));
+
+ if (desc->pd_ops_vector->co_dual_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_dual_ops,
+ sizeof (crypto_dual_ops_t));
+
+ if (desc->pd_ops_vector->co_dual_cipher_mac_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_dual_cipher_mac_ops,
+ sizeof (crypto_dual_cipher_mac_ops_t));
+
+ if (desc->pd_ops_vector->co_random_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_random_ops,
+ sizeof (crypto_random_number_ops_t));
+
+ if (desc->pd_ops_vector->co_session_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_session_ops,
+ sizeof (crypto_session_ops_t));
+
+ if (desc->pd_ops_vector->co_object_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_object_ops,
+ sizeof (crypto_object_ops_t));
+
+ if (desc->pd_ops_vector->co_key_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_key_ops,
+ sizeof (crypto_key_ops_t));
+
+ if (desc->pd_ops_vector->co_provider_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_provider_ops,
+ sizeof (crypto_provider_management_ops_t));
+
+ if (desc->pd_ops_vector->co_ctx_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_ctx_ops,
+ sizeof (crypto_ctx_ops_t));
+
+ if (desc->pd_ops_vector->co_mech_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_mech_ops,
+ sizeof (crypto_mech_ops_t));
+
+ if (desc->pd_ops_vector->co_nostore_key_ops != NULL)
+ kmem_free(desc->pd_ops_vector->co_nostore_key_ops,
+ sizeof (crypto_nostore_key_ops_t));
+
+ kmem_free(desc->pd_ops_vector, sizeof (crypto_ops_t));
+ }
+
+ if (desc->pd_mechanisms != NULL)
+ /* free the memory associated with the mechanism info's */
+ kmem_free(desc->pd_mechanisms, sizeof (crypto_mech_info_t) *
+ desc->pd_mech_list_count);
+
+ if (desc->pd_sched_info.ks_taskq != NULL)
+ taskq_destroy(desc->pd_sched_info.ks_taskq);
+
+ kmem_free(desc, sizeof (kcf_provider_desc_t));
+}
+
+/*
+ * Returns an array of hardware and logical provider descriptors,
+ * a.k.a the PKCS#11 slot list. A REFHOLD is done on each descriptor
+ * before the array is returned. The entire table can be freed by
+ * calling kcf_free_provider_tab().
+ */
+int
+kcf_get_slot_list(uint_t *count, kcf_provider_desc_t ***array,
+ boolean_t unverified)
+{
+ kcf_provider_desc_t *prov_desc;
+ kcf_provider_desc_t **p = NULL;
+ char *last;
+ uint_t cnt = 0;
+ uint_t i, j;
+ int rval = CRYPTO_SUCCESS;
+ size_t n, final_size;
+
+ /* count the providers */
+ mutex_enter(&prov_tab_mutex);
+ for (i = 0; i < KCF_MAX_PROVIDERS; i++) {
+ if ((prov_desc = prov_tab[i]) != NULL &&
+ ((prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER &&
+ (prov_desc->pd_flags & CRYPTO_HIDE_PROVIDER) == 0) ||
+ prov_desc->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)) {
+ if (KCF_IS_PROV_USABLE(prov_desc) ||
+ (unverified && KCF_IS_PROV_UNVERIFIED(prov_desc))) {
+ cnt++;
+ }
+ }
+ }
+ mutex_exit(&prov_tab_mutex);
+
+ if (cnt == 0)
+ goto out;
+
+ n = cnt * sizeof (kcf_provider_desc_t *);
+again:
+ p = kmem_zalloc(n, KM_SLEEP);
+
+ /* pointer to last entry in the array */
+ last = (char *)&p[cnt-1];
+
+ mutex_enter(&prov_tab_mutex);
+ /* fill the slot list */
+ for (i = 0, j = 0; i < KCF_MAX_PROVIDERS; i++) {
+ if ((prov_desc = prov_tab[i]) != NULL &&
+ ((prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER &&
+ (prov_desc->pd_flags & CRYPTO_HIDE_PROVIDER) == 0) ||
+ prov_desc->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)) {
+ if (KCF_IS_PROV_USABLE(prov_desc) ||
+ (unverified && KCF_IS_PROV_UNVERIFIED(prov_desc))) {
+ if ((char *)&p[j] > last) {
+ mutex_exit(&prov_tab_mutex);
+ kcf_free_provider_tab(cnt, p);
+ n = n << 1;
+ cnt = cnt << 1;
+ goto again;
+ }
+ p[j++] = prov_desc;
+ KCF_PROV_REFHOLD(prov_desc);
+ }
+ }
+ }
+ mutex_exit(&prov_tab_mutex);
+
+ final_size = j * sizeof (kcf_provider_desc_t *);
+ cnt = j;
+ ASSERT(final_size <= n);
+
+ /* check if buffer we allocated is too large */
+ if (final_size < n) {
+ char *final_buffer = NULL;
+
+ if (final_size > 0) {
+ final_buffer = kmem_alloc(final_size, KM_SLEEP);
+ bcopy(p, final_buffer, final_size);
+ }
+ kmem_free(p, n);
+ p = (kcf_provider_desc_t **)final_buffer;
+ }
+out:
+ *count = cnt;
+ *array = p;
+ return (rval);
+}
+
+/*
+ * Free an array of hardware provider descriptors. A REFRELE
+ * is done on each descriptor before the table is freed.
+ */
+void
+kcf_free_provider_tab(uint_t count, kcf_provider_desc_t **array)
+{
+ kcf_provider_desc_t *prov_desc;
+ int i;
+
+ for (i = 0; i < count; i++) {
+ if ((prov_desc = array[i]) != NULL) {
+ KCF_PROV_REFRELE(prov_desc);
+ }
+ }
+ kmem_free(array, count * sizeof (kcf_provider_desc_t *));
+}
+
+/*
+ * Returns in the location pointed to by pd a pointer to the descriptor
+ * for the software provider for the specified mechanism.
+ * The provider descriptor is returned held and it is the caller's
+ * responsibility to release it when done. The mechanism entry
+ * is returned if the optional argument mep is non NULL.
+ *
+ * Returns one of the CRYPTO_ * error codes on failure, and
+ * CRYPTO_SUCCESS on success.
+ */
+int
+kcf_get_sw_prov(crypto_mech_type_t mech_type, kcf_provider_desc_t **pd,
+ kcf_mech_entry_t **mep, boolean_t log_warn)
+{
+ kcf_mech_entry_t *me;
+
+ /* get the mechanism entry for this mechanism */
+ if (kcf_get_mech_entry(mech_type, &me) != KCF_SUCCESS)
+ return (CRYPTO_MECHANISM_INVALID);
+
+ /*
+ * Get the software provider for this mechanism.
+ * Lock the mech_entry until we grab the 'pd'.
+ */
+ mutex_enter(&me->me_mutex);
+
+ if (me->me_sw_prov == NULL ||
+ (*pd = me->me_sw_prov->pm_prov_desc) == NULL) {
+ /* no SW provider for this mechanism */
+ if (log_warn)
+ cmn_err(CE_WARN, "no SW provider for \"%s\"\n",
+ me->me_name);
+ mutex_exit(&me->me_mutex);
+ return (CRYPTO_MECH_NOT_SUPPORTED);
+ }
+
+ KCF_PROV_REFHOLD(*pd);
+ mutex_exit(&me->me_mutex);
+
+ if (mep != NULL)
+ *mep = me;
+
+ return (CRYPTO_SUCCESS);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * This file contains the core framework routines for the
+ * kernel cryptographic framework. These routines are at the
+ * layer, between the kernel API/ioctls and the SPI.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/sched_impl.h>
+#include <sys/crypto/api.h>
+
+kcf_global_swq_t *gswq; /* Global software queue */
+
+/* Thread pool related variables */
+static kcf_pool_t *kcfpool; /* Thread pool of kcfd LWPs */
+int kcf_maxthreads = 2;
+int kcf_minthreads = 1;
+int kcf_thr_multiple = 2; /* Boot-time tunable for experimentation */
+static ulong_t kcf_idlethr_timeout;
+#define KCF_DEFAULT_THRTIMEOUT 60000000 /* 60 seconds */
+
+/* kmem caches used by the scheduler */
+static kmem_cache_t *kcf_sreq_cache;
+static kmem_cache_t *kcf_areq_cache;
+static kmem_cache_t *kcf_context_cache;
+
+/* Global request ID table */
+static kcf_reqid_table_t *kcf_reqid_table[REQID_TABLES];
+
+/* KCF stats. Not protected. */
+static kcf_stats_t kcf_ksdata = {
+ { "total threads in pool", KSTAT_DATA_UINT32},
+ { "idle threads in pool", KSTAT_DATA_UINT32},
+ { "min threads in pool", KSTAT_DATA_UINT32},
+ { "max threads in pool", KSTAT_DATA_UINT32},
+ { "requests in gswq", KSTAT_DATA_UINT32},
+ { "max requests in gswq", KSTAT_DATA_UINT32},
+ { "threads for HW taskq", KSTAT_DATA_UINT32},
+ { "minalloc for HW taskq", KSTAT_DATA_UINT32},
+ { "maxalloc for HW taskq", KSTAT_DATA_UINT32}
+};
+
+static kstat_t *kcf_misc_kstat = NULL;
+ulong_t kcf_swprov_hndl = 0;
+
+static kcf_areq_node_t *kcf_areqnode_alloc(kcf_provider_desc_t *,
+ kcf_context_t *, crypto_call_req_t *, kcf_req_params_t *, boolean_t);
+static int kcf_disp_sw_request(kcf_areq_node_t *);
+static void process_req_hwp(void *);
+static int kcf_enqueue(kcf_areq_node_t *);
+static void kcfpool_alloc(void);
+static void kcf_reqid_delete(kcf_areq_node_t *areq);
+static crypto_req_id_t kcf_reqid_insert(kcf_areq_node_t *areq);
+static int kcf_misc_kstat_update(kstat_t *ksp, int rw);
+
+/*
+ * Create a new context.
+ */
+crypto_ctx_t *
+kcf_new_ctx(crypto_call_req_t *crq, kcf_provider_desc_t *pd,
+ crypto_session_id_t sid)
+{
+ crypto_ctx_t *ctx;
+ kcf_context_t *kcf_ctx;
+
+ kcf_ctx = kmem_cache_alloc(kcf_context_cache,
+ (crq == NULL) ? KM_SLEEP : KM_NOSLEEP);
+ if (kcf_ctx == NULL)
+ return (NULL);
+
+ /* initialize the context for the consumer */
+ kcf_ctx->kc_refcnt = 1;
+ kcf_ctx->kc_req_chain_first = NULL;
+ kcf_ctx->kc_req_chain_last = NULL;
+ kcf_ctx->kc_secondctx = NULL;
+ KCF_PROV_REFHOLD(pd);
+ kcf_ctx->kc_prov_desc = pd;
+ kcf_ctx->kc_sw_prov_desc = NULL;
+ kcf_ctx->kc_mech = NULL;
+
+ ctx = &kcf_ctx->kc_glbl_ctx;
+ ctx->cc_provider = pd->pd_prov_handle;
+ ctx->cc_session = sid;
+ ctx->cc_provider_private = NULL;
+ ctx->cc_framework_private = (void *)kcf_ctx;
+ ctx->cc_flags = 0;
+ ctx->cc_opstate = NULL;
+
+ return (ctx);
+}
+
+/*
+ * Allocate a new async request node.
+ *
+ * ictx - Framework private context pointer
+ * crq - Has callback function and argument. Should be non NULL.
+ * req - The parameters to pass to the SPI
+ */
+static kcf_areq_node_t *
+kcf_areqnode_alloc(kcf_provider_desc_t *pd, kcf_context_t *ictx,
+ crypto_call_req_t *crq, kcf_req_params_t *req, boolean_t isdual)
+{
+ kcf_areq_node_t *arptr, *areq;
+
+ ASSERT(crq != NULL);
+ arptr = kmem_cache_alloc(kcf_areq_cache, KM_NOSLEEP);
+ if (arptr == NULL)
+ return (NULL);
+
+ arptr->an_state = REQ_ALLOCATED;
+ arptr->an_reqarg = *crq;
+ arptr->an_params = *req;
+ arptr->an_context = ictx;
+ arptr->an_isdual = isdual;
+
+ arptr->an_next = arptr->an_prev = NULL;
+ KCF_PROV_REFHOLD(pd);
+ arptr->an_provider = pd;
+ arptr->an_tried_plist = NULL;
+ arptr->an_refcnt = 1;
+ arptr->an_idnext = arptr->an_idprev = NULL;
+
+ /*
+ * Requests for context-less operations do not use the
+ * fields - an_is_my_turn, and an_ctxchain_next.
+ */
+ if (ictx == NULL)
+ return (arptr);
+
+ KCF_CONTEXT_REFHOLD(ictx);
+ /*
+ * Chain this request to the context.
+ */
+ mutex_enter(&ictx->kc_in_use_lock);
+ arptr->an_ctxchain_next = NULL;
+ if ((areq = ictx->kc_req_chain_last) == NULL) {
+ arptr->an_is_my_turn = B_TRUE;
+ ictx->kc_req_chain_last =
+ ictx->kc_req_chain_first = arptr;
+ } else {
+ ASSERT(ictx->kc_req_chain_first != NULL);
+ arptr->an_is_my_turn = B_FALSE;
+ /* Insert the new request to the end of the chain. */
+ areq->an_ctxchain_next = arptr;
+ ictx->kc_req_chain_last = arptr;
+ }
+ mutex_exit(&ictx->kc_in_use_lock);
+
+ return (arptr);
+}
+
+/*
+ * Queue the request node and do one of the following:
+ * - If there is an idle thread signal it to run.
+ * - If there is no idle thread and max running threads is not
+ * reached, signal the creator thread for more threads.
+ *
+ * If the two conditions above are not met, we don't need to do
+ * any thing. The request will be picked up by one of the
+ * worker threads when it becomes available.
+ */
+static int
+kcf_disp_sw_request(kcf_areq_node_t *areq)
+{
+ int err;
+ int cnt = 0;
+
+ if ((err = kcf_enqueue(areq)) != 0)
+ return (err);
+
+ if (kcfpool->kp_idlethreads > 0) {
+ /* Signal an idle thread to run */
+ mutex_enter(&gswq->gs_lock);
+ cv_signal(&gswq->gs_cv);
+ mutex_exit(&gswq->gs_lock);
+
+ return (CRYPTO_QUEUED);
+ }
+
+ /*
+ * We keep the number of running threads to be at
+ * kcf_minthreads to reduce gs_lock contention.
+ */
+ cnt = kcf_minthreads -
+ (kcfpool->kp_threads - kcfpool->kp_blockedthreads);
+ if (cnt > 0) {
+ /*
+ * The following ensures the number of threads in pool
+ * does not exceed kcf_maxthreads.
+ */
+ cnt = MIN(cnt, kcf_maxthreads - (int)kcfpool->kp_threads);
+ if (cnt > 0) {
+ /* Signal the creator thread for more threads */
+ mutex_enter(&kcfpool->kp_user_lock);
+ if (!kcfpool->kp_signal_create_thread) {
+ kcfpool->kp_signal_create_thread = B_TRUE;
+ kcfpool->kp_nthrs = cnt;
+ cv_signal(&kcfpool->kp_user_cv);
+ }
+ mutex_exit(&kcfpool->kp_user_lock);
+ }
+ }
+
+ return (CRYPTO_QUEUED);
+}
+
+/*
+ * This routine is called by the taskq associated with
+ * each hardware provider. We notify the kernel consumer
+ * via the callback routine in case of CRYPTO_SUCCESS or
+ * a failure.
+ *
+ * A request can be of type kcf_areq_node_t or of type
+ * kcf_sreq_node_t.
+ */
+static void
+process_req_hwp(void *ireq)
+{
+ int error = 0;
+ crypto_ctx_t *ctx;
+ kcf_call_type_t ctype;
+ kcf_provider_desc_t *pd;
+ kcf_areq_node_t *areq = (kcf_areq_node_t *)ireq;
+ kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)ireq;
+
+ pd = ((ctype = GET_REQ_TYPE(ireq)) == CRYPTO_SYNCH) ?
+ sreq->sn_provider : areq->an_provider;
+
+ /*
+ * Wait if flow control is in effect for the provider. A
+ * CRYPTO_PROVIDER_READY or CRYPTO_PROVIDER_FAILED
+ * notification will signal us. We also get signaled if
+ * the provider is unregistering.
+ */
+ if (pd->pd_state == KCF_PROV_BUSY) {
+ mutex_enter(&pd->pd_lock);
+ while (pd->pd_state == KCF_PROV_BUSY)
+ cv_wait(&pd->pd_resume_cv, &pd->pd_lock);
+ mutex_exit(&pd->pd_lock);
+ }
+
+ /*
+ * Bump the internal reference count while the request is being
+ * processed. This is how we know when it's safe to unregister
+ * a provider. This step must precede the pd_state check below.
+ */
+ KCF_PROV_IREFHOLD(pd);
+
+ /*
+ * Fail the request if the provider has failed. We return a
+ * recoverable error and the notified clients attempt any
+ * recovery. For async clients this is done in kcf_aop_done()
+ * and for sync clients it is done in the k-api routines.
+ */
+ if (pd->pd_state >= KCF_PROV_FAILED) {
+ error = CRYPTO_DEVICE_ERROR;
+ goto bail;
+ }
+
+ if (ctype == CRYPTO_SYNCH) {
+ mutex_enter(&sreq->sn_lock);
+ sreq->sn_state = REQ_INPROGRESS;
+ mutex_exit(&sreq->sn_lock);
+
+ ctx = sreq->sn_context ? &sreq->sn_context->kc_glbl_ctx : NULL;
+ error = common_submit_request(sreq->sn_provider, ctx,
+ sreq->sn_params, sreq);
+ } else {
+ kcf_context_t *ictx;
+ ASSERT(ctype == CRYPTO_ASYNCH);
+
+ /*
+ * We are in the per-hardware provider thread context and
+ * hence can sleep. Note that the caller would have done
+ * a taskq_dispatch(..., TQ_NOSLEEP) and would have returned.
+ */
+ ctx = (ictx = areq->an_context) ? &ictx->kc_glbl_ctx : NULL;
+
+ mutex_enter(&areq->an_lock);
+ /*
+ * We need to maintain ordering for multi-part requests.
+ * an_is_my_turn is set to B_TRUE initially for a request
+ * when it is enqueued and there are no other requests
+ * for that context. It is set later from kcf_aop_done() when
+ * the request before us in the chain of requests for the
+ * context completes. We get signaled at that point.
+ */
+ if (ictx != NULL) {
+ ASSERT(ictx->kc_prov_desc == areq->an_provider);
+
+ while (areq->an_is_my_turn == B_FALSE) {
+ cv_wait(&areq->an_turn_cv, &areq->an_lock);
+ }
+ }
+ areq->an_state = REQ_INPROGRESS;
+ mutex_exit(&areq->an_lock);
+
+ error = common_submit_request(areq->an_provider, ctx,
+ &areq->an_params, areq);
+ }
+
+bail:
+ if (error == CRYPTO_QUEUED) {
+ /*
+ * The request is queued by the provider and we should
+ * get a crypto_op_notification() from the provider later.
+ * We notify the consumer at that time.
+ */
+ return;
+ } else { /* CRYPTO_SUCCESS or other failure */
+ KCF_PROV_IREFRELE(pd);
+ if (ctype == CRYPTO_SYNCH)
+ kcf_sop_done(sreq, error);
+ else
+ kcf_aop_done(areq, error);
+ }
+}
+
+/*
+ * This routine checks if a request can be retried on another
+ * provider. If true, mech1 is initialized to point to the mechanism
+ * structure. mech2 is also initialized in case of a dual operation. fg
+ * is initialized to the correct crypto_func_group_t bit flag. They are
+ * initialized by this routine, so that the caller can pass them to a
+ * kcf_get_mech_provider() or kcf_get_dual_provider() with no further change.
+ *
+ * We check that the request is for a init or atomic routine and that
+ * it is for one of the operation groups used from k-api .
+ */
+static boolean_t
+can_resubmit(kcf_areq_node_t *areq, crypto_mechanism_t **mech1,
+ crypto_mechanism_t **mech2, crypto_func_group_t *fg)
+{
+ kcf_req_params_t *params;
+ kcf_op_type_t optype;
+
+ params = &areq->an_params;
+ optype = params->rp_optype;
+
+ if (!(IS_INIT_OP(optype) || IS_ATOMIC_OP(optype)))
+ return (B_FALSE);
+
+ switch (params->rp_opgrp) {
+ case KCF_OG_DIGEST: {
+ kcf_digest_ops_params_t *dops = ¶ms->rp_u.digest_params;
+
+ dops->do_mech.cm_type = dops->do_framework_mechtype;
+ *mech1 = &dops->do_mech;
+ *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DIGEST :
+ CRYPTO_FG_DIGEST_ATOMIC;
+ break;
+ }
+
+ case KCF_OG_MAC: {
+ kcf_mac_ops_params_t *mops = ¶ms->rp_u.mac_params;
+
+ mops->mo_mech.cm_type = mops->mo_framework_mechtype;
+ *mech1 = &mops->mo_mech;
+ *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC :
+ CRYPTO_FG_MAC_ATOMIC;
+ break;
+ }
+
+ case KCF_OG_SIGN: {
+ kcf_sign_ops_params_t *sops = ¶ms->rp_u.sign_params;
+
+ sops->so_mech.cm_type = sops->so_framework_mechtype;
+ *mech1 = &sops->so_mech;
+ switch (optype) {
+ case KCF_OP_INIT:
+ *fg = CRYPTO_FG_SIGN;
+ break;
+ case KCF_OP_ATOMIC:
+ *fg = CRYPTO_FG_SIGN_ATOMIC;
+ break;
+ default:
+ ASSERT(optype == KCF_OP_SIGN_RECOVER_ATOMIC);
+ *fg = CRYPTO_FG_SIGN_RECOVER_ATOMIC;
+ }
+ break;
+ }
+
+ case KCF_OG_VERIFY: {
+ kcf_verify_ops_params_t *vops = ¶ms->rp_u.verify_params;
+
+ vops->vo_mech.cm_type = vops->vo_framework_mechtype;
+ *mech1 = &vops->vo_mech;
+ switch (optype) {
+ case KCF_OP_INIT:
+ *fg = CRYPTO_FG_VERIFY;
+ break;
+ case KCF_OP_ATOMIC:
+ *fg = CRYPTO_FG_VERIFY_ATOMIC;
+ break;
+ default:
+ ASSERT(optype == KCF_OP_VERIFY_RECOVER_ATOMIC);
+ *fg = CRYPTO_FG_VERIFY_RECOVER_ATOMIC;
+ }
+ break;
+ }
+
+ case KCF_OG_ENCRYPT: {
+ kcf_encrypt_ops_params_t *eops = ¶ms->rp_u.encrypt_params;
+
+ eops->eo_mech.cm_type = eops->eo_framework_mechtype;
+ *mech1 = &eops->eo_mech;
+ *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT :
+ CRYPTO_FG_ENCRYPT_ATOMIC;
+ break;
+ }
+
+ case KCF_OG_DECRYPT: {
+ kcf_decrypt_ops_params_t *dcrops = ¶ms->rp_u.decrypt_params;
+
+ dcrops->dop_mech.cm_type = dcrops->dop_framework_mechtype;
+ *mech1 = &dcrops->dop_mech;
+ *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DECRYPT :
+ CRYPTO_FG_DECRYPT_ATOMIC;
+ break;
+ }
+
+ case KCF_OG_ENCRYPT_MAC: {
+ kcf_encrypt_mac_ops_params_t *eops =
+ ¶ms->rp_u.encrypt_mac_params;
+
+ eops->em_encr_mech.cm_type = eops->em_framework_encr_mechtype;
+ *mech1 = &eops->em_encr_mech;
+ eops->em_mac_mech.cm_type = eops->em_framework_mac_mechtype;
+ *mech2 = &eops->em_mac_mech;
+ *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT_MAC :
+ CRYPTO_FG_ENCRYPT_MAC_ATOMIC;
+ break;
+ }
+
+ case KCF_OG_MAC_DECRYPT: {
+ kcf_mac_decrypt_ops_params_t *dops =
+ ¶ms->rp_u.mac_decrypt_params;
+
+ dops->md_mac_mech.cm_type = dops->md_framework_mac_mechtype;
+ *mech1 = &dops->md_mac_mech;
+ dops->md_decr_mech.cm_type = dops->md_framework_decr_mechtype;
+ *mech2 = &dops->md_decr_mech;
+ *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC_DECRYPT :
+ CRYPTO_FG_MAC_DECRYPT_ATOMIC;
+ break;
+ }
+
+ default:
+ return (B_FALSE);
+ }
+
+ return (B_TRUE);
+}
+
+/*
+ * This routine is called when a request to a provider has failed
+ * with a recoverable error. This routine tries to find another provider
+ * and dispatches the request to the new provider, if one is available.
+ * We reuse the request structure.
+ *
+ * A return value of NULL from kcf_get_mech_provider() indicates
+ * we have tried the last provider.
+ */
+static int
+kcf_resubmit_request(kcf_areq_node_t *areq)
+{
+ int error = CRYPTO_FAILED;
+ kcf_context_t *ictx;
+ kcf_provider_desc_t *old_pd;
+ kcf_provider_desc_t *new_pd;
+ crypto_mechanism_t *mech1 = NULL, *mech2 = NULL;
+ crypto_mech_type_t prov_mt1, prov_mt2;
+ crypto_func_group_t fg;
+
+ if (!can_resubmit(areq, &mech1, &mech2, &fg))
+ return (error);
+
+ old_pd = areq->an_provider;
+ /*
+ * Add old_pd to the list of providers already tried. We release
+ * the hold on old_pd (from the earlier kcf_get_mech_provider()) in
+ * kcf_free_triedlist().
+ */
+ if (kcf_insert_triedlist(&areq->an_tried_plist, old_pd,
+ KM_NOSLEEP) == NULL)
+ return (error);
+
+ if (mech1 && !mech2) {
+ new_pd = kcf_get_mech_provider(mech1->cm_type, NULL, &error,
+ areq->an_tried_plist, fg,
+ (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), 0);
+ } else {
+ ASSERT(mech1 != NULL && mech2 != NULL);
+
+ new_pd = kcf_get_dual_provider(mech1, mech2, NULL, &prov_mt1,
+ &prov_mt2, &error, areq->an_tried_plist, fg, fg,
+ (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), 0);
+ }
+
+ if (new_pd == NULL)
+ return (error);
+
+ /*
+ * We reuse the old context by resetting provider specific
+ * fields in it.
+ */
+ if ((ictx = areq->an_context) != NULL) {
+ crypto_ctx_t *ctx;
+
+ ASSERT(old_pd == ictx->kc_prov_desc);
+ KCF_PROV_REFRELE(ictx->kc_prov_desc);
+ KCF_PROV_REFHOLD(new_pd);
+ ictx->kc_prov_desc = new_pd;
+
+ ctx = &ictx->kc_glbl_ctx;
+ ctx->cc_provider = new_pd->pd_prov_handle;
+ ctx->cc_session = new_pd->pd_sid;
+ ctx->cc_provider_private = NULL;
+ }
+
+ /* We reuse areq. by resetting the provider and context fields. */
+ KCF_PROV_REFRELE(old_pd);
+ KCF_PROV_REFHOLD(new_pd);
+ areq->an_provider = new_pd;
+ mutex_enter(&areq->an_lock);
+ areq->an_state = REQ_WAITING;
+ mutex_exit(&areq->an_lock);
+
+ switch (new_pd->pd_prov_type) {
+ case CRYPTO_SW_PROVIDER:
+ error = kcf_disp_sw_request(areq);
+ break;
+
+ case CRYPTO_HW_PROVIDER: {
+ taskq_t *taskq = new_pd->pd_sched_info.ks_taskq;
+
+ if (taskq_dispatch(taskq, process_req_hwp, areq, TQ_NOSLEEP) ==
+ (taskqid_t)0) {
+ error = CRYPTO_HOST_MEMORY;
+ } else {
+ error = CRYPTO_QUEUED;
+ }
+
+ break;
+ default:
+ break;
+ }
+ }
+
+ return (error);
+}
+
+static inline int EMPTY_TASKQ(taskq_t *tq)
+{
+#ifdef _KERNEL
+ return (tq->tq_lowest_id == tq->tq_next_id);
+#else
+ return (tq->tq_task.tqent_next == &tq->tq_task || tq->tq_active == 0);
+#endif
+}
+
+/*
+ * Routine called by both ioctl and k-api. The consumer should
+ * bundle the parameters into a kcf_req_params_t structure. A bunch
+ * of macros are available in ops_impl.h for this bundling. They are:
+ *
+ * KCF_WRAP_DIGEST_OPS_PARAMS()
+ * KCF_WRAP_MAC_OPS_PARAMS()
+ * KCF_WRAP_ENCRYPT_OPS_PARAMS()
+ * KCF_WRAP_DECRYPT_OPS_PARAMS() ... etc.
+ *
+ * It is the caller's responsibility to free the ctx argument when
+ * appropriate. See the KCF_CONTEXT_COND_RELEASE macro for details.
+ */
+int
+kcf_submit_request(kcf_provider_desc_t *pd, crypto_ctx_t *ctx,
+ crypto_call_req_t *crq, kcf_req_params_t *params, boolean_t cont)
+{
+ int error = CRYPTO_SUCCESS;
+ kcf_areq_node_t *areq;
+ kcf_sreq_node_t *sreq;
+ kcf_context_t *kcf_ctx;
+ taskq_t *taskq = pd->pd_sched_info.ks_taskq;
+
+ kcf_ctx = ctx ? (kcf_context_t *)ctx->cc_framework_private : NULL;
+
+ /* Synchronous cases */
+ if (crq == NULL) {
+ switch (pd->pd_prov_type) {
+ case CRYPTO_SW_PROVIDER:
+ error = common_submit_request(pd, ctx, params,
+ KCF_RHNDL(KM_SLEEP));
+ break;
+
+ case CRYPTO_HW_PROVIDER:
+ /*
+ * Special case for CRYPTO_SYNCHRONOUS providers that
+ * never return a CRYPTO_QUEUED error. We skip any
+ * request allocation and call the SPI directly.
+ */
+ if ((pd->pd_flags & CRYPTO_SYNCHRONOUS) &&
+ EMPTY_TASKQ(taskq)) {
+ KCF_PROV_IREFHOLD(pd);
+ if (pd->pd_state == KCF_PROV_READY) {
+ error = common_submit_request(pd, ctx,
+ params, KCF_RHNDL(KM_SLEEP));
+ KCF_PROV_IREFRELE(pd);
+ ASSERT(error != CRYPTO_QUEUED);
+ break;
+ }
+ KCF_PROV_IREFRELE(pd);
+ }
+
+ sreq = kmem_cache_alloc(kcf_sreq_cache, KM_SLEEP);
+ sreq->sn_state = REQ_ALLOCATED;
+ sreq->sn_rv = CRYPTO_FAILED;
+ sreq->sn_params = params;
+
+ /*
+ * Note that we do not need to hold the context
+ * for synchronous case as the context will never
+ * become invalid underneath us. We do not need to hold
+ * the provider here either as the caller has a hold.
+ */
+ sreq->sn_context = kcf_ctx;
+ ASSERT(KCF_PROV_REFHELD(pd));
+ sreq->sn_provider = pd;
+
+ ASSERT(taskq != NULL);
+ /*
+ * Call the SPI directly if the taskq is empty and the
+ * provider is not busy, else dispatch to the taskq.
+ * Calling directly is fine as this is the synchronous
+ * case. This is unlike the asynchronous case where we
+ * must always dispatch to the taskq.
+ */
+ if (EMPTY_TASKQ(taskq) &&
+ pd->pd_state == KCF_PROV_READY) {
+ process_req_hwp(sreq);
+ } else {
+ /*
+ * We can not tell from taskq_dispatch() return
+ * value if we exceeded maxalloc. Hence the
+ * check here. Since we are allowed to wait in
+ * the synchronous case, we wait for the taskq
+ * to become empty.
+ */
+ if (taskq->tq_nalloc >= crypto_taskq_maxalloc) {
+ taskq_wait(taskq);
+ }
+
+ (void) taskq_dispatch(taskq, process_req_hwp,
+ sreq, TQ_SLEEP);
+ }
+
+ /*
+ * Wait for the notification to arrive,
+ * if the operation is not done yet.
+ * Bug# 4722589 will make the wait a cv_wait_sig().
+ */
+ mutex_enter(&sreq->sn_lock);
+ while (sreq->sn_state < REQ_DONE)
+ cv_wait(&sreq->sn_cv, &sreq->sn_lock);
+ mutex_exit(&sreq->sn_lock);
+
+ error = sreq->sn_rv;
+ kmem_cache_free(kcf_sreq_cache, sreq);
+
+ break;
+
+ default:
+ error = CRYPTO_FAILED;
+ break;
+ }
+
+ } else { /* Asynchronous cases */
+ switch (pd->pd_prov_type) {
+ case CRYPTO_SW_PROVIDER:
+ if (!(crq->cr_flag & CRYPTO_ALWAYS_QUEUE)) {
+ /*
+ * This case has less overhead since there is
+ * no switching of context.
+ */
+ error = common_submit_request(pd, ctx, params,
+ KCF_RHNDL(KM_NOSLEEP));
+ } else {
+ /*
+ * CRYPTO_ALWAYS_QUEUE is set. We need to
+ * queue the request and return.
+ */
+ areq = kcf_areqnode_alloc(pd, kcf_ctx, crq,
+ params, cont);
+ if (areq == NULL)
+ error = CRYPTO_HOST_MEMORY;
+ else {
+ if (!(crq->cr_flag
+ & CRYPTO_SKIP_REQID)) {
+ /*
+ * Set the request handle. This handle
+ * is used for any crypto_cancel_req(9f)
+ * calls from the consumer. We have to
+ * do this before dispatching the
+ * request.
+ */
+ crq->cr_reqid = kcf_reqid_insert(areq);
+ }
+
+ error = kcf_disp_sw_request(areq);
+ /*
+ * There is an error processing this
+ * request. Remove the handle and
+ * release the request structure.
+ */
+ if (error != CRYPTO_QUEUED) {
+ if (!(crq->cr_flag
+ & CRYPTO_SKIP_REQID))
+ kcf_reqid_delete(areq);
+ KCF_AREQ_REFRELE(areq);
+ }
+ }
+ }
+ break;
+
+ case CRYPTO_HW_PROVIDER:
+ /*
+ * We need to queue the request and return.
+ */
+ areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, params,
+ cont);
+ if (areq == NULL) {
+ error = CRYPTO_HOST_MEMORY;
+ goto done;
+ }
+
+ ASSERT(taskq != NULL);
+ /*
+ * We can not tell from taskq_dispatch() return
+ * value if we exceeded maxalloc. Hence the check
+ * here.
+ */
+ if (taskq->tq_nalloc >= crypto_taskq_maxalloc) {
+ error = CRYPTO_BUSY;
+ KCF_AREQ_REFRELE(areq);
+ goto done;
+ }
+
+ if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) {
+ /*
+ * Set the request handle. This handle is used
+ * for any crypto_cancel_req(9f) calls from the
+ * consumer. We have to do this before dispatching
+ * the request.
+ */
+ crq->cr_reqid = kcf_reqid_insert(areq);
+ }
+
+ if (taskq_dispatch(taskq,
+ process_req_hwp, areq, TQ_NOSLEEP) ==
+ (taskqid_t)0) {
+ error = CRYPTO_HOST_MEMORY;
+ if (!(crq->cr_flag & CRYPTO_SKIP_REQID))
+ kcf_reqid_delete(areq);
+ KCF_AREQ_REFRELE(areq);
+ } else {
+ error = CRYPTO_QUEUED;
+ }
+ break;
+
+ default:
+ error = CRYPTO_FAILED;
+ break;
+ }
+ }
+
+done:
+ return (error);
+}
+
+/*
+ * We're done with this framework context, so free it. Note that freeing
+ * framework context (kcf_context) frees the global context (crypto_ctx).
+ *
+ * The provider is responsible for freeing provider private context after a
+ * final or single operation and resetting the cc_provider_private field
+ * to NULL. It should do this before it notifies the framework of the
+ * completion. We still need to call KCF_PROV_FREE_CONTEXT to handle cases
+ * like crypto_cancel_ctx(9f).
+ */
+void
+kcf_free_context(kcf_context_t *kcf_ctx)
+{
+ kcf_provider_desc_t *pd = kcf_ctx->kc_prov_desc;
+ crypto_ctx_t *gctx = &kcf_ctx->kc_glbl_ctx;
+ kcf_context_t *kcf_secondctx = kcf_ctx->kc_secondctx;
+
+ /* Release the second context, if any */
+
+ if (kcf_secondctx != NULL)
+ KCF_CONTEXT_REFRELE(kcf_secondctx);
+
+ if (gctx->cc_provider_private != NULL) {
+ mutex_enter(&pd->pd_lock);
+ if (!KCF_IS_PROV_REMOVED(pd)) {
+ /*
+ * Increment the provider's internal refcnt so it
+ * doesn't unregister from the framework while
+ * we're calling the entry point.
+ */
+ KCF_PROV_IREFHOLD(pd);
+ mutex_exit(&pd->pd_lock);
+ (void) KCF_PROV_FREE_CONTEXT(pd, gctx);
+ KCF_PROV_IREFRELE(pd);
+ } else {
+ mutex_exit(&pd->pd_lock);
+ }
+ }
+
+ /* kcf_ctx->kc_prov_desc has a hold on pd */
+ KCF_PROV_REFRELE(kcf_ctx->kc_prov_desc);
+
+ /* check if this context is shared with a software provider */
+ if ((gctx->cc_flags & CRYPTO_INIT_OPSTATE) &&
+ kcf_ctx->kc_sw_prov_desc != NULL) {
+ KCF_PROV_REFRELE(kcf_ctx->kc_sw_prov_desc);
+ }
+
+ kmem_cache_free(kcf_context_cache, kcf_ctx);
+}
+
+/*
+ * Free the request after releasing all the holds.
+ */
+void
+kcf_free_req(kcf_areq_node_t *areq)
+{
+ KCF_PROV_REFRELE(areq->an_provider);
+ if (areq->an_context != NULL)
+ KCF_CONTEXT_REFRELE(areq->an_context);
+
+ if (areq->an_tried_plist != NULL)
+ kcf_free_triedlist(areq->an_tried_plist);
+ kmem_cache_free(kcf_areq_cache, areq);
+}
+
+/*
+ * Utility routine to remove a request from the chain of requests
+ * hanging off a context.
+ */
+void
+kcf_removereq_in_ctxchain(kcf_context_t *ictx, kcf_areq_node_t *areq)
+{
+ kcf_areq_node_t *cur, *prev;
+
+ /*
+ * Get context lock, search for areq in the chain and remove it.
+ */
+ ASSERT(ictx != NULL);
+ mutex_enter(&ictx->kc_in_use_lock);
+ prev = cur = ictx->kc_req_chain_first;
+
+ while (cur != NULL) {
+ if (cur == areq) {
+ if (prev == cur) {
+ if ((ictx->kc_req_chain_first =
+ cur->an_ctxchain_next) == NULL)
+ ictx->kc_req_chain_last = NULL;
+ } else {
+ if (cur == ictx->kc_req_chain_last)
+ ictx->kc_req_chain_last = prev;
+ prev->an_ctxchain_next = cur->an_ctxchain_next;
+ }
+
+ break;
+ }
+ prev = cur;
+ cur = cur->an_ctxchain_next;
+ }
+ mutex_exit(&ictx->kc_in_use_lock);
+}
+
+/*
+ * Remove the specified node from the global software queue.
+ *
+ * The caller must hold the queue lock and request lock (an_lock).
+ */
+void
+kcf_remove_node(kcf_areq_node_t *node)
+{
+ kcf_areq_node_t *nextp = node->an_next;
+ kcf_areq_node_t *prevp = node->an_prev;
+
+ if (nextp != NULL)
+ nextp->an_prev = prevp;
+ else
+ gswq->gs_last = prevp;
+
+ if (prevp != NULL)
+ prevp->an_next = nextp;
+ else
+ gswq->gs_first = nextp;
+
+ node->an_state = REQ_CANCELED;
+}
+
+/*
+ * Add the request node to the end of the global software queue.
+ *
+ * The caller should not hold the queue lock. Returns 0 if the
+ * request is successfully queued. Returns CRYPTO_BUSY if the limit
+ * on the number of jobs is exceeded.
+ */
+static int
+kcf_enqueue(kcf_areq_node_t *node)
+{
+ kcf_areq_node_t *tnode;
+
+ mutex_enter(&gswq->gs_lock);
+
+ if (gswq->gs_njobs >= gswq->gs_maxjobs) {
+ mutex_exit(&gswq->gs_lock);
+ return (CRYPTO_BUSY);
+ }
+
+ if (gswq->gs_last == NULL) {
+ gswq->gs_first = gswq->gs_last = node;
+ } else {
+ ASSERT(gswq->gs_last->an_next == NULL);
+ tnode = gswq->gs_last;
+ tnode->an_next = node;
+ gswq->gs_last = node;
+ node->an_prev = tnode;
+ }
+
+ gswq->gs_njobs++;
+
+ /* an_lock not needed here as we hold gs_lock */
+ node->an_state = REQ_WAITING;
+
+ mutex_exit(&gswq->gs_lock);
+
+ return (0);
+}
+
+/*
+ * kmem_cache_alloc constructor for sync request structure.
+ */
+/* ARGSUSED */
+static int
+kcf_sreq_cache_constructor(void *buf, void *cdrarg, int kmflags)
+{
+ kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf;
+
+ sreq->sn_type = CRYPTO_SYNCH;
+ cv_init(&sreq->sn_cv, NULL, CV_DEFAULT, NULL);
+ mutex_init(&sreq->sn_lock, NULL, MUTEX_DEFAULT, NULL);
+
+ return (0);
+}
+
+/* ARGSUSED */
+static void
+kcf_sreq_cache_destructor(void *buf, void *cdrarg)
+{
+ kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf;
+
+ mutex_destroy(&sreq->sn_lock);
+ cv_destroy(&sreq->sn_cv);
+}
+
+/*
+ * kmem_cache_alloc constructor for async request structure.
+ */
+/* ARGSUSED */
+static int
+kcf_areq_cache_constructor(void *buf, void *cdrarg, int kmflags)
+{
+ kcf_areq_node_t *areq = (kcf_areq_node_t *)buf;
+
+ areq->an_type = CRYPTO_ASYNCH;
+ areq->an_refcnt = 0;
+ mutex_init(&areq->an_lock, NULL, MUTEX_DEFAULT, NULL);
+ cv_init(&areq->an_done, NULL, CV_DEFAULT, NULL);
+ cv_init(&areq->an_turn_cv, NULL, CV_DEFAULT, NULL);
+
+ return (0);
+}
+
+/* ARGSUSED */
+static void
+kcf_areq_cache_destructor(void *buf, void *cdrarg)
+{
+ kcf_areq_node_t *areq = (kcf_areq_node_t *)buf;
+
+ ASSERT(areq->an_refcnt == 0);
+ mutex_destroy(&areq->an_lock);
+ cv_destroy(&areq->an_done);
+ cv_destroy(&areq->an_turn_cv);
+}
+
+/*
+ * kmem_cache_alloc constructor for kcf_context structure.
+ */
+/* ARGSUSED */
+static int
+kcf_context_cache_constructor(void *buf, void *cdrarg, int kmflags)
+{
+ kcf_context_t *kctx = (kcf_context_t *)buf;
+
+ kctx->kc_refcnt = 0;
+ mutex_init(&kctx->kc_in_use_lock, NULL, MUTEX_DEFAULT, NULL);
+
+ return (0);
+}
+
+/* ARGSUSED */
+static void
+kcf_context_cache_destructor(void *buf, void *cdrarg)
+{
+ kcf_context_t *kctx = (kcf_context_t *)buf;
+
+ ASSERT(kctx->kc_refcnt == 0);
+ mutex_destroy(&kctx->kc_in_use_lock);
+}
+
+void
+kcf_sched_destroy(void)
+{
+ int i;
+
+ if (kcf_misc_kstat)
+ kstat_delete(kcf_misc_kstat);
+
+ if (kcfpool)
+ kmem_free(kcfpool, sizeof (kcf_pool_t));
+
+ for (i = 0; i < REQID_TABLES; i++) {
+ if (kcf_reqid_table[i])
+ kmem_free(kcf_reqid_table[i],
+ sizeof (kcf_reqid_table_t));
+ }
+
+ if (gswq)
+ kmem_free(gswq, sizeof (kcf_global_swq_t));
+
+ if (kcf_context_cache)
+ kmem_cache_destroy(kcf_context_cache);
+ if (kcf_areq_cache)
+ kmem_cache_destroy(kcf_areq_cache);
+ if (kcf_sreq_cache)
+ kmem_cache_destroy(kcf_sreq_cache);
+}
+
+/*
+ * Creates and initializes all the structures needed by the framework.
+ */
+void
+kcf_sched_init(void)
+{
+ int i;
+ kcf_reqid_table_t *rt;
+
+ /*
+ * Create all the kmem caches needed by the framework. We set the
+ * align argument to 64, to get a slab aligned to 64-byte as well as
+ * have the objects (cache_chunksize) to be a 64-byte multiple.
+ * This helps to avoid false sharing as this is the size of the
+ * CPU cache line.
+ */
+ kcf_sreq_cache = kmem_cache_create("kcf_sreq_cache",
+ sizeof (struct kcf_sreq_node), 64, kcf_sreq_cache_constructor,
+ kcf_sreq_cache_destructor, NULL, NULL, NULL, 0);
+
+ kcf_areq_cache = kmem_cache_create("kcf_areq_cache",
+ sizeof (struct kcf_areq_node), 64, kcf_areq_cache_constructor,
+ kcf_areq_cache_destructor, NULL, NULL, NULL, 0);
+
+ kcf_context_cache = kmem_cache_create("kcf_context_cache",
+ sizeof (struct kcf_context), 64, kcf_context_cache_constructor,
+ kcf_context_cache_destructor, NULL, NULL, NULL, 0);
+
+ gswq = kmem_alloc(sizeof (kcf_global_swq_t), KM_SLEEP);
+
+ mutex_init(&gswq->gs_lock, NULL, MUTEX_DEFAULT, NULL);
+ cv_init(&gswq->gs_cv, NULL, CV_DEFAULT, NULL);
+ gswq->gs_njobs = 0;
+ gswq->gs_maxjobs = kcf_maxthreads * crypto_taskq_maxalloc;
+ gswq->gs_first = gswq->gs_last = NULL;
+
+ /* Initialize the global reqid table */
+ for (i = 0; i < REQID_TABLES; i++) {
+ rt = kmem_zalloc(sizeof (kcf_reqid_table_t), KM_SLEEP);
+ kcf_reqid_table[i] = rt;
+ mutex_init(&rt->rt_lock, NULL, MUTEX_DEFAULT, NULL);
+ rt->rt_curid = i;
+ }
+
+ /* Allocate and initialize the thread pool */
+ kcfpool_alloc();
+
+ /* Initialize the event notification list variables */
+ mutex_init(&ntfy_list_lock, NULL, MUTEX_DEFAULT, NULL);
+ cv_init(&ntfy_list_cv, NULL, CV_DEFAULT, NULL);
+
+ /* Create the kcf kstat */
+ kcf_misc_kstat = kstat_create("kcf", 0, "framework_stats", "crypto",
+ KSTAT_TYPE_NAMED, sizeof (kcf_stats_t) / sizeof (kstat_named_t),
+ KSTAT_FLAG_VIRTUAL);
+
+ if (kcf_misc_kstat != NULL) {
+ kcf_misc_kstat->ks_data = &kcf_ksdata;
+ kcf_misc_kstat->ks_update = kcf_misc_kstat_update;
+ kstat_install(kcf_misc_kstat);
+ }
+}
+
+/*
+ * Signal the waiting sync client.
+ */
+void
+kcf_sop_done(kcf_sreq_node_t *sreq, int error)
+{
+ mutex_enter(&sreq->sn_lock);
+ sreq->sn_state = REQ_DONE;
+ sreq->sn_rv = error;
+ cv_signal(&sreq->sn_cv);
+ mutex_exit(&sreq->sn_lock);
+}
+
+/*
+ * Callback the async client with the operation status.
+ * We free the async request node and possibly the context.
+ * We also handle any chain of requests hanging off of
+ * the context.
+ */
+void
+kcf_aop_done(kcf_areq_node_t *areq, int error)
+{
+ kcf_op_type_t optype;
+ boolean_t skip_notify = B_FALSE;
+ kcf_context_t *ictx;
+ kcf_areq_node_t *nextreq;
+
+ /*
+ * Handle recoverable errors. This has to be done first
+ * before doing any thing else in this routine so that
+ * we do not change the state of the request.
+ */
+ if (error != CRYPTO_SUCCESS && IS_RECOVERABLE(error)) {
+ /*
+ * We try another provider, if one is available. Else
+ * we continue with the failure notification to the
+ * client.
+ */
+ if (kcf_resubmit_request(areq) == CRYPTO_QUEUED)
+ return;
+ }
+
+ mutex_enter(&areq->an_lock);
+ areq->an_state = REQ_DONE;
+ mutex_exit(&areq->an_lock);
+
+ optype = (&areq->an_params)->rp_optype;
+ if ((ictx = areq->an_context) != NULL) {
+ /*
+ * A request after it is removed from the request
+ * queue, still stays on a chain of requests hanging
+ * of its context structure. It needs to be removed
+ * from this chain at this point.
+ */
+ mutex_enter(&ictx->kc_in_use_lock);
+ nextreq = areq->an_ctxchain_next;
+ if (nextreq != NULL) {
+ mutex_enter(&nextreq->an_lock);
+ nextreq->an_is_my_turn = B_TRUE;
+ cv_signal(&nextreq->an_turn_cv);
+ mutex_exit(&nextreq->an_lock);
+ }
+
+ ictx->kc_req_chain_first = nextreq;
+ if (nextreq == NULL)
+ ictx->kc_req_chain_last = NULL;
+ mutex_exit(&ictx->kc_in_use_lock);
+
+ if (IS_SINGLE_OP(optype) || IS_FINAL_OP(optype)) {
+ ASSERT(nextreq == NULL);
+ KCF_CONTEXT_REFRELE(ictx);
+ } else if (error != CRYPTO_SUCCESS && IS_INIT_OP(optype)) {
+ /*
+ * NOTE - We do not release the context in case of update
+ * operations. We require the consumer to free it explicitly,
+ * in case it wants to abandon an update operation. This is done
+ * as there may be mechanisms in ECB mode that can continue
+ * even if an operation on a block fails.
+ */
+ KCF_CONTEXT_REFRELE(ictx);
+ }
+ }
+
+ /* Deal with the internal continuation to this request first */
+
+ if (areq->an_isdual) {
+ kcf_dual_req_t *next_arg;
+ next_arg = (kcf_dual_req_t *)areq->an_reqarg.cr_callback_arg;
+ next_arg->kr_areq = areq;
+ KCF_AREQ_REFHOLD(areq);
+ areq->an_isdual = B_FALSE;
+
+ NOTIFY_CLIENT(areq, error);
+ return;
+ }
+
+ /*
+ * If CRYPTO_NOTIFY_OPDONE flag is set, we should notify
+ * always. If this flag is clear, we skip the notification
+ * provided there are no errors. We check this flag for only
+ * init or update operations. It is ignored for single, final or
+ * atomic operations.
+ */
+ skip_notify = (IS_UPDATE_OP(optype) || IS_INIT_OP(optype)) &&
+ (!(areq->an_reqarg.cr_flag & CRYPTO_NOTIFY_OPDONE)) &&
+ (error == CRYPTO_SUCCESS);
+
+ if (!skip_notify) {
+ NOTIFY_CLIENT(areq, error);
+ }
+
+ if (!(areq->an_reqarg.cr_flag & CRYPTO_SKIP_REQID))
+ kcf_reqid_delete(areq);
+
+ KCF_AREQ_REFRELE(areq);
+}
+
+/*
+ * Allocate the thread pool and initialize all the fields.
+ */
+static void
+kcfpool_alloc()
+{
+ kcfpool = kmem_alloc(sizeof (kcf_pool_t), KM_SLEEP);
+
+ kcfpool->kp_threads = kcfpool->kp_idlethreads = 0;
+ kcfpool->kp_blockedthreads = 0;
+ kcfpool->kp_signal_create_thread = B_FALSE;
+ kcfpool->kp_nthrs = 0;
+ kcfpool->kp_user_waiting = B_FALSE;
+
+ mutex_init(&kcfpool->kp_thread_lock, NULL, MUTEX_DEFAULT, NULL);
+ cv_init(&kcfpool->kp_nothr_cv, NULL, CV_DEFAULT, NULL);
+
+ mutex_init(&kcfpool->kp_user_lock, NULL, MUTEX_DEFAULT, NULL);
+ cv_init(&kcfpool->kp_user_cv, NULL, CV_DEFAULT, NULL);
+
+ kcf_idlethr_timeout = KCF_DEFAULT_THRTIMEOUT;
+}
+
+/*
+ * Insert the async request in the hash table after assigning it
+ * an ID. Returns the ID.
+ *
+ * The ID is used by the caller to pass as an argument to a
+ * cancel_req() routine later.
+ */
+static crypto_req_id_t
+kcf_reqid_insert(kcf_areq_node_t *areq)
+{
+ int indx;
+ crypto_req_id_t id;
+ kcf_areq_node_t *headp;
+ kcf_reqid_table_t *rt =
+ kcf_reqid_table[CPU_SEQID & REQID_TABLE_MASK];
+
+ mutex_enter(&rt->rt_lock);
+
+ rt->rt_curid = id =
+ (rt->rt_curid - REQID_COUNTER_LOW) | REQID_COUNTER_HIGH;
+ SET_REQID(areq, id);
+ indx = REQID_HASH(id);
+ headp = areq->an_idnext = rt->rt_idhash[indx];
+ areq->an_idprev = NULL;
+ if (headp != NULL)
+ headp->an_idprev = areq;
+
+ rt->rt_idhash[indx] = areq;
+ mutex_exit(&rt->rt_lock);
+
+ return (id);
+}
+
+/*
+ * Delete the async request from the hash table.
+ */
+static void
+kcf_reqid_delete(kcf_areq_node_t *areq)
+{
+ int indx;
+ kcf_areq_node_t *nextp, *prevp;
+ crypto_req_id_t id = GET_REQID(areq);
+ kcf_reqid_table_t *rt;
+
+ rt = kcf_reqid_table[id & REQID_TABLE_MASK];
+ indx = REQID_HASH(id);
+
+ mutex_enter(&rt->rt_lock);
+
+ nextp = areq->an_idnext;
+ prevp = areq->an_idprev;
+ if (nextp != NULL)
+ nextp->an_idprev = prevp;
+ if (prevp != NULL)
+ prevp->an_idnext = nextp;
+ else
+ rt->rt_idhash[indx] = nextp;
+
+ SET_REQID(areq, 0);
+ cv_broadcast(&areq->an_done);
+
+ mutex_exit(&rt->rt_lock);
+}
+
+/*
+ * Cancel a single asynchronous request.
+ *
+ * We guarantee that no problems will result from calling
+ * crypto_cancel_req() for a request which is either running, or
+ * has already completed. We remove the request from any queues
+ * if it is possible. We wait for request completion if the
+ * request is dispatched to a provider.
+ *
+ * Calling context:
+ * Can be called from user context only.
+ *
+ * NOTE: We acquire the following locks in this routine (in order):
+ * - rt_lock (kcf_reqid_table_t)
+ * - gswq->gs_lock
+ * - areq->an_lock
+ * - ictx->kc_in_use_lock (from kcf_removereq_in_ctxchain())
+ *
+ * This locking order MUST be maintained in code every where else.
+ */
+void
+crypto_cancel_req(crypto_req_id_t id)
+{
+ int indx;
+ kcf_areq_node_t *areq;
+ kcf_provider_desc_t *pd;
+ kcf_context_t *ictx;
+ kcf_reqid_table_t *rt;
+
+ rt = kcf_reqid_table[id & REQID_TABLE_MASK];
+ indx = REQID_HASH(id);
+
+ mutex_enter(&rt->rt_lock);
+ for (areq = rt->rt_idhash[indx]; areq; areq = areq->an_idnext) {
+ if (GET_REQID(areq) == id) {
+ /*
+ * We found the request. It is either still waiting
+ * in the framework queues or running at the provider.
+ */
+ pd = areq->an_provider;
+ ASSERT(pd != NULL);
+
+ switch (pd->pd_prov_type) {
+ case CRYPTO_SW_PROVIDER:
+ mutex_enter(&gswq->gs_lock);
+ mutex_enter(&areq->an_lock);
+
+ /* This request can be safely canceled. */
+ if (areq->an_state <= REQ_WAITING) {
+ /* Remove from gswq, global software queue. */
+ kcf_remove_node(areq);
+ if ((ictx = areq->an_context) != NULL)
+ kcf_removereq_in_ctxchain(ictx, areq);
+
+ mutex_exit(&areq->an_lock);
+ mutex_exit(&gswq->gs_lock);
+ mutex_exit(&rt->rt_lock);
+
+ /* Remove areq from hash table and free it. */
+ kcf_reqid_delete(areq);
+ KCF_AREQ_REFRELE(areq);
+ return;
+ }
+
+ mutex_exit(&areq->an_lock);
+ mutex_exit(&gswq->gs_lock);
+ break;
+
+ case CRYPTO_HW_PROVIDER:
+ /*
+ * There is no interface to remove an entry
+ * once it is on the taskq. So, we do not do
+ * any thing for a hardware provider.
+ */
+ break;
+ default:
+ break;
+ }
+
+ /*
+ * The request is running. Wait for the request completion
+ * to notify us.
+ */
+ KCF_AREQ_REFHOLD(areq);
+ while (GET_REQID(areq) == id)
+ cv_wait(&areq->an_done, &rt->rt_lock);
+ KCF_AREQ_REFRELE(areq);
+ break;
+ }
+ }
+
+ mutex_exit(&rt->rt_lock);
+}
+
+/*
+ * Cancel all asynchronous requests associated with the
+ * passed in crypto context and free it.
+ *
+ * A client SHOULD NOT call this routine after calling a crypto_*_final
+ * routine. This routine is called only during intermediate operations.
+ * The client should not use the crypto context after this function returns
+ * since we destroy it.
+ *
+ * Calling context:
+ * Can be called from user context only.
+ */
+void
+crypto_cancel_ctx(crypto_context_t ctx)
+{
+ kcf_context_t *ictx;
+ kcf_areq_node_t *areq;
+
+ if (ctx == NULL)
+ return;
+
+ ictx = (kcf_context_t *)((crypto_ctx_t *)ctx)->cc_framework_private;
+
+ mutex_enter(&ictx->kc_in_use_lock);
+
+ /* Walk the chain and cancel each request */
+ while ((areq = ictx->kc_req_chain_first) != NULL) {
+ /*
+ * We have to drop the lock here as we may have
+ * to wait for request completion. We hold the
+ * request before dropping the lock though, so that it
+ * won't be freed underneath us.
+ */
+ KCF_AREQ_REFHOLD(areq);
+ mutex_exit(&ictx->kc_in_use_lock);
+
+ crypto_cancel_req(GET_REQID(areq));
+ KCF_AREQ_REFRELE(areq);
+
+ mutex_enter(&ictx->kc_in_use_lock);
+ }
+
+ mutex_exit(&ictx->kc_in_use_lock);
+ KCF_CONTEXT_REFRELE(ictx);
+}
+
+/*
+ * Update kstats.
+ */
+static int
+kcf_misc_kstat_update(kstat_t *ksp, int rw)
+{
+ uint_t tcnt;
+ kcf_stats_t *ks_data;
+
+ if (rw == KSTAT_WRITE)
+ return (EACCES);
+
+ ks_data = ksp->ks_data;
+
+ ks_data->ks_thrs_in_pool.value.ui32 = kcfpool->kp_threads;
+ /*
+ * The failover thread is counted in kp_idlethreads in
+ * some corner cases. This is done to avoid doing more checks
+ * when submitting a request. We account for those cases below.
+ */
+ if ((tcnt = kcfpool->kp_idlethreads) == (kcfpool->kp_threads + 1))
+ tcnt--;
+ ks_data->ks_idle_thrs.value.ui32 = tcnt;
+ ks_data->ks_minthrs.value.ui32 = kcf_minthreads;
+ ks_data->ks_maxthrs.value.ui32 = kcf_maxthreads;
+ ks_data->ks_swq_njobs.value.ui32 = gswq->gs_njobs;
+ ks_data->ks_swq_maxjobs.value.ui32 = gswq->gs_maxjobs;
+ ks_data->ks_taskq_threads.value.ui32 = crypto_taskq_threads;
+ ks_data->ks_taskq_minalloc.value.ui32 = crypto_taskq_minalloc;
+ ks_data->ks_taskq_maxalloc.value.ui32 = crypto_taskq_maxalloc;
+
+ return (0);
+}
+
+/*
+ * Allocate and initiatize a kcf_dual_req, used for saving the arguments of
+ * a dual operation or an atomic operation that has to be internally
+ * simulated with multiple single steps.
+ * crq determines the memory allocation flags.
+ */
+
+kcf_dual_req_t *
+kcf_alloc_req(crypto_call_req_t *crq)
+{
+ kcf_dual_req_t *kcr;
+
+ kcr = kmem_alloc(sizeof (kcf_dual_req_t), KCF_KMFLAG(crq));
+
+ if (kcr == NULL)
+ return (NULL);
+
+ /* Copy the whole crypto_call_req struct, as it isn't persistant */
+ if (crq != NULL)
+ kcr->kr_callreq = *crq;
+ else
+ bzero(&(kcr->kr_callreq), sizeof (crypto_call_req_t));
+ kcr->kr_areq = NULL;
+ kcr->kr_saveoffset = 0;
+ kcr->kr_savelen = 0;
+
+ return (kcr);
+}
+
+/*
+ * Callback routine for the next part of a simulated dual part.
+ * Schedules the next step.
+ *
+ * This routine can be called from interrupt context.
+ */
+void
+kcf_next_req(void *next_req_arg, int status)
+{
+ kcf_dual_req_t *next_req = (kcf_dual_req_t *)next_req_arg;
+ kcf_req_params_t *params = &(next_req->kr_params);
+ kcf_areq_node_t *areq = next_req->kr_areq;
+ int error = status;
+ kcf_provider_desc_t *pd = NULL;
+ crypto_dual_data_t *ct = NULL;
+
+ /* Stop the processing if an error occured at this step */
+ if (error != CRYPTO_SUCCESS) {
+out:
+ areq->an_reqarg = next_req->kr_callreq;
+ KCF_AREQ_REFRELE(areq);
+ kmem_free(next_req, sizeof (kcf_dual_req_t));
+ areq->an_isdual = B_FALSE;
+ kcf_aop_done(areq, error);
+ return;
+ }
+
+ switch (params->rp_opgrp) {
+ case KCF_OG_MAC: {
+
+ /*
+ * The next req is submitted with the same reqid as the
+ * first part. The consumer only got back that reqid, and
+ * should still be able to cancel the operation during its
+ * second step.
+ */
+ kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params);
+ crypto_ctx_template_t mac_tmpl;
+ kcf_mech_entry_t *me;
+
+ ct = (crypto_dual_data_t *)mops->mo_data;
+ mac_tmpl = (crypto_ctx_template_t)mops->mo_templ;
+
+ /* No expected recoverable failures, so no retry list */
+ pd = kcf_get_mech_provider(mops->mo_framework_mechtype,
+ &me, &error, NULL, CRYPTO_FG_MAC_ATOMIC,
+ (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), ct->dd_len2);
+
+ if (pd == NULL) {
+ error = CRYPTO_MECH_NOT_SUPPORTED;
+ goto out;
+ }
+ /* Validate the MAC context template here */
+ if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) &&
+ (mac_tmpl != NULL)) {
+ kcf_ctx_template_t *ctx_mac_tmpl;
+
+ ctx_mac_tmpl = (kcf_ctx_template_t *)mac_tmpl;
+
+ if (ctx_mac_tmpl->ct_generation != me->me_gen_swprov) {
+ KCF_PROV_REFRELE(pd);
+ error = CRYPTO_OLD_CTX_TEMPLATE;
+ goto out;
+ }
+ mops->mo_templ = ctx_mac_tmpl->ct_prov_tmpl;
+ }
+
+ break;
+ }
+ case KCF_OG_DECRYPT: {
+ kcf_decrypt_ops_params_t *dcrops =
+ &(params->rp_u.decrypt_params);
+
+ ct = (crypto_dual_data_t *)dcrops->dop_ciphertext;
+ /* No expected recoverable failures, so no retry list */
+ pd = kcf_get_mech_provider(dcrops->dop_framework_mechtype,
+ NULL, &error, NULL, CRYPTO_FG_DECRYPT_ATOMIC,
+ (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), ct->dd_len1);
+
+ if (pd == NULL) {
+ error = CRYPTO_MECH_NOT_SUPPORTED;
+ goto out;
+ }
+ break;
+ }
+ default:
+ break;
+ }
+
+ /* The second step uses len2 and offset2 of the dual_data */
+ next_req->kr_saveoffset = ct->dd_offset1;
+ next_req->kr_savelen = ct->dd_len1;
+ ct->dd_offset1 = ct->dd_offset2;
+ ct->dd_len1 = ct->dd_len2;
+
+ /* preserve if the caller is restricted */
+ if (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED) {
+ areq->an_reqarg.cr_flag = CRYPTO_RESTRICTED;
+ } else {
+ areq->an_reqarg.cr_flag = 0;
+ }
+
+ areq->an_reqarg.cr_callback_func = kcf_last_req;
+ areq->an_reqarg.cr_callback_arg = next_req;
+ areq->an_isdual = B_TRUE;
+
+ /*
+ * We would like to call kcf_submit_request() here. But,
+ * that is not possible as that routine allocates a new
+ * kcf_areq_node_t request structure, while we need to
+ * reuse the existing request structure.
+ */
+ switch (pd->pd_prov_type) {
+ case CRYPTO_SW_PROVIDER:
+ error = common_submit_request(pd, NULL, params,
+ KCF_RHNDL(KM_NOSLEEP));
+ break;
+
+ case CRYPTO_HW_PROVIDER: {
+ kcf_provider_desc_t *old_pd;
+ taskq_t *taskq = pd->pd_sched_info.ks_taskq;
+
+ /*
+ * Set the params for the second step in the
+ * dual-ops.
+ */
+ areq->an_params = *params;
+ old_pd = areq->an_provider;
+ KCF_PROV_REFRELE(old_pd);
+ KCF_PROV_REFHOLD(pd);
+ areq->an_provider = pd;
+
+ /*
+ * Note that we have to do a taskq_dispatch()
+ * here as we may be in interrupt context.
+ */
+ if (taskq_dispatch(taskq, process_req_hwp, areq,
+ TQ_NOSLEEP) == (taskqid_t)0) {
+ error = CRYPTO_HOST_MEMORY;
+ } else {
+ error = CRYPTO_QUEUED;
+ }
+ break;
+ }
+ default:
+ break;
+ }
+
+ /*
+ * We have to release the holds on the request and the provider
+ * in all cases.
+ */
+ KCF_AREQ_REFRELE(areq);
+ KCF_PROV_REFRELE(pd);
+
+ if (error != CRYPTO_QUEUED) {
+ /* restore, clean up, and invoke the client's callback */
+
+ ct->dd_offset1 = next_req->kr_saveoffset;
+ ct->dd_len1 = next_req->kr_savelen;
+ areq->an_reqarg = next_req->kr_callreq;
+ kmem_free(next_req, sizeof (kcf_dual_req_t));
+ areq->an_isdual = B_FALSE;
+ kcf_aop_done(areq, error);
+ }
+}
+
+/*
+ * Last part of an emulated dual operation.
+ * Clean up and restore ...
+ */
+void
+kcf_last_req(void *last_req_arg, int status)
+{
+ kcf_dual_req_t *last_req = (kcf_dual_req_t *)last_req_arg;
+
+ kcf_req_params_t *params = &(last_req->kr_params);
+ kcf_areq_node_t *areq = last_req->kr_areq;
+ crypto_dual_data_t *ct = NULL;
+
+ switch (params->rp_opgrp) {
+ case KCF_OG_MAC: {
+ kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params);
+
+ ct = (crypto_dual_data_t *)mops->mo_data;
+ break;
+ }
+ case KCF_OG_DECRYPT: {
+ kcf_decrypt_ops_params_t *dcrops =
+ &(params->rp_u.decrypt_params);
+
+ ct = (crypto_dual_data_t *)dcrops->dop_ciphertext;
+ break;
+ }
+ default:
+ break;
+ }
+ ct->dd_offset1 = last_req->kr_saveoffset;
+ ct->dd_len1 = last_req->kr_savelen;
+
+ /* The submitter used kcf_last_req as its callback */
+
+ if (areq == NULL) {
+ crypto_call_req_t *cr = &last_req->kr_callreq;
+
+ (*(cr->cr_callback_func))(cr->cr_callback_arg, status);
+ kmem_free(last_req, sizeof (kcf_dual_req_t));
+ return;
+ }
+ areq->an_reqarg = last_req->kr_callreq;
+ KCF_AREQ_REFRELE(areq);
+ kmem_free(last_req, sizeof (kcf_dual_req_t));
+ areq->an_isdual = B_FALSE;
+ kcf_aop_done(areq, status);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2016, Datto, Inc. All rights reserved.
+ */
+
+#ifdef _KERNEL
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#else
+#define __exit
+#define __init
+#endif
+
+#include <sys/crypto/common.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/sched_impl.h>
+#include <sys/modhash_impl.h>
+#include <sys/crypto/icp.h>
+
+/*
+ * Changes made to the original Illumos Crypto Layer for the ICP:
+ *
+ * Several changes were needed to allow the Illumos Crypto Layer
+ * to work in the Linux kernel. Almost all of the changes fall into
+ * one of the following categories:
+ *
+ * 1) Moving the syntax to the C90: This was mostly a matter of
+ * changing func() definitions to func(void). In a few cases,
+ * initializations of structs with unions needed to have brackets
+ * added.
+ *
+ * 2) Changes to allow userspace compilation: The ICP is meant to be
+ * compiled and used in both userspace and kernel space (for ztest and
+ * libzfs), so the _KERNEL macros did not make sense anymore. For the
+ * same reason, many header includes were also changed to use
+ * sys/zfs_context.h
+ *
+ * 3) Moving to a statically compiled architecture: At some point in
+ * the future it may make sense to have encryption algorithms that are
+ * loadable into the ICP at runtime via separate kernel modules.
+ * However, considering that this code will probably not see much use
+ * outside of zfs and zfs encryption only requires aes and sha256
+ * algorithms it seemed like more trouble than it was worth to port over
+ * Illumos's kernel module structure to a Linux kernel module. In
+ * addition, The Illumos code related to keeping track of kernel modules
+ * is very much tied to the Illumos OS and proved difficult to port to
+ * Linux. Therefore, the structure of the ICP was simplified to work
+ * statically and several pieces of code responsible for keeping track
+ * of Illumos kernel modules were removed and simplified. All module
+ * initialization and destruction is now called in this file during
+ * Linux kernel module loading and unloading.
+ *
+ * 4) Adding destructors: The Illumos Crypto Layer is built into
+ * the Illumos kernel and is not meant to be unloaded. Some destructors
+ * were added to allow the ICP to be unloaded without leaking
+ * structures.
+ *
+ * 5) Removing CRYPTO_DATA_MBLK related structures and code:
+ * crypto_data_t can have 3 formats, CRYPTO_DATA_RAW, CRYPTO_DATA_UIO,
+ * and CRYPTO_DATA_MBLK. ZFS only requires the first 2 formats, as the
+ * last one is related to streamed data. To simplify the port, code
+ * related to this format was removed.
+ *
+ * 6) Changes for architecture specific code: Some changes were needed
+ * to make architecture specific assembly compile. The biggest change
+ * here was to functions related to detecting CPU capabilities for amd64.
+ * The Illumos Crypto Layer used called into the Illumos kernel's API
+ * to discover these. They have been converted to instead use the
+ * 'cpuid' instruction as per the Intel spec. In addition, references to
+ * the sun4u' and sparc architectures have been removed so that these
+ * will use the generic implementation.
+ *
+ * 7) Removing sha384 and sha512 code: The sha code was actually very
+ * wasy to port. However, the generic sha384 and sha512 code actually
+ * exceeds the stack size on arm and powerpc architectures. In an effort
+ * to remove warnings, this code was removed.
+ *
+ * 8) Change large allocations from kmem_alloc() to vmem_alloc(): In
+ * testing the ICP with the ZFS encryption code, a few allocations were
+ * found that could potentially be very large. These caused the SPL to
+ * throw warnings and so they were changed to use vmem_alloc().
+ *
+ * 9) Makefiles: Makefiles were added that would work with the existing
+ * ZFS Makefiles.
+ */
+
+void __exit
+icp_fini(void)
+{
+ sha2_mod_fini();
+ sha1_mod_fini();
+ aes_mod_fini();
+ kcf_sched_destroy();
+ kcf_prov_tab_destroy();
+ kcf_destroy_mech_tabs();
+ mod_hash_fini();
+}
+
+/* roughly equivalent to kcf.c: _init() */
+int __init
+icp_init(void)
+{
+ /* initialize the mod hash module */
+ mod_hash_init();
+
+ /* initialize the mechanisms tables supported out-of-the-box */
+ kcf_init_mech_tabs();
+
+ /* initialize the providers tables */
+ kcf_prov_tab_init();
+
+ /*
+ * Initialize scheduling structures. Note that this does NOT
+ * start any threads since it might not be safe to do so.
+ */
+ kcf_sched_init();
+
+ /* initialize algorithms */
+ aes_mod_init();
+ sha1_mod_init();
+ sha2_mod_init();
+
+ return (0);
+}
+
+#if defined(_KERNEL) && defined(HAVE_SPL)
+module_exit(icp_fini);
+module_init(icp_init);
+MODULE_LICENSE("CDDL");
+#endif
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _AES_IMPL_H
+#define _AES_IMPL_H
+
+/*
+ * Common definitions used by AES.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+
+/* Similar to sysmacros.h IS_P2ALIGNED, but checks two pointers: */
+#define IS_P2ALIGNED2(v, w, a) \
+ ((((uintptr_t)(v) | (uintptr_t)(w)) & ((uintptr_t)(a) - 1)) == 0)
+
+#define AES_BLOCK_LEN 16 /* bytes */
+/* Round constant length, in number of 32-bit elements: */
+#define RC_LENGTH (5 * ((AES_BLOCK_LEN) / 4 - 2))
+
+#define AES_COPY_BLOCK(src, dst) \
+ (dst)[0] = (src)[0]; \
+ (dst)[1] = (src)[1]; \
+ (dst)[2] = (src)[2]; \
+ (dst)[3] = (src)[3]; \
+ (dst)[4] = (src)[4]; \
+ (dst)[5] = (src)[5]; \
+ (dst)[6] = (src)[6]; \
+ (dst)[7] = (src)[7]; \
+ (dst)[8] = (src)[8]; \
+ (dst)[9] = (src)[9]; \
+ (dst)[10] = (src)[10]; \
+ (dst)[11] = (src)[11]; \
+ (dst)[12] = (src)[12]; \
+ (dst)[13] = (src)[13]; \
+ (dst)[14] = (src)[14]; \
+ (dst)[15] = (src)[15]
+
+#define AES_XOR_BLOCK(src, dst) \
+ (dst)[0] ^= (src)[0]; \
+ (dst)[1] ^= (src)[1]; \
+ (dst)[2] ^= (src)[2]; \
+ (dst)[3] ^= (src)[3]; \
+ (dst)[4] ^= (src)[4]; \
+ (dst)[5] ^= (src)[5]; \
+ (dst)[6] ^= (src)[6]; \
+ (dst)[7] ^= (src)[7]; \
+ (dst)[8] ^= (src)[8]; \
+ (dst)[9] ^= (src)[9]; \
+ (dst)[10] ^= (src)[10]; \
+ (dst)[11] ^= (src)[11]; \
+ (dst)[12] ^= (src)[12]; \
+ (dst)[13] ^= (src)[13]; \
+ (dst)[14] ^= (src)[14]; \
+ (dst)[15] ^= (src)[15]
+
+/* AES key size definitions */
+#define AES_MINBITS 128
+#define AES_MINBYTES ((AES_MINBITS) >> 3)
+#define AES_MAXBITS 256
+#define AES_MAXBYTES ((AES_MAXBITS) >> 3)
+
+#define AES_MIN_KEY_BYTES ((AES_MINBITS) >> 3)
+#define AES_MAX_KEY_BYTES ((AES_MAXBITS) >> 3)
+#define AES_192_KEY_BYTES 24
+#define AES_IV_LEN 16
+
+/* AES key schedule may be implemented with 32- or 64-bit elements: */
+#define AES_32BIT_KS 32
+#define AES_64BIT_KS 64
+
+#define MAX_AES_NR 14 /* Maximum number of rounds */
+#define MAX_AES_NB 4 /* Number of columns comprising a state */
+
+typedef union {
+#ifdef sun4u
+ uint64_t ks64[((MAX_AES_NR) + 1) * (MAX_AES_NB)];
+#endif
+ uint32_t ks32[((MAX_AES_NR) + 1) * (MAX_AES_NB)];
+} aes_ks_t;
+
+/* aes_key.flags value: */
+#define INTEL_AES_NI_CAPABLE 0x1 /* AES-NI instructions present */
+
+typedef struct aes_key aes_key_t;
+struct aes_key {
+ aes_ks_t encr_ks; /* encryption key schedule */
+ aes_ks_t decr_ks; /* decryption key schedule */
+#ifdef __amd64
+ long double align128; /* Align fields above for Intel AES-NI */
+ int flags; /* implementation-dependent flags */
+#endif /* __amd64 */
+ int nr; /* number of rounds (10, 12, or 14) */
+ int type; /* key schedule size (32 or 64 bits) */
+};
+
+/*
+ * Core AES functions.
+ * ks and keysched are pointers to aes_key_t.
+ * They are declared void* as they are intended to be opaque types.
+ * Use function aes_alloc_keysched() to allocate memory for ks and keysched.
+ */
+extern void *aes_alloc_keysched(size_t *size, int kmflag);
+extern void aes_init_keysched(const uint8_t *cipherKey, uint_t keyBits,
+ void *keysched);
+extern int aes_encrypt_block(const void *ks, const uint8_t *pt, uint8_t *ct);
+extern int aes_decrypt_block(const void *ks, const uint8_t *ct, uint8_t *pt);
+
+/*
+ * AES mode functions.
+ * The first 2 functions operate on 16-byte AES blocks.
+ */
+extern void aes_copy_block(uint8_t *in, uint8_t *out);
+extern void aes_xor_block(uint8_t *data, uint8_t *dst);
+
+/* Note: ctx is a pointer to aes_ctx_t defined in modes.h */
+extern int aes_encrypt_contiguous_blocks(void *ctx, char *data, size_t length,
+ crypto_data_t *out);
+extern int aes_decrypt_contiguous_blocks(void *ctx, char *data, size_t length,
+ crypto_data_t *out);
+
+/*
+ * The following definitions and declarations are only used by AES FIPS POST
+ */
+#ifdef _AES_IMPL
+
+typedef enum aes_mech_type {
+ AES_ECB_MECH_INFO_TYPE, /* SUN_CKM_AES_ECB */
+ AES_CBC_MECH_INFO_TYPE, /* SUN_CKM_AES_CBC */
+ AES_CBC_PAD_MECH_INFO_TYPE, /* SUN_CKM_AES_CBC_PAD */
+ AES_CTR_MECH_INFO_TYPE, /* SUN_CKM_AES_CTR */
+ AES_CCM_MECH_INFO_TYPE, /* SUN_CKM_AES_CCM */
+ AES_GCM_MECH_INFO_TYPE, /* SUN_CKM_AES_GCM */
+ AES_GMAC_MECH_INFO_TYPE /* SUN_CKM_AES_GMAC */
+} aes_mech_type_t;
+
+#endif /* _AES_IMPL */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _AES_IMPL_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _COMMON_CRYPTO_MODES_H
+#define _COMMON_CRYPTO_MODES_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+
+#define ECB_MODE 0x00000002
+#define CBC_MODE 0x00000004
+#define CTR_MODE 0x00000008
+#define CCM_MODE 0x00000010
+#define GCM_MODE 0x00000020
+#define GMAC_MODE 0x00000040
+
+/*
+ * cc_keysched: Pointer to key schedule.
+ *
+ * cc_keysched_len: Length of the key schedule.
+ *
+ * cc_remainder: This is for residual data, i.e. data that can't
+ * be processed because there are too few bytes.
+ * Must wait until more data arrives.
+ *
+ * cc_remainder_len: Number of bytes in cc_remainder.
+ *
+ * cc_iv: Scratch buffer that sometimes contains the IV.
+ *
+ * cc_lastp: Pointer to previous block of ciphertext.
+ *
+ * cc_copy_to: Pointer to where encrypted residual data needs
+ * to be copied.
+ *
+ * cc_flags: PROVIDER_OWNS_KEY_SCHEDULE
+ * When a context is freed, it is necessary
+ * to know whether the key schedule was allocated
+ * by the caller, or internally, e.g. an init routine.
+ * If allocated by the latter, then it needs to be freed.
+ *
+ * ECB_MODE, CBC_MODE, CTR_MODE, or CCM_MODE
+ */
+struct common_ctx {
+ void *cc_keysched;
+ size_t cc_keysched_len;
+ uint64_t cc_iv[2];
+ uint64_t cc_remainder[2];
+ size_t cc_remainder_len;
+ uint8_t *cc_lastp;
+ uint8_t *cc_copy_to;
+ uint32_t cc_flags;
+};
+
+typedef struct common_ctx common_ctx_t;
+
+typedef struct ecb_ctx {
+ struct common_ctx ecb_common;
+ uint64_t ecb_lastblock[2];
+} ecb_ctx_t;
+
+#define ecb_keysched ecb_common.cc_keysched
+#define ecb_keysched_len ecb_common.cc_keysched_len
+#define ecb_iv ecb_common.cc_iv
+#define ecb_remainder ecb_common.cc_remainder
+#define ecb_remainder_len ecb_common.cc_remainder_len
+#define ecb_lastp ecb_common.cc_lastp
+#define ecb_copy_to ecb_common.cc_copy_to
+#define ecb_flags ecb_common.cc_flags
+
+typedef struct cbc_ctx {
+ struct common_ctx cbc_common;
+ uint64_t cbc_lastblock[2];
+} cbc_ctx_t;
+
+#define cbc_keysched cbc_common.cc_keysched
+#define cbc_keysched_len cbc_common.cc_keysched_len
+#define cbc_iv cbc_common.cc_iv
+#define cbc_remainder cbc_common.cc_remainder
+#define cbc_remainder_len cbc_common.cc_remainder_len
+#define cbc_lastp cbc_common.cc_lastp
+#define cbc_copy_to cbc_common.cc_copy_to
+#define cbc_flags cbc_common.cc_flags
+
+/*
+ * ctr_lower_mask Bit-mask for lower 8 bytes of counter block.
+ * ctr_upper_mask Bit-mask for upper 8 bytes of counter block.
+ */
+typedef struct ctr_ctx {
+ struct common_ctx ctr_common;
+ uint64_t ctr_lower_mask;
+ uint64_t ctr_upper_mask;
+ uint32_t ctr_tmp[4];
+} ctr_ctx_t;
+
+/*
+ * ctr_cb Counter block.
+ */
+#define ctr_keysched ctr_common.cc_keysched
+#define ctr_keysched_len ctr_common.cc_keysched_len
+#define ctr_cb ctr_common.cc_iv
+#define ctr_remainder ctr_common.cc_remainder
+#define ctr_remainder_len ctr_common.cc_remainder_len
+#define ctr_lastp ctr_common.cc_lastp
+#define ctr_copy_to ctr_common.cc_copy_to
+#define ctr_flags ctr_common.cc_flags
+
+/*
+ *
+ * ccm_mac_len: Stores length of the MAC in CCM mode.
+ * ccm_mac_buf: Stores the intermediate value for MAC in CCM encrypt.
+ * In CCM decrypt, stores the input MAC value.
+ * ccm_data_len: Length of the plaintext for CCM mode encrypt, or
+ * length of the ciphertext for CCM mode decrypt.
+ * ccm_processed_data_len:
+ * Length of processed plaintext in CCM mode encrypt,
+ * or length of processed ciphertext for CCM mode decrypt.
+ * ccm_processed_mac_len:
+ * Length of MAC data accumulated in CCM mode decrypt.
+ *
+ * ccm_pt_buf: Only used in CCM mode decrypt. It stores the
+ * decrypted plaintext to be returned when
+ * MAC verification succeeds in decrypt_final.
+ * Memory for this should be allocated in the AES module.
+ *
+ */
+typedef struct ccm_ctx {
+ struct common_ctx ccm_common;
+ uint32_t ccm_tmp[4];
+ size_t ccm_mac_len;
+ uint64_t ccm_mac_buf[2];
+ size_t ccm_data_len;
+ size_t ccm_processed_data_len;
+ size_t ccm_processed_mac_len;
+ uint8_t *ccm_pt_buf;
+ uint64_t ccm_mac_input_buf[2];
+ uint64_t ccm_counter_mask;
+} ccm_ctx_t;
+
+#define ccm_keysched ccm_common.cc_keysched
+#define ccm_keysched_len ccm_common.cc_keysched_len
+#define ccm_cb ccm_common.cc_iv
+#define ccm_remainder ccm_common.cc_remainder
+#define ccm_remainder_len ccm_common.cc_remainder_len
+#define ccm_lastp ccm_common.cc_lastp
+#define ccm_copy_to ccm_common.cc_copy_to
+#define ccm_flags ccm_common.cc_flags
+
+/*
+ * gcm_tag_len: Length of authentication tag.
+ *
+ * gcm_ghash: Stores output from the GHASH function.
+ *
+ * gcm_processed_data_len:
+ * Length of processed plaintext (encrypt) or
+ * length of processed ciphertext (decrypt).
+ *
+ * gcm_pt_buf: Stores the decrypted plaintext returned by
+ * decrypt_final when the computed authentication
+ * tag matches the user supplied tag.
+ *
+ * gcm_pt_buf_len: Length of the plaintext buffer.
+ *
+ * gcm_H: Subkey.
+ *
+ * gcm_J0: Pre-counter block generated from the IV.
+ *
+ * gcm_len_a_len_c: 64-bit representations of the bit lengths of
+ * AAD and ciphertext.
+ *
+ * gcm_kmflag: Current value of kmflag. Used only for allocating
+ * the plaintext buffer during decryption.
+ */
+typedef struct gcm_ctx {
+ struct common_ctx gcm_common;
+ size_t gcm_tag_len;
+ size_t gcm_processed_data_len;
+ size_t gcm_pt_buf_len;
+ uint32_t gcm_tmp[4];
+ uint64_t gcm_ghash[2];
+ uint64_t gcm_H[2];
+ uint64_t gcm_J0[2];
+ uint64_t gcm_len_a_len_c[2];
+ uint8_t *gcm_pt_buf;
+ int gcm_kmflag;
+} gcm_ctx_t;
+
+#define gcm_keysched gcm_common.cc_keysched
+#define gcm_keysched_len gcm_common.cc_keysched_len
+#define gcm_cb gcm_common.cc_iv
+#define gcm_remainder gcm_common.cc_remainder
+#define gcm_remainder_len gcm_common.cc_remainder_len
+#define gcm_lastp gcm_common.cc_lastp
+#define gcm_copy_to gcm_common.cc_copy_to
+#define gcm_flags gcm_common.cc_flags
+
+#define AES_GMAC_IV_LEN 12
+#define AES_GMAC_TAG_BITS 128
+
+typedef struct aes_ctx {
+ union {
+ ecb_ctx_t acu_ecb;
+ cbc_ctx_t acu_cbc;
+ ctr_ctx_t acu_ctr;
+ ccm_ctx_t acu_ccm;
+ gcm_ctx_t acu_gcm;
+ } acu;
+} aes_ctx_t;
+
+#define ac_flags acu.acu_ecb.ecb_common.cc_flags
+#define ac_remainder_len acu.acu_ecb.ecb_common.cc_remainder_len
+#define ac_keysched acu.acu_ecb.ecb_common.cc_keysched
+#define ac_keysched_len acu.acu_ecb.ecb_common.cc_keysched_len
+#define ac_iv acu.acu_ecb.ecb_common.cc_iv
+#define ac_lastp acu.acu_ecb.ecb_common.cc_lastp
+#define ac_pt_buf acu.acu_ccm.ccm_pt_buf
+#define ac_mac_len acu.acu_ccm.ccm_mac_len
+#define ac_data_len acu.acu_ccm.ccm_data_len
+#define ac_processed_mac_len acu.acu_ccm.ccm_processed_mac_len
+#define ac_processed_data_len acu.acu_ccm.ccm_processed_data_len
+#define ac_tag_len acu.acu_gcm.gcm_tag_len
+
+typedef struct blowfish_ctx {
+ union {
+ ecb_ctx_t bcu_ecb;
+ cbc_ctx_t bcu_cbc;
+ } bcu;
+} blowfish_ctx_t;
+
+#define bc_flags bcu.bcu_ecb.ecb_common.cc_flags
+#define bc_remainder_len bcu.bcu_ecb.ecb_common.cc_remainder_len
+#define bc_keysched bcu.bcu_ecb.ecb_common.cc_keysched
+#define bc_keysched_len bcu.bcu_ecb.ecb_common.cc_keysched_len
+#define bc_iv bcu.bcu_ecb.ecb_common.cc_iv
+#define bc_lastp bcu.bcu_ecb.ecb_common.cc_lastp
+
+typedef struct des_ctx {
+ union {
+ ecb_ctx_t dcu_ecb;
+ cbc_ctx_t dcu_cbc;
+ } dcu;
+} des_ctx_t;
+
+#define dc_flags dcu.dcu_ecb.ecb_common.cc_flags
+#define dc_remainder_len dcu.dcu_ecb.ecb_common.cc_remainder_len
+#define dc_keysched dcu.dcu_ecb.ecb_common.cc_keysched
+#define dc_keysched_len dcu.dcu_ecb.ecb_common.cc_keysched_len
+#define dc_iv dcu.dcu_ecb.ecb_common.cc_iv
+#define dc_lastp dcu.dcu_ecb.ecb_common.cc_lastp
+
+extern int ecb_cipher_contiguous_blocks(ecb_ctx_t *, char *, size_t,
+ crypto_data_t *, size_t, int (*cipher)(const void *, const uint8_t *,
+ uint8_t *));
+
+extern int cbc_encrypt_contiguous_blocks(cbc_ctx_t *, char *, size_t,
+ crypto_data_t *, size_t,
+ int (*encrypt)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int cbc_decrypt_contiguous_blocks(cbc_ctx_t *, char *, size_t,
+ crypto_data_t *, size_t,
+ int (*decrypt)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int ctr_mode_contiguous_blocks(ctr_ctx_t *, char *, size_t,
+ crypto_data_t *, size_t,
+ int (*cipher)(const void *, const uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t,
+ crypto_data_t *, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t,
+ crypto_data_t *, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t,
+ crypto_data_t *, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t,
+ crypto_data_t *, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+int ccm_encrypt_final(ccm_ctx_t *, crypto_data_t *, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+int gcm_encrypt_final(gcm_ctx_t *, crypto_data_t *, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int ccm_decrypt_final(ccm_ctx_t *, crypto_data_t *, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int gcm_decrypt_final(gcm_ctx_t *, crypto_data_t *, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int ctr_mode_final(ctr_ctx_t *, crypto_data_t *,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *));
+
+extern int cbc_init_ctx(cbc_ctx_t *, char *, size_t, size_t,
+ void (*copy_block)(uint8_t *, uint64_t *));
+
+extern int ctr_init_ctx(ctr_ctx_t *, ulong_t, uint8_t *,
+ void (*copy_block)(uint8_t *, uint8_t *));
+
+extern int ccm_init_ctx(ccm_ctx_t *, char *, int, boolean_t, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int gcm_init_ctx(gcm_ctx_t *, char *, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern int gmac_init_ctx(gcm_ctx_t *, char *, size_t,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+ void (*copy_block)(uint8_t *, uint8_t *),
+ void (*xor_block)(uint8_t *, uint8_t *));
+
+extern void calculate_ccm_mac(ccm_ctx_t *, uint8_t *,
+ int (*encrypt_block)(const void *, const uint8_t *, uint8_t *));
+
+extern void gcm_mul(uint64_t *, uint64_t *, uint64_t *);
+
+extern void crypto_init_ptrs(crypto_data_t *, void **, offset_t *);
+extern void crypto_get_ptrs(crypto_data_t *, void **, offset_t *,
+ uint8_t **, size_t *, uint8_t **, size_t);
+
+extern void *ecb_alloc_ctx(int);
+extern void *cbc_alloc_ctx(int);
+extern void *ctr_alloc_ctx(int);
+extern void *ccm_alloc_ctx(int);
+extern void *gcm_alloc_ctx(int);
+extern void *gmac_alloc_ctx(int);
+extern void crypto_free_mode_ctx(void *);
+extern void gcm_set_kmflag(gcm_ctx_t *, int);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _COMMON_CRYPTO_MODES_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_SHA1_H
+#define _SYS_SHA1_H
+
+#include <sys/types.h> /* for uint_* */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * NOTE: n2rng (Niagara2 RNG driver) accesses the state field of
+ * SHA1_CTX directly. NEVER change this structure without verifying
+ * compatiblity with n2rng. The important thing is that the state
+ * must be in a field declared as uint32_t state[5].
+ */
+/* SHA-1 context. */
+typedef struct {
+ uint32_t state[5]; /* state (ABCDE) */
+ uint32_t count[2]; /* number of bits, modulo 2^64 (msb first) */
+ union {
+ uint8_t buf8[64]; /* undigested input */
+ uint32_t buf32[16]; /* realigned input */
+ } buf_un;
+} SHA1_CTX;
+
+#define SHA1_DIGEST_LENGTH 20
+
+void SHA1Init(SHA1_CTX *);
+void SHA1Update(SHA1_CTX *, const void *, size_t);
+void SHA1Final(void *, SHA1_CTX *);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_SHA1_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 1998, by Sun Microsystems, Inc.
+ * All rights reserved.
+ */
+
+#ifndef _SYS_SHA1_CONSTS_H
+#define _SYS_SHA1_CONSTS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * as explained in sha1.c, loading 32-bit constants on a sparc is expensive
+ * since it involves both a `sethi' and an `or'. thus, we instead use `ld'
+ * to load the constants from an array called `sha1_consts'. however, on
+ * intel (and perhaps other processors), it is cheaper to load the constant
+ * directly. thus, the c code in SHA1Transform() uses the macro SHA1_CONST()
+ * which either expands to a constant or an array reference, depending on
+ * the architecture the code is being compiled for.
+ */
+
+#include <sys/types.h> /* uint32_t */
+
+extern const uint32_t sha1_consts[];
+
+#if defined(__sparc)
+#define SHA1_CONST(x) (sha1_consts[x])
+#else
+#define SHA1_CONST(x) (SHA1_CONST_ ## x)
+#endif
+
+/* constants, as provided in FIPS 180-1 */
+
+#define SHA1_CONST_0 0x5a827999U
+#define SHA1_CONST_1 0x6ed9eba1U
+#define SHA1_CONST_2 0x8f1bbcdcU
+#define SHA1_CONST_3 0xca62c1d6U
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_SHA1_CONSTS_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SHA1_IMPL_H
+#define _SHA1_IMPL_H
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SHA1_HASH_SIZE 20 /* SHA_1 digest length in bytes */
+#define SHA1_DIGEST_LENGTH 20 /* SHA1 digest length in bytes */
+#define SHA1_HMAC_BLOCK_SIZE 64 /* SHA1-HMAC block size */
+#define SHA1_HMAC_MIN_KEY_LEN 1 /* SHA1-HMAC min key length in bytes */
+#define SHA1_HMAC_MAX_KEY_LEN INT_MAX /* SHA1-HMAC max key length in bytes */
+#define SHA1_HMAC_INTS_PER_BLOCK (SHA1_HMAC_BLOCK_SIZE/sizeof (uint32_t))
+
+/*
+ * CSPI information (entry points, provider info, etc.)
+ */
+typedef enum sha1_mech_type {
+ SHA1_MECH_INFO_TYPE, /* SUN_CKM_SHA1 */
+ SHA1_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA1_HMAC */
+ SHA1_HMAC_GEN_MECH_INFO_TYPE /* SUN_CKM_SHA1_HMAC_GENERAL */
+} sha1_mech_type_t;
+
+/*
+ * Context for SHA1 mechanism.
+ */
+typedef struct sha1_ctx {
+ sha1_mech_type_t sc_mech_type; /* type of context */
+ SHA1_CTX sc_sha1_ctx; /* SHA1 context */
+} sha1_ctx_t;
+
+/*
+ * Context for SHA1-HMAC and SHA1-HMAC-GENERAL mechanisms.
+ */
+typedef struct sha1_hmac_ctx {
+ sha1_mech_type_t hc_mech_type; /* type of context */
+ uint32_t hc_digest_len; /* digest len in bytes */
+ SHA1_CTX hc_icontext; /* inner SHA1 context */
+ SHA1_CTX hc_ocontext; /* outer SHA1 context */
+} sha1_hmac_ctx_t;
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SHA1_IMPL_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+/* Copyright 2013 Saso Kiselkov. All rights reserved. */
+
+#ifndef _SYS_SHA2_H
+#define _SYS_SHA2_H
+
+#include <sys/types.h> /* for uint_* */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SHA2_HMAC_MIN_KEY_LEN 1 /* SHA2-HMAC min key length in bytes */
+#define SHA2_HMAC_MAX_KEY_LEN INT_MAX /* SHA2-HMAC max key length in bytes */
+
+#define SHA256_DIGEST_LENGTH 32 /* SHA256 digest length in bytes */
+
+#define SHA256_HMAC_BLOCK_SIZE 64 /* SHA256-HMAC block size */
+
+#define SHA256 0
+#define SHA256_HMAC 1
+#define SHA256_HMAC_GEN 2
+
+/*
+ * SHA2 context.
+ * The contents of this structure are a private interface between the
+ * Init/Update/Final calls of the functions defined below.
+ * Callers must never attempt to read or write any of the fields
+ * in this structure directly.
+ */
+typedef struct {
+ uint32_t algotype; /* Algorithm Type */
+
+ /* state (ABCDEFGH) */
+ union {
+ uint32_t s32[8]; /* for SHA256 */
+ uint64_t s64[8]; /* for SHA384/512 */
+ } state;
+ /* number of bits */
+ union {
+ uint32_t c32[2]; /* for SHA256 , modulo 2^64 */
+ uint64_t c64[2]; /* for SHA384/512, modulo 2^128 */
+ } count;
+ union {
+ uint8_t buf8[128]; /* undigested input */
+ uint32_t buf32[32]; /* realigned input */
+ uint64_t buf64[16]; /* realigned input */
+ } buf_un;
+} SHA2_CTX;
+
+typedef SHA2_CTX SHA256_CTX;
+typedef SHA2_CTX SHA384_CTX;
+typedef SHA2_CTX SHA512_CTX;
+
+extern void SHA2Init(uint64_t mech, SHA2_CTX *);
+
+extern void SHA2Update(SHA2_CTX *, const void *, size_t);
+
+extern void SHA2Final(void *, SHA2_CTX *);
+
+extern void SHA256Init(SHA256_CTX *);
+
+extern void SHA256Update(SHA256_CTX *, const void *, size_t);
+
+extern void SHA256Final(void *, SHA256_CTX *);
+
+#ifdef _SHA2_IMPL
+/*
+ * The following types/functions are all private to the implementation
+ * of the SHA2 functions and must not be used by consumers of the interface
+ */
+
+/*
+ * List of support mechanisms in this module.
+ *
+ * It is important to note that in the module, division or modulus calculations
+ * are used on the enumerated type to determine which mechanism is being used;
+ * therefore, changing the order or additional mechanisms should be done
+ * carefully
+ */
+typedef enum sha2_mech_type {
+ SHA256_MECH_INFO_TYPE, /* SUN_CKM_SHA256 */
+ SHA256_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC */
+ SHA256_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC_GENERAL */
+} sha2_mech_type_t;
+
+#endif /* _SHA2_IMPL */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_SHA2_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_SHA2_CONSTS_H
+#define _SYS_SHA2_CONSTS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Loading 32-bit constants on a sparc is expensive since it involves both
+ * a `sethi' and an `or'. thus, we instead use `ld' to load the constants
+ * from an array called `sha2_consts'. however, on intel (and perhaps other
+ * processors), it is cheaper to load the constant directly. thus, the c
+ * code in SHA transform functions uses the macro SHA2_CONST() which either
+ * expands to a constant or an array reference, depending on
+ * the architecture the code is being compiled for.
+ *
+ * SHA512 constants are used for SHA384
+ */
+
+#include <sys/types.h> /* uint32_t */
+
+extern const uint32_t sha256_consts[];
+extern const uint64_t sha512_consts[];
+
+#if defined(__sparc)
+#define SHA256_CONST(x) (sha256_consts[x])
+#define SHA512_CONST(x) (sha512_consts[x])
+#else
+#define SHA256_CONST(x) (SHA256_CONST_ ## x)
+#define SHA512_CONST(x) (SHA512_CONST_ ## x)
+#endif
+
+/* constants, as provided in FIPS 180-2 */
+
+#define SHA256_CONST_0 0x428a2f98U
+#define SHA256_CONST_1 0x71374491U
+#define SHA256_CONST_2 0xb5c0fbcfU
+#define SHA256_CONST_3 0xe9b5dba5U
+#define SHA256_CONST_4 0x3956c25bU
+#define SHA256_CONST_5 0x59f111f1U
+#define SHA256_CONST_6 0x923f82a4U
+#define SHA256_CONST_7 0xab1c5ed5U
+
+#define SHA256_CONST_8 0xd807aa98U
+#define SHA256_CONST_9 0x12835b01U
+#define SHA256_CONST_10 0x243185beU
+#define SHA256_CONST_11 0x550c7dc3U
+#define SHA256_CONST_12 0x72be5d74U
+#define SHA256_CONST_13 0x80deb1feU
+#define SHA256_CONST_14 0x9bdc06a7U
+#define SHA256_CONST_15 0xc19bf174U
+
+#define SHA256_CONST_16 0xe49b69c1U
+#define SHA256_CONST_17 0xefbe4786U
+#define SHA256_CONST_18 0x0fc19dc6U
+#define SHA256_CONST_19 0x240ca1ccU
+#define SHA256_CONST_20 0x2de92c6fU
+#define SHA256_CONST_21 0x4a7484aaU
+#define SHA256_CONST_22 0x5cb0a9dcU
+#define SHA256_CONST_23 0x76f988daU
+
+#define SHA256_CONST_24 0x983e5152U
+#define SHA256_CONST_25 0xa831c66dU
+#define SHA256_CONST_26 0xb00327c8U
+#define SHA256_CONST_27 0xbf597fc7U
+#define SHA256_CONST_28 0xc6e00bf3U
+#define SHA256_CONST_29 0xd5a79147U
+#define SHA256_CONST_30 0x06ca6351U
+#define SHA256_CONST_31 0x14292967U
+
+#define SHA256_CONST_32 0x27b70a85U
+#define SHA256_CONST_33 0x2e1b2138U
+#define SHA256_CONST_34 0x4d2c6dfcU
+#define SHA256_CONST_35 0x53380d13U
+#define SHA256_CONST_36 0x650a7354U
+#define SHA256_CONST_37 0x766a0abbU
+#define SHA256_CONST_38 0x81c2c92eU
+#define SHA256_CONST_39 0x92722c85U
+
+#define SHA256_CONST_40 0xa2bfe8a1U
+#define SHA256_CONST_41 0xa81a664bU
+#define SHA256_CONST_42 0xc24b8b70U
+#define SHA256_CONST_43 0xc76c51a3U
+#define SHA256_CONST_44 0xd192e819U
+#define SHA256_CONST_45 0xd6990624U
+#define SHA256_CONST_46 0xf40e3585U
+#define SHA256_CONST_47 0x106aa070U
+
+#define SHA256_CONST_48 0x19a4c116U
+#define SHA256_CONST_49 0x1e376c08U
+#define SHA256_CONST_50 0x2748774cU
+#define SHA256_CONST_51 0x34b0bcb5U
+#define SHA256_CONST_52 0x391c0cb3U
+#define SHA256_CONST_53 0x4ed8aa4aU
+#define SHA256_CONST_54 0x5b9cca4fU
+#define SHA256_CONST_55 0x682e6ff3U
+
+#define SHA256_CONST_56 0x748f82eeU
+#define SHA256_CONST_57 0x78a5636fU
+#define SHA256_CONST_58 0x84c87814U
+#define SHA256_CONST_59 0x8cc70208U
+#define SHA256_CONST_60 0x90befffaU
+#define SHA256_CONST_61 0xa4506cebU
+#define SHA256_CONST_62 0xbef9a3f7U
+#define SHA256_CONST_63 0xc67178f2U
+
+#define SHA512_CONST_0 0x428a2f98d728ae22ULL
+#define SHA512_CONST_1 0x7137449123ef65cdULL
+#define SHA512_CONST_2 0xb5c0fbcfec4d3b2fULL
+#define SHA512_CONST_3 0xe9b5dba58189dbbcULL
+#define SHA512_CONST_4 0x3956c25bf348b538ULL
+#define SHA512_CONST_5 0x59f111f1b605d019ULL
+#define SHA512_CONST_6 0x923f82a4af194f9bULL
+#define SHA512_CONST_7 0xab1c5ed5da6d8118ULL
+#define SHA512_CONST_8 0xd807aa98a3030242ULL
+#define SHA512_CONST_9 0x12835b0145706fbeULL
+#define SHA512_CONST_10 0x243185be4ee4b28cULL
+#define SHA512_CONST_11 0x550c7dc3d5ffb4e2ULL
+#define SHA512_CONST_12 0x72be5d74f27b896fULL
+#define SHA512_CONST_13 0x80deb1fe3b1696b1ULL
+#define SHA512_CONST_14 0x9bdc06a725c71235ULL
+#define SHA512_CONST_15 0xc19bf174cf692694ULL
+#define SHA512_CONST_16 0xe49b69c19ef14ad2ULL
+#define SHA512_CONST_17 0xefbe4786384f25e3ULL
+#define SHA512_CONST_18 0x0fc19dc68b8cd5b5ULL
+#define SHA512_CONST_19 0x240ca1cc77ac9c65ULL
+#define SHA512_CONST_20 0x2de92c6f592b0275ULL
+#define SHA512_CONST_21 0x4a7484aa6ea6e483ULL
+#define SHA512_CONST_22 0x5cb0a9dcbd41fbd4ULL
+#define SHA512_CONST_23 0x76f988da831153b5ULL
+#define SHA512_CONST_24 0x983e5152ee66dfabULL
+#define SHA512_CONST_25 0xa831c66d2db43210ULL
+#define SHA512_CONST_26 0xb00327c898fb213fULL
+#define SHA512_CONST_27 0xbf597fc7beef0ee4ULL
+#define SHA512_CONST_28 0xc6e00bf33da88fc2ULL
+#define SHA512_CONST_29 0xd5a79147930aa725ULL
+#define SHA512_CONST_30 0x06ca6351e003826fULL
+#define SHA512_CONST_31 0x142929670a0e6e70ULL
+#define SHA512_CONST_32 0x27b70a8546d22ffcULL
+#define SHA512_CONST_33 0x2e1b21385c26c926ULL
+#define SHA512_CONST_34 0x4d2c6dfc5ac42aedULL
+#define SHA512_CONST_35 0x53380d139d95b3dfULL
+#define SHA512_CONST_36 0x650a73548baf63deULL
+#define SHA512_CONST_37 0x766a0abb3c77b2a8ULL
+#define SHA512_CONST_38 0x81c2c92e47edaee6ULL
+#define SHA512_CONST_39 0x92722c851482353bULL
+#define SHA512_CONST_40 0xa2bfe8a14cf10364ULL
+#define SHA512_CONST_41 0xa81a664bbc423001ULL
+#define SHA512_CONST_42 0xc24b8b70d0f89791ULL
+#define SHA512_CONST_43 0xc76c51a30654be30ULL
+#define SHA512_CONST_44 0xd192e819d6ef5218ULL
+#define SHA512_CONST_45 0xd69906245565a910ULL
+#define SHA512_CONST_46 0xf40e35855771202aULL
+#define SHA512_CONST_47 0x106aa07032bbd1b8ULL
+#define SHA512_CONST_48 0x19a4c116b8d2d0c8ULL
+#define SHA512_CONST_49 0x1e376c085141ab53ULL
+#define SHA512_CONST_50 0x2748774cdf8eeb99ULL
+#define SHA512_CONST_51 0x34b0bcb5e19b48a8ULL
+#define SHA512_CONST_52 0x391c0cb3c5c95a63ULL
+#define SHA512_CONST_53 0x4ed8aa4ae3418acbULL
+#define SHA512_CONST_54 0x5b9cca4f7763e373ULL
+#define SHA512_CONST_55 0x682e6ff3d6b2b8a3ULL
+#define SHA512_CONST_56 0x748f82ee5defb2fcULL
+#define SHA512_CONST_57 0x78a5636f43172f60ULL
+#define SHA512_CONST_58 0x84c87814a1f0ab72ULL
+#define SHA512_CONST_59 0x8cc702081a6439ecULL
+#define SHA512_CONST_60 0x90befffa23631e28ULL
+#define SHA512_CONST_61 0xa4506cebde82bde9ULL
+#define SHA512_CONST_62 0xbef9a3f7b2c67915ULL
+#define SHA512_CONST_63 0xc67178f2e372532bULL
+#define SHA512_CONST_64 0xca273eceea26619cULL
+#define SHA512_CONST_65 0xd186b8c721c0c207ULL
+#define SHA512_CONST_66 0xeada7dd6cde0eb1eULL
+#define SHA512_CONST_67 0xf57d4f7fee6ed178ULL
+#define SHA512_CONST_68 0x06f067aa72176fbaULL
+#define SHA512_CONST_69 0x0a637dc5a2c898a6ULL
+#define SHA512_CONST_70 0x113f9804bef90daeULL
+#define SHA512_CONST_71 0x1b710b35131c471bULL
+#define SHA512_CONST_72 0x28db77f523047d84ULL
+#define SHA512_CONST_73 0x32caab7b40c72493ULL
+#define SHA512_CONST_74 0x3c9ebe0a15c9bebcULL
+#define SHA512_CONST_75 0x431d67c49c100d4cULL
+#define SHA512_CONST_76 0x4cc5d4becb3e42b6ULL
+#define SHA512_CONST_77 0x597f299cfc657e2aULL
+#define SHA512_CONST_78 0x5fcb6fab3ad6faecULL
+#define SHA512_CONST_79 0x6c44198c4a475817ULL
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_SHA2_CONSTS_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SHA2_IMPL_H
+#define _SHA2_IMPL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum {
+ SHA1_TYPE,
+ SHA256_TYPE,
+ SHA384_TYPE,
+ SHA512_TYPE
+} sha2_mech_t;
+
+/*
+ * Context for SHA2 mechanism.
+ */
+typedef struct sha2_ctx {
+ sha2_mech_type_t sc_mech_type; /* type of context */
+ SHA2_CTX sc_sha2_ctx; /* SHA2 context */
+} sha2_ctx_t;
+
+/*
+ * Context for SHA2 HMAC and HMAC GENERAL mechanisms.
+ */
+typedef struct sha2_hmac_ctx {
+ sha2_mech_type_t hc_mech_type; /* type of context */
+ uint32_t hc_digest_len; /* digest len in bytes */
+ SHA2_CTX hc_icontext; /* inner SHA2 context */
+ SHA2_CTX hc_ocontext; /* outer SHA2 context */
+} sha2_hmac_ctx_t;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SHA2_IMPL_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_ASM_LINKAGE_H
+#define _SYS_ASM_LINKAGE_H
+
+#if defined(__i386) || defined(__amd64)
+
+#include <sys/ia32/asm_linkage.h> /* XX64 x86/sys/asm_linkage.h */
+
+#endif
+
+#endif /* _SYS_ASM_LINKAGE_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
+/* All Rights Reserved */
+
+
+#ifndef _SYS_BITMAP_H
+#define _SYS_BITMAP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if defined(__GNUC__) && defined(_ASM_INLINES) && \
+ (defined(__i386) || defined(__amd64))
+#include <asm/bitmap.h>
+#endif
+
+/*
+ * Operations on bitmaps of arbitrary size
+ * A bitmap is a vector of 1 or more ulong_t's.
+ * The user of the package is responsible for range checks and keeping
+ * track of sizes.
+ */
+
+#ifdef _LP64
+#define BT_ULSHIFT 6 /* log base 2 of BT_NBIPUL, to extract word index */
+#define BT_ULSHIFT32 5 /* log base 2 of BT_NBIPUL, to extract word index */
+#else
+#define BT_ULSHIFT 5 /* log base 2 of BT_NBIPUL, to extract word index */
+#endif
+
+#define BT_NBIPUL (1 << BT_ULSHIFT) /* n bits per ulong_t */
+#define BT_ULMASK (BT_NBIPUL - 1) /* to extract bit index */
+
+#ifdef _LP64
+#define BT_NBIPUL32 (1 << BT_ULSHIFT32) /* n bits per ulong_t */
+#define BT_ULMASK32 (BT_NBIPUL32 - 1) /* to extract bit index */
+#define BT_ULMAXMASK 0xffffffffffffffff /* used by bt_getlowbit */
+#else
+#define BT_ULMAXMASK 0xffffffff
+#endif
+
+/*
+ * bitmap is a ulong_t *, bitindex an index_t
+ *
+ * The macros BT_WIM and BT_BIW internal; there is no need
+ * for users of this package to use them.
+ */
+
+/*
+ * word in map
+ */
+#define BT_WIM(bitmap, bitindex) \
+ ((bitmap)[(bitindex) >> BT_ULSHIFT])
+/*
+ * bit in word
+ */
+#define BT_BIW(bitindex) \
+ (1UL << ((bitindex) & BT_ULMASK))
+
+#ifdef _LP64
+#define BT_WIM32(bitmap, bitindex) \
+ ((bitmap)[(bitindex) >> BT_ULSHIFT32])
+
+#define BT_BIW32(bitindex) \
+ (1UL << ((bitindex) & BT_ULMASK32))
+#endif
+
+/*
+ * These are public macros
+ *
+ * BT_BITOUL == n bits to n ulong_t's
+ */
+#define BT_BITOUL(nbits) \
+ (((nbits) + BT_NBIPUL - 1l) / BT_NBIPUL)
+#define BT_SIZEOFMAP(nbits) \
+ (BT_BITOUL(nbits) * sizeof (ulong_t))
+#define BT_TEST(bitmap, bitindex) \
+ ((BT_WIM((bitmap), (bitindex)) & BT_BIW(bitindex)) ? 1 : 0)
+#define BT_SET(bitmap, bitindex) \
+ { BT_WIM((bitmap), (bitindex)) |= BT_BIW(bitindex); }
+#define BT_CLEAR(bitmap, bitindex) \
+ { BT_WIM((bitmap), (bitindex)) &= ~BT_BIW(bitindex); }
+
+#ifdef _LP64
+#define BT_BITOUL32(nbits) \
+ (((nbits) + BT_NBIPUL32 - 1l) / BT_NBIPUL32)
+#define BT_SIZEOFMAP32(nbits) \
+ (BT_BITOUL32(nbits) * sizeof (uint_t))
+#define BT_TEST32(bitmap, bitindex) \
+ ((BT_WIM32((bitmap), (bitindex)) & BT_BIW32(bitindex)) ? 1 : 0)
+#define BT_SET32(bitmap, bitindex) \
+ { BT_WIM32((bitmap), (bitindex)) |= BT_BIW32(bitindex); }
+#define BT_CLEAR32(bitmap, bitindex) \
+ { BT_WIM32((bitmap), (bitindex)) &= ~BT_BIW32(bitindex); }
+#endif /* _LP64 */
+
+
+/*
+ * BIT_ONLYONESET is a private macro not designed for bitmaps of
+ * arbitrary size. u must be an unsigned integer/long. It returns
+ * true if one and only one bit is set in u.
+ */
+#define BIT_ONLYONESET(u) \
+ ((((u) == 0) ? 0 : ((u) & ((u) - 1)) == 0))
+
+#ifndef _ASM
+
+/*
+ * return next available bit index from map with specified number of bits
+ */
+extern index_t bt_availbit(ulong_t *bitmap, size_t nbits);
+/*
+ * find the highest order bit that is on, and is within or below
+ * the word specified by wx
+ */
+extern int bt_gethighbit(ulong_t *mapp, int wx);
+extern int bt_range(ulong_t *bitmap, size_t *pos1, size_t *pos2,
+ size_t end_pos);
+extern int bt_getlowbit(ulong_t *bitmap, size_t start, size_t stop);
+extern void bt_copy(ulong_t *, ulong_t *, ulong_t);
+
+/*
+ * find the parity
+ */
+extern int odd_parity(ulong_t);
+
+/*
+ * Atomically set/clear bits
+ * Atomic exclusive operations will set "result" to "-1"
+ * if the bit is already set/cleared. "result" will be set
+ * to 0 otherwise.
+ */
+#define BT_ATOMIC_SET(bitmap, bitindex) \
+ { atomic_or_long(&(BT_WIM(bitmap, bitindex)), BT_BIW(bitindex)); }
+#define BT_ATOMIC_CLEAR(bitmap, bitindex) \
+ { atomic_and_long(&(BT_WIM(bitmap, bitindex)), ~BT_BIW(bitindex)); }
+
+#define BT_ATOMIC_SET_EXCL(bitmap, bitindex, result) \
+ { result = atomic_set_long_excl(&(BT_WIM(bitmap, bitindex)), \
+ (bitindex) % BT_NBIPUL); }
+#define BT_ATOMIC_CLEAR_EXCL(bitmap, bitindex, result) \
+ { result = atomic_clear_long_excl(&(BT_WIM(bitmap, bitindex)), \
+ (bitindex) % BT_NBIPUL); }
+
+/*
+ * Extracts bits between index h (high, inclusive) and l (low, exclusive) from
+ * u, which must be an unsigned integer.
+ */
+#define BITX(u, h, l) (((u) >> (l)) & ((1LU << ((h) - (l) + 1LU)) - 1LU))
+
+#endif /* _ASM */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_BITMAP_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_CRYPTO_ELFSIGN_H
+#define _SYS_CRYPTO_ELFSIGN_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Consolidation Private Interface for elfsign/libpkcs11/kcfd
+ */
+
+#include <sys/zfs_context.h>
+
+/*
+ * Project Private structures and types used for communication between kcfd
+ * and KCF over the door.
+ */
+
+typedef enum ELFsign_status_e {
+ ELFSIGN_UNKNOWN,
+ ELFSIGN_SUCCESS,
+ ELFSIGN_FAILED,
+ ELFSIGN_NOTSIGNED,
+ ELFSIGN_INVALID_CERTPATH,
+ ELFSIGN_INVALID_ELFOBJ,
+ ELFSIGN_RESTRICTED
+} ELFsign_status_t;
+
+#define KCF_KCFD_VERSION1 1
+#define SIG_MAX_LENGTH 1024
+
+#define ELF_SIGNATURE_SECTION ".SUNW_signature"
+
+typedef struct kcf_door_arg_s {
+ short da_version;
+ boolean_t da_iskernel;
+
+ union {
+ char filename[MAXPATHLEN]; /* For request */
+
+ struct kcf_door_result_s { /* For response */
+ ELFsign_status_t status;
+ uint32_t siglen;
+ uchar_t signature[1];
+ } result;
+ } da_u;
+} kcf_door_arg_t;
+
+typedef uint32_t filesig_vers_t;
+
+/*
+ * File Signature Structure
+ * Applicable to ELF and other file formats
+ */
+struct filesignatures {
+ uint32_t filesig_cnt; /* count of signatures */
+ uint32_t filesig_pad; /* unused */
+ union {
+ char filesig_data[1];
+ struct filesig { /* one of these for each signature */
+ uint32_t filesig_size;
+ filesig_vers_t filesig_version;
+ union {
+ struct filesig_version1 {
+ uint32_t filesig_v1_dnsize;
+ uint32_t filesig_v1_sigsize;
+ uint32_t filesig_v1_oidsize;
+ char filesig_v1_data[1];
+ } filesig_v1;
+ struct filesig_version3 {
+ uint64_t filesig_v3_time;
+ uint32_t filesig_v3_dnsize;
+ uint32_t filesig_v3_sigsize;
+ uint32_t filesig_v3_oidsize;
+ char filesig_v3_data[1];
+ } filesig_v3;
+ } _u2;
+ } filesig_sig;
+ uint64_t filesig_align;
+ } _u1;
+};
+#define filesig_sig _u1.filesig_sig
+
+#define filesig_v1_dnsize _u2.filesig_v1.filesig_v1_dnsize
+#define filesig_v1_sigsize _u2.filesig_v1.filesig_v1_sigsize
+#define filesig_v1_oidsize _u2.filesig_v1.filesig_v1_oidsize
+#define filesig_v1_data _u2.filesig_v1.filesig_v1_data
+
+#define filesig_v3_time _u2.filesig_v3.filesig_v3_time
+#define filesig_v3_dnsize _u2.filesig_v3.filesig_v3_dnsize
+#define filesig_v3_sigsize _u2.filesig_v3.filesig_v3_sigsize
+#define filesig_v3_oidsize _u2.filesig_v3.filesig_v3_oidsize
+#define filesig_v3_data _u2.filesig_v3.filesig_v3_data
+
+#define filesig_ALIGN(s) (((s) + sizeof (uint64_t) - 1) & \
+ (-sizeof (uint64_t)))
+#define filesig_next(ptr) (struct filesig *)((void *)((char *)(ptr) + \
+ filesig_ALIGN((ptr)->filesig_size)))
+
+#define FILESIG_UNKNOWN 0 /* unrecognized version */
+#define FILESIG_VERSION1 1 /* version1, all but sig section */
+#define FILESIG_VERSION2 2 /* version1 format, SHF_ALLOC only */
+#define FILESIG_VERSION3 3 /* version3, all but sig section */
+#define FILESIG_VERSION4 4 /* version3 format, SHF_ALLOC only */
+
+#define _PATH_KCFD_DOOR "/etc/svc/volatile/kcfd_door"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_CRYPTO_ELFSIGN_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_CRYPTO_IMPL_H
+#define _SYS_CRYPTO_IMPL_H
+
+/*
+ * Kernel Cryptographic Framework private implementation definitions.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/spi.h>
+#include <sys/crypto/ioctl.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define KCF_MODULE "kcf"
+
+/*
+ * Prefixes convention: structures internal to the kernel cryptographic
+ * framework start with 'kcf_'. Exposed structure start with 'crypto_'.
+ */
+
+/* Provider stats. Not protected. */
+typedef struct kcf_prov_stats {
+ kstat_named_t ps_ops_total;
+ kstat_named_t ps_ops_passed;
+ kstat_named_t ps_ops_failed;
+ kstat_named_t ps_ops_busy_rval;
+} kcf_prov_stats_t;
+
+/* Various kcf stats. Not protected. */
+typedef struct kcf_stats {
+ kstat_named_t ks_thrs_in_pool;
+ kstat_named_t ks_idle_thrs;
+ kstat_named_t ks_minthrs;
+ kstat_named_t ks_maxthrs;
+ kstat_named_t ks_swq_njobs;
+ kstat_named_t ks_swq_maxjobs;
+ kstat_named_t ks_taskq_threads;
+ kstat_named_t ks_taskq_minalloc;
+ kstat_named_t ks_taskq_maxalloc;
+} kcf_stats_t;
+
+/*
+ * Keep all the information needed by the scheduler from
+ * this provider.
+ */
+typedef struct kcf_sched_info {
+ /* The number of operations dispatched. */
+ uint64_t ks_ndispatches;
+
+ /* The number of operations that failed. */
+ uint64_t ks_nfails;
+
+ /* The number of operations that returned CRYPTO_BUSY. */
+ uint64_t ks_nbusy_rval;
+
+ /* taskq used to dispatch crypto requests */
+ taskq_t *ks_taskq;
+} kcf_sched_info_t;
+
+/*
+ * pd_irefcnt approximates the number of inflight requests to the
+ * provider. Though we increment this counter during registration for
+ * other purposes, that base value is mostly same across all providers.
+ * So, it is a good measure of the load on a provider when it is not
+ * in a busy state. Once a provider notifies it is busy, requests
+ * backup in the taskq. So, we use tq_nalloc in that case which gives
+ * the number of task entries in the task queue. Note that we do not
+ * acquire any locks here as it is not critical to get the exact number
+ * and the lock contention may be too costly for this code path.
+ */
+#define KCF_PROV_LOAD(pd) ((pd)->pd_state != KCF_PROV_BUSY ? \
+ (pd)->pd_irefcnt : (pd)->pd_sched_info.ks_taskq->tq_nalloc)
+
+#define KCF_PROV_INCRSTATS(pd, error) { \
+ (pd)->pd_sched_info.ks_ndispatches++; \
+ if (error == CRYPTO_BUSY) \
+ (pd)->pd_sched_info.ks_nbusy_rval++; \
+ else if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED) \
+ (pd)->pd_sched_info.ks_nfails++; \
+}
+
+
+/*
+ * The following two macros should be
+ * #define KCF_OPS_CLASSSIZE (KCF_LAST_OPSCLASS - KCF_FIRST_OPSCLASS + 2)
+ * #define KCF_MAXMECHTAB KCF_MAXCIPHER
+ *
+ * However, doing that would involve reorganizing the header file a bit.
+ * When impl.h is broken up (bug# 4703218), this will be done. For now,
+ * we hardcode these values.
+ */
+#define KCF_OPS_CLASSSIZE 8
+#define KCF_MAXMECHTAB 32
+
+/*
+ * Valid values for the state of a provider. The order of
+ * the elements is important.
+ *
+ * Routines which get a provider or the list of providers
+ * should pick only those that are either in KCF_PROV_READY state
+ * or in KCF_PROV_BUSY state.
+ */
+typedef enum {
+ KCF_PROV_ALLOCATED = 1,
+ KCF_PROV_UNVERIFIED,
+ KCF_PROV_VERIFICATION_FAILED,
+ /*
+ * state < KCF_PROV_READY means the provider can not
+ * be used at all.
+ */
+ KCF_PROV_READY,
+ KCF_PROV_BUSY,
+ /*
+ * state > KCF_PROV_BUSY means the provider can not
+ * be used for new requests.
+ */
+ KCF_PROV_FAILED,
+ /*
+ * Threads setting the following two states should do so only
+ * if the current state < KCF_PROV_DISABLED.
+ */
+ KCF_PROV_DISABLED,
+ KCF_PROV_REMOVED,
+ KCF_PROV_FREED
+} kcf_prov_state_t;
+
+#define KCF_IS_PROV_UNVERIFIED(pd) ((pd)->pd_state == KCF_PROV_UNVERIFIED)
+#define KCF_IS_PROV_USABLE(pd) ((pd)->pd_state == KCF_PROV_READY || \
+ (pd)->pd_state == KCF_PROV_BUSY)
+#define KCF_IS_PROV_REMOVED(pd) ((pd)->pd_state >= KCF_PROV_REMOVED)
+
+/* Internal flags valid for pd_flags field */
+#define KCF_PROV_RESTRICTED 0x40000000
+#define KCF_LPROV_MEMBER 0x80000000 /* is member of a logical provider */
+
+/*
+ * A provider descriptor structure. There is one such structure per
+ * provider. It is allocated and initialized at registration time and
+ * freed when the provider unregisters.
+ *
+ * pd_prov_type: Provider type, hardware or software
+ * pd_sid: Session ID of the provider used by kernel clients.
+ * This is valid only for session-oriented providers.
+ * pd_refcnt: Reference counter to this provider descriptor
+ * pd_irefcnt: References held by the framework internal structs
+ * pd_lock: lock protects pd_state and pd_provider_list
+ * pd_state: State value of the provider
+ * pd_provider_list: Used to cross-reference logical providers and their
+ * members. Not used for software providers.
+ * pd_resume_cv: cv to wait for state to change from KCF_PROV_BUSY
+ * pd_prov_handle: Provider handle specified by provider
+ * pd_ops_vector: The ops vector specified by Provider
+ * pd_mech_indx: Lookup table which maps a core framework mechanism
+ * number to an index in pd_mechanisms array
+ * pd_mechanisms: Array of mechanisms supported by the provider, specified
+ * by the provider during registration
+ * pd_sched_info: Scheduling information associated with the provider
+ * pd_mech_list_count: The number of entries in pi_mechanisms, specified
+ * by the provider during registration
+ * pd_name: Device name or module name
+ * pd_instance: Device instance
+ * pd_module_id: Module ID returned by modload
+ * pd_mctlp: Pointer to modctl structure for this provider
+ * pd_remove_cv: cv to wait on while the provider queue drains
+ * pd_description: Provider description string
+ * pd_flags bitwise OR of pi_flags from crypto_provider_info_t
+ * and other internal flags defined above.
+ * pd_hash_limit Maximum data size that hash mechanisms of this provider
+ * can support.
+ * pd_kcf_prov_handle: KCF-private handle assigned by KCF
+ * pd_prov_id: Identification # assigned by KCF to provider
+ * pd_kstat: kstat associated with the provider
+ * pd_ks_data: kstat data
+ */
+typedef struct kcf_provider_desc {
+ crypto_provider_type_t pd_prov_type;
+ crypto_session_id_t pd_sid;
+ uint_t pd_refcnt;
+ uint_t pd_irefcnt;
+ kmutex_t pd_lock;
+ kcf_prov_state_t pd_state;
+ struct kcf_provider_list *pd_provider_list;
+ kcondvar_t pd_resume_cv;
+ crypto_provider_handle_t pd_prov_handle;
+ crypto_ops_t *pd_ops_vector;
+ ushort_t pd_mech_indx[KCF_OPS_CLASSSIZE]\
+ [KCF_MAXMECHTAB];
+ crypto_mech_info_t *pd_mechanisms;
+ kcf_sched_info_t pd_sched_info;
+ uint_t pd_mech_list_count;
+ // char *pd_name;
+ // uint_t pd_instance;
+ // int pd_module_id;
+ // struct modctl *pd_mctlp;
+ kcondvar_t pd_remove_cv;
+ char *pd_description;
+ uint_t pd_flags;
+ uint_t pd_hash_limit;
+ crypto_kcf_provider_handle_t pd_kcf_prov_handle;
+ crypto_provider_id_t pd_prov_id;
+ kstat_t *pd_kstat;
+ kcf_prov_stats_t pd_ks_data;
+} kcf_provider_desc_t;
+
+/* useful for making a list of providers */
+typedef struct kcf_provider_list {
+ struct kcf_provider_list *pl_next;
+ struct kcf_provider_desc *pl_provider;
+} kcf_provider_list_t;
+
+/* atomic operations in linux implictly form a memory barrier */
+#define membar_exit()
+
+/*
+ * If a component has a reference to a kcf_provider_desc_t,
+ * it REFHOLD()s. A new provider descriptor which is referenced only
+ * by the providers table has a reference counter of one.
+ */
+#define KCF_PROV_REFHOLD(desc) { \
+ atomic_add_32(&(desc)->pd_refcnt, 1); \
+ ASSERT((desc)->pd_refcnt != 0); \
+}
+
+#define KCF_PROV_IREFHOLD(desc) { \
+ atomic_add_32(&(desc)->pd_irefcnt, 1); \
+ ASSERT((desc)->pd_irefcnt != 0); \
+}
+
+#define KCF_PROV_IREFRELE(desc) { \
+ ASSERT((desc)->pd_irefcnt != 0); \
+ membar_exit(); \
+ if (atomic_add_32_nv(&(desc)->pd_irefcnt, -1) == 0) { \
+ cv_broadcast(&(desc)->pd_remove_cv); \
+ } \
+}
+
+#define KCF_PROV_REFHELD(desc) ((desc)->pd_refcnt >= 1)
+
+#define KCF_PROV_REFRELE(desc) { \
+ ASSERT((desc)->pd_refcnt != 0); \
+ membar_exit(); \
+ if (atomic_add_32_nv(&(desc)->pd_refcnt, -1) == 0) { \
+ kcf_provider_zero_refcnt((desc)); \
+ } \
+}
+
+
+/* list of crypto_mech_info_t valid as the second mech in a dual operation */
+
+typedef struct crypto_mech_info_list {
+ struct crypto_mech_info_list *ml_next;
+ crypto_mech_type_t ml_kcf_mechid; /* KCF's id */
+ crypto_mech_info_t ml_mech_info;
+} crypto_mech_info_list_t;
+
+/*
+ * An element in a mechanism provider descriptors chain.
+ * The kcf_prov_mech_desc_t is duplicated in every chain the provider belongs
+ * to. This is a small tradeoff memory vs mutex spinning time to access the
+ * common provider field.
+ */
+
+typedef struct kcf_prov_mech_desc {
+ struct kcf_mech_entry *pm_me; /* Back to the head */
+ struct kcf_prov_mech_desc *pm_next; /* Next in the chain */
+ crypto_mech_info_t pm_mech_info; /* Provider mech info */
+ crypto_mech_info_list_t *pm_mi_list; /* list for duals */
+ kcf_provider_desc_t *pm_prov_desc; /* Common desc. */
+} kcf_prov_mech_desc_t;
+
+/* and the notation shortcuts ... */
+#define pm_provider_type pm_prov_desc.pd_provider_type
+#define pm_provider_handle pm_prov_desc.pd_provider_handle
+#define pm_ops_vector pm_prov_desc.pd_ops_vector
+
+
+#define KCF_CPU_PAD (128 - sizeof (crypto_mech_name_t) - \
+ sizeof (crypto_mech_type_t) - \
+ sizeof (kmutex_t) - 2 * sizeof (kcf_prov_mech_desc_t *) - \
+ sizeof (int) - sizeof (uint32_t) - sizeof (size_t))
+
+/*
+ * A mechanism entry in an xxx_mech_tab[]. KCF_CPU_PAD needs
+ * to be adjusted if this structure is changed.
+ */
+typedef struct kcf_mech_entry {
+ crypto_mech_name_t me_name; /* mechanism name */
+ crypto_mech_type_t me_mechid; /* Internal id for mechanism */
+ kmutex_t me_mutex; /* access protection */
+ kcf_prov_mech_desc_t *me_hw_prov_chain; /* list of HW providers */
+ kcf_prov_mech_desc_t *me_sw_prov; /* SW provider */
+ /*
+ * Number of HW providers in the chain. There is only one
+ * SW provider. So, we need only a count of HW providers.
+ */
+ int me_num_hwprov;
+ /*
+ * When a SW provider is present, this is the generation number that
+ * ensures no objects from old SW providers are used in the new one
+ */
+ uint32_t me_gen_swprov;
+ /*
+ * threshold for using hardware providers for this mech
+ */
+ size_t me_threshold;
+ uint8_t me_pad[KCF_CPU_PAD];
+} kcf_mech_entry_t;
+
+/*
+ * A policy descriptor structure. It is allocated and initialized
+ * when administrative ioctls load disabled mechanisms.
+ *
+ * pd_prov_type: Provider type, hardware or software
+ * pd_name: Device name or module name.
+ * pd_instance: Device instance.
+ * pd_refcnt: Reference counter for this policy descriptor
+ * pd_mutex: Protects array and count of disabled mechanisms.
+ * pd_disabled_count: Count of disabled mechanisms.
+ * pd_disabled_mechs: Array of disabled mechanisms.
+ */
+typedef struct kcf_policy_desc {
+ crypto_provider_type_t pd_prov_type;
+ char *pd_name;
+ uint_t pd_instance;
+ uint_t pd_refcnt;
+ kmutex_t pd_mutex;
+ uint_t pd_disabled_count;
+ crypto_mech_name_t *pd_disabled_mechs;
+} kcf_policy_desc_t;
+
+/*
+ * If a component has a reference to a kcf_policy_desc_t,
+ * it REFHOLD()s. A new policy descriptor which is referenced only
+ * by the policy table has a reference count of one.
+ */
+#define KCF_POLICY_REFHOLD(desc) { \
+ atomic_add_32(&(desc)->pd_refcnt, 1); \
+ ASSERT((desc)->pd_refcnt != 0); \
+}
+
+/*
+ * Releases a reference to a policy descriptor. When the last
+ * reference is released, the descriptor is freed.
+ */
+#define KCF_POLICY_REFRELE(desc) { \
+ ASSERT((desc)->pd_refcnt != 0); \
+ membar_exit(); \
+ if (atomic_add_32_nv(&(desc)->pd_refcnt, -1) == 0) \
+ kcf_policy_free_desc(desc); \
+}
+
+/*
+ * This entry stores the name of a software module and its
+ * mechanisms. The mechanisms are 'hints' that are used to
+ * trigger loading of the module.
+ */
+typedef struct kcf_soft_conf_entry {
+ struct kcf_soft_conf_entry *ce_next;
+ char *ce_name;
+ crypto_mech_name_t *ce_mechs;
+ uint_t ce_count;
+} kcf_soft_conf_entry_t;
+
+extern kmutex_t soft_config_mutex;
+extern kcf_soft_conf_entry_t *soft_config_list;
+
+/*
+ * Global tables. The sizes are from the predefined PKCS#11 v2.20 mechanisms,
+ * with a margin of few extra empty entry points
+ */
+
+#define KCF_MAXDIGEST 16 /* Digests */
+#define KCF_MAXCIPHER 64 /* Ciphers */
+#define KCF_MAXMAC 40 /* Message authentication codes */
+#define KCF_MAXSIGN 24 /* Sign/Verify */
+#define KCF_MAXKEYOPS 116 /* Key generation and derivation */
+#define KCF_MAXMISC 16 /* Others ... */
+
+#define KCF_MAXMECHS KCF_MAXDIGEST + KCF_MAXCIPHER + KCF_MAXMAC + \
+ KCF_MAXSIGN + KCF_MAXKEYOPS + \
+ KCF_MAXMISC
+
+extern kcf_mech_entry_t kcf_digest_mechs_tab[];
+extern kcf_mech_entry_t kcf_cipher_mechs_tab[];
+extern kcf_mech_entry_t kcf_mac_mechs_tab[];
+extern kcf_mech_entry_t kcf_sign_mechs_tab[];
+extern kcf_mech_entry_t kcf_keyops_mechs_tab[];
+extern kcf_mech_entry_t kcf_misc_mechs_tab[];
+
+extern kmutex_t kcf_mech_tabs_lock;
+
+typedef enum {
+ KCF_DIGEST_CLASS = 1,
+ KCF_CIPHER_CLASS,
+ KCF_MAC_CLASS,
+ KCF_SIGN_CLASS,
+ KCF_KEYOPS_CLASS,
+ KCF_MISC_CLASS
+} kcf_ops_class_t;
+
+#define KCF_FIRST_OPSCLASS KCF_DIGEST_CLASS
+#define KCF_LAST_OPSCLASS KCF_MISC_CLASS
+
+/* The table of all the kcf_xxx_mech_tab[]s, indexed by kcf_ops_class */
+
+typedef struct kcf_mech_entry_tab {
+ int met_size; /* Size of the met_tab[] */
+ kcf_mech_entry_t *met_tab; /* the table */
+} kcf_mech_entry_tab_t;
+
+extern kcf_mech_entry_tab_t kcf_mech_tabs_tab[];
+
+#define KCF_MECHID(class, index) \
+ (((crypto_mech_type_t)(class) << 32) | (crypto_mech_type_t)(index))
+
+#define KCF_MECH2CLASS(mech_type) ((kcf_ops_class_t)((mech_type) >> 32))
+
+#define KCF_MECH2INDEX(mech_type) ((int)(mech_type))
+
+#define KCF_TO_PROV_MECH_INDX(pd, mech_type) \
+ ((pd)->pd_mech_indx[KCF_MECH2CLASS(mech_type)] \
+ [KCF_MECH2INDEX(mech_type)])
+
+#define KCF_TO_PROV_MECHINFO(pd, mech_type) \
+ ((pd)->pd_mechanisms[KCF_TO_PROV_MECH_INDX(pd, mech_type)])
+
+#define KCF_TO_PROV_MECHNUM(pd, mech_type) \
+ (KCF_TO_PROV_MECHINFO(pd, mech_type).cm_mech_number)
+
+#define KCF_CAN_SHARE_OPSTATE(pd, mech_type) \
+ ((KCF_TO_PROV_MECHINFO(pd, mech_type).cm_mech_flags) & \
+ CRYPTO_CAN_SHARE_OPSTATE)
+
+/* ps_refcnt is protected by cm_lock in the crypto_minor structure */
+typedef struct crypto_provider_session {
+ struct crypto_provider_session *ps_next;
+ crypto_session_id_t ps_session;
+ kcf_provider_desc_t *ps_provider;
+ kcf_provider_desc_t *ps_real_provider;
+ uint_t ps_refcnt;
+} crypto_provider_session_t;
+
+typedef struct crypto_session_data {
+ kmutex_t sd_lock;
+ kcondvar_t sd_cv;
+ uint32_t sd_flags;
+ int sd_pre_approved_amount;
+ crypto_ctx_t *sd_digest_ctx;
+ crypto_ctx_t *sd_encr_ctx;
+ crypto_ctx_t *sd_decr_ctx;
+ crypto_ctx_t *sd_sign_ctx;
+ crypto_ctx_t *sd_verify_ctx;
+ crypto_ctx_t *sd_sign_recover_ctx;
+ crypto_ctx_t *sd_verify_recover_ctx;
+ kcf_provider_desc_t *sd_provider;
+ void *sd_find_init_cookie;
+ crypto_provider_session_t *sd_provider_session;
+} crypto_session_data_t;
+
+#define CRYPTO_SESSION_IN_USE 0x00000001
+#define CRYPTO_SESSION_IS_BUSY 0x00000002
+#define CRYPTO_SESSION_IS_CLOSED 0x00000004
+
+#define KCF_MAX_PIN_LEN 1024
+
+/*
+ * Per-minor info.
+ *
+ * cm_lock protects everything in this structure except for cm_refcnt.
+ */
+typedef struct crypto_minor {
+ uint_t cm_refcnt;
+ kmutex_t cm_lock;
+ kcondvar_t cm_cv;
+ crypto_session_data_t **cm_session_table;
+ uint_t cm_session_table_count;
+ kcf_provider_desc_t **cm_provider_array;
+ uint_t cm_provider_count;
+ crypto_provider_session_t *cm_provider_session;
+} crypto_minor_t;
+
+/*
+ * Return codes for internal functions
+ */
+#define KCF_SUCCESS 0x0 /* Successful call */
+#define KCF_INVALID_MECH_NUMBER 0x1 /* invalid mechanism number */
+#define KCF_INVALID_MECH_NAME 0x2 /* invalid mechanism name */
+#define KCF_INVALID_MECH_CLASS 0x3 /* invalid mechanism class */
+#define KCF_MECH_TAB_FULL 0x4 /* Need more room in the mech tabs. */
+#define KCF_INVALID_INDX ((ushort_t)-1)
+
+/*
+ * kCF internal mechanism and function group for tracking RNG providers.
+ */
+#define SUN_RANDOM "random"
+#define CRYPTO_FG_RANDOM 0x80000000 /* generate_random() */
+
+/*
+ * Wrappers for ops vectors. In the wrapper definitions below, the pd
+ * argument always corresponds to a pointer to a provider descriptor
+ * of type kcf_prov_desc_t.
+ */
+
+#define KCF_PROV_CONTROL_OPS(pd) ((pd)->pd_ops_vector->co_control_ops)
+#define KCF_PROV_CTX_OPS(pd) ((pd)->pd_ops_vector->co_ctx_ops)
+#define KCF_PROV_DIGEST_OPS(pd) ((pd)->pd_ops_vector->co_digest_ops)
+#define KCF_PROV_CIPHER_OPS(pd) ((pd)->pd_ops_vector->co_cipher_ops)
+#define KCF_PROV_MAC_OPS(pd) ((pd)->pd_ops_vector->co_mac_ops)
+#define KCF_PROV_SIGN_OPS(pd) ((pd)->pd_ops_vector->co_sign_ops)
+#define KCF_PROV_VERIFY_OPS(pd) ((pd)->pd_ops_vector->co_verify_ops)
+#define KCF_PROV_DUAL_OPS(pd) ((pd)->pd_ops_vector->co_dual_ops)
+#define KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) \
+ ((pd)->pd_ops_vector->co_dual_cipher_mac_ops)
+#define KCF_PROV_RANDOM_OPS(pd) ((pd)->pd_ops_vector->co_random_ops)
+#define KCF_PROV_SESSION_OPS(pd) ((pd)->pd_ops_vector->co_session_ops)
+#define KCF_PROV_OBJECT_OPS(pd) ((pd)->pd_ops_vector->co_object_ops)
+#define KCF_PROV_KEY_OPS(pd) ((pd)->pd_ops_vector->co_key_ops)
+#define KCF_PROV_PROVIDER_OPS(pd) ((pd)->pd_ops_vector->co_provider_ops)
+#define KCF_PROV_MECH_OPS(pd) ((pd)->pd_ops_vector->co_mech_ops)
+#define KCF_PROV_NOSTORE_KEY_OPS(pd) \
+ ((pd)->pd_ops_vector->co_nostore_key_ops)
+
+/*
+ * Wrappers for crypto_control_ops(9S) entry points.
+ */
+
+#define KCF_PROV_STATUS(pd, status) ( \
+ (KCF_PROV_CONTROL_OPS(pd) && \
+ KCF_PROV_CONTROL_OPS(pd)->provider_status) ? \
+ KCF_PROV_CONTROL_OPS(pd)->provider_status( \
+ (pd)->pd_prov_handle, status) : \
+ CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_ctx_ops(9S) entry points.
+ */
+
+#define KCF_PROV_CREATE_CTX_TEMPLATE(pd, mech, key, template, size, req) ( \
+ (KCF_PROV_CTX_OPS(pd) && KCF_PROV_CTX_OPS(pd)->create_ctx_template) ? \
+ KCF_PROV_CTX_OPS(pd)->create_ctx_template( \
+ (pd)->pd_prov_handle, mech, key, template, size, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_FREE_CONTEXT(pd, ctx) ( \
+ (KCF_PROV_CTX_OPS(pd) && KCF_PROV_CTX_OPS(pd)->free_context) ? \
+ KCF_PROV_CTX_OPS(pd)->free_context(ctx) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_COPYIN_MECH(pd, umech, kmech, errorp, mode) ( \
+ (KCF_PROV_MECH_OPS(pd) && KCF_PROV_MECH_OPS(pd)->copyin_mechanism) ? \
+ KCF_PROV_MECH_OPS(pd)->copyin_mechanism( \
+ (pd)->pd_prov_handle, umech, kmech, errorp, mode) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_COPYOUT_MECH(pd, kmech, umech, errorp, mode) ( \
+ (KCF_PROV_MECH_OPS(pd) && KCF_PROV_MECH_OPS(pd)->copyout_mechanism) ? \
+ KCF_PROV_MECH_OPS(pd)->copyout_mechanism( \
+ (pd)->pd_prov_handle, kmech, umech, errorp, mode) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_FREE_MECH(pd, prov_mech) ( \
+ (KCF_PROV_MECH_OPS(pd) && KCF_PROV_MECH_OPS(pd)->free_mechanism) ? \
+ KCF_PROV_MECH_OPS(pd)->free_mechanism( \
+ (pd)->pd_prov_handle, prov_mech) : CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_digest_ops(9S) entry points.
+ */
+
+#define KCF_PROV_DIGEST_INIT(pd, ctx, mech, req) ( \
+ (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_init) ? \
+ KCF_PROV_DIGEST_OPS(pd)->digest_init(ctx, mech, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+/*
+ * The _ (underscore) in _digest is needed to avoid replacing the
+ * function digest().
+ */
+#define KCF_PROV_DIGEST(pd, ctx, data, _digest, req) ( \
+ (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest) ? \
+ KCF_PROV_DIGEST_OPS(pd)->digest(ctx, data, _digest, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DIGEST_UPDATE(pd, ctx, data, req) ( \
+ (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_update) ? \
+ KCF_PROV_DIGEST_OPS(pd)->digest_update(ctx, data, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DIGEST_KEY(pd, ctx, key, req) ( \
+ (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_key) ? \
+ KCF_PROV_DIGEST_OPS(pd)->digest_key(ctx, key, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DIGEST_FINAL(pd, ctx, digest, req) ( \
+ (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_final) ? \
+ KCF_PROV_DIGEST_OPS(pd)->digest_final(ctx, digest, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DIGEST_ATOMIC(pd, session, mech, data, digest, req) ( \
+ (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_atomic) ? \
+ KCF_PROV_DIGEST_OPS(pd)->digest_atomic( \
+ (pd)->pd_prov_handle, session, mech, data, digest, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_cipher_ops(9S) entry points.
+ */
+
+#define KCF_PROV_ENCRYPT_INIT(pd, ctx, mech, key, template, req) ( \
+ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt_init) ? \
+ KCF_PROV_CIPHER_OPS(pd)->encrypt_init(ctx, mech, key, template, \
+ req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_ENCRYPT(pd, ctx, plaintext, ciphertext, req) ( \
+ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt) ? \
+ KCF_PROV_CIPHER_OPS(pd)->encrypt(ctx, plaintext, ciphertext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_ENCRYPT_UPDATE(pd, ctx, plaintext, ciphertext, req) ( \
+ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt_update) ? \
+ KCF_PROV_CIPHER_OPS(pd)->encrypt_update(ctx, plaintext, \
+ ciphertext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_ENCRYPT_FINAL(pd, ctx, ciphertext, req) ( \
+ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt_final) ? \
+ KCF_PROV_CIPHER_OPS(pd)->encrypt_final(ctx, ciphertext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_ENCRYPT_ATOMIC(pd, session, mech, key, plaintext, ciphertext, \
+ template, req) ( \
+ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt_atomic) ? \
+ KCF_PROV_CIPHER_OPS(pd)->encrypt_atomic( \
+ (pd)->pd_prov_handle, session, mech, key, plaintext, ciphertext, \
+ template, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DECRYPT_INIT(pd, ctx, mech, key, template, req) ( \
+ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->decrypt_init) ? \
+ KCF_PROV_CIPHER_OPS(pd)->decrypt_init(ctx, mech, key, template, \
+ req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DECRYPT(pd, ctx, ciphertext, plaintext, req) ( \
+ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->decrypt) ? \
+ KCF_PROV_CIPHER_OPS(pd)->decrypt(ctx, ciphertext, plaintext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DECRYPT_UPDATE(pd, ctx, ciphertext, plaintext, req) ( \
+ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->decrypt_update) ? \
+ KCF_PROV_CIPHER_OPS(pd)->decrypt_update(ctx, ciphertext, \
+ plaintext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DECRYPT_FINAL(pd, ctx, plaintext, req) ( \
+ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->decrypt_final) ? \
+ KCF_PROV_CIPHER_OPS(pd)->decrypt_final(ctx, plaintext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DECRYPT_ATOMIC(pd, session, mech, key, ciphertext, plaintext, \
+ template, req) ( \
+ (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->decrypt_atomic) ? \
+ KCF_PROV_CIPHER_OPS(pd)->decrypt_atomic( \
+ (pd)->pd_prov_handle, session, mech, key, ciphertext, plaintext, \
+ template, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_mac_ops(9S) entry points.
+ */
+
+#define KCF_PROV_MAC_INIT(pd, ctx, mech, key, template, req) ( \
+ (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac_init) ? \
+ KCF_PROV_MAC_OPS(pd)->mac_init(ctx, mech, key, template, req) \
+ : CRYPTO_NOT_SUPPORTED)
+
+/*
+ * The _ (underscore) in _mac is needed to avoid replacing the
+ * function mac().
+ */
+#define KCF_PROV_MAC(pd, ctx, data, _mac, req) ( \
+ (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac) ? \
+ KCF_PROV_MAC_OPS(pd)->mac(ctx, data, _mac, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_MAC_UPDATE(pd, ctx, data, req) ( \
+ (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac_update) ? \
+ KCF_PROV_MAC_OPS(pd)->mac_update(ctx, data, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_MAC_FINAL(pd, ctx, mac, req) ( \
+ (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac_final) ? \
+ KCF_PROV_MAC_OPS(pd)->mac_final(ctx, mac, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_MAC_ATOMIC(pd, session, mech, key, data, mac, template, \
+ req) ( \
+ (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac_atomic) ? \
+ KCF_PROV_MAC_OPS(pd)->mac_atomic( \
+ (pd)->pd_prov_handle, session, mech, key, data, mac, template, \
+ req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_MAC_VERIFY_ATOMIC(pd, session, mech, key, data, mac, \
+ template, req) ( \
+ (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac_verify_atomic) ? \
+ KCF_PROV_MAC_OPS(pd)->mac_verify_atomic( \
+ (pd)->pd_prov_handle, session, mech, key, data, mac, template, \
+ req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_sign_ops(9S) entry points.
+ */
+
+#define KCF_PROV_SIGN_INIT(pd, ctx, mech, key, template, req) ( \
+ (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_init) ? \
+ KCF_PROV_SIGN_OPS(pd)->sign_init( \
+ ctx, mech, key, template, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SIGN(pd, ctx, data, sig, req) ( \
+ (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign) ? \
+ KCF_PROV_SIGN_OPS(pd)->sign(ctx, data, sig, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SIGN_UPDATE(pd, ctx, data, req) ( \
+ (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_update) ? \
+ KCF_PROV_SIGN_OPS(pd)->sign_update(ctx, data, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SIGN_FINAL(pd, ctx, sig, req) ( \
+ (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_final) ? \
+ KCF_PROV_SIGN_OPS(pd)->sign_final(ctx, sig, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SIGN_ATOMIC(pd, session, mech, key, data, template, \
+ sig, req) ( \
+ (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_atomic) ? \
+ KCF_PROV_SIGN_OPS(pd)->sign_atomic( \
+ (pd)->pd_prov_handle, session, mech, key, data, sig, template, \
+ req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SIGN_RECOVER_INIT(pd, ctx, mech, key, template, \
+ req) ( \
+ (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_recover_init) ? \
+ KCF_PROV_SIGN_OPS(pd)->sign_recover_init(ctx, mech, key, template, \
+ req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SIGN_RECOVER(pd, ctx, data, sig, req) ( \
+ (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_recover) ? \
+ KCF_PROV_SIGN_OPS(pd)->sign_recover(ctx, data, sig, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SIGN_RECOVER_ATOMIC(pd, session, mech, key, data, template, \
+ sig, req) ( \
+ (KCF_PROV_SIGN_OPS(pd) && \
+ KCF_PROV_SIGN_OPS(pd)->sign_recover_atomic) ? \
+ KCF_PROV_SIGN_OPS(pd)->sign_recover_atomic( \
+ (pd)->pd_prov_handle, session, mech, key, data, sig, template, \
+ req) : CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_verify_ops(9S) entry points.
+ */
+
+#define KCF_PROV_VERIFY_INIT(pd, ctx, mech, key, template, req) ( \
+ (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->verify_init) ? \
+ KCF_PROV_VERIFY_OPS(pd)->verify_init(ctx, mech, key, template, \
+ req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_VERIFY(pd, ctx, data, sig, req) ( \
+ (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->do_verify) ? \
+ KCF_PROV_VERIFY_OPS(pd)->do_verify(ctx, data, sig, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_VERIFY_UPDATE(pd, ctx, data, req) ( \
+ (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->verify_update) ? \
+ KCF_PROV_VERIFY_OPS(pd)->verify_update(ctx, data, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_VERIFY_FINAL(pd, ctx, sig, req) ( \
+ (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->verify_final) ? \
+ KCF_PROV_VERIFY_OPS(pd)->verify_final(ctx, sig, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_VERIFY_ATOMIC(pd, session, mech, key, data, template, sig, \
+ req) ( \
+ (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->verify_atomic) ? \
+ KCF_PROV_VERIFY_OPS(pd)->verify_atomic( \
+ (pd)->pd_prov_handle, session, mech, key, data, sig, template, \
+ req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_VERIFY_RECOVER_INIT(pd, ctx, mech, key, template, \
+ req) ( \
+ (KCF_PROV_VERIFY_OPS(pd) && \
+ KCF_PROV_VERIFY_OPS(pd)->verify_recover_init) ? \
+ KCF_PROV_VERIFY_OPS(pd)->verify_recover_init(ctx, mech, key, \
+ template, req) : CRYPTO_NOT_SUPPORTED)
+
+/* verify_recover() CSPI routine has different argument order than verify() */
+#define KCF_PROV_VERIFY_RECOVER(pd, ctx, sig, data, req) ( \
+ (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->verify_recover) ? \
+ KCF_PROV_VERIFY_OPS(pd)->verify_recover(ctx, sig, data, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+/*
+ * verify_recover_atomic() CSPI routine has different argument order
+ * than verify_atomic().
+ */
+#define KCF_PROV_VERIFY_RECOVER_ATOMIC(pd, session, mech, key, sig, \
+ template, data, req) ( \
+ (KCF_PROV_VERIFY_OPS(pd) && \
+ KCF_PROV_VERIFY_OPS(pd)->verify_recover_atomic) ? \
+ KCF_PROV_VERIFY_OPS(pd)->verify_recover_atomic( \
+ (pd)->pd_prov_handle, session, mech, key, sig, data, template, \
+ req) : CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_dual_ops(9S) entry points.
+ */
+
+#define KCF_PROV_DIGEST_ENCRYPT_UPDATE(digest_ctx, encrypt_ctx, plaintext, \
+ ciphertext, req) ( \
+ (KCF_PROV_DUAL_OPS(pd) && \
+ KCF_PROV_DUAL_OPS(pd)->digest_encrypt_update) ? \
+ KCF_PROV_DUAL_OPS(pd)->digest_encrypt_update( \
+ digest_ctx, encrypt_ctx, plaintext, ciphertext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DECRYPT_DIGEST_UPDATE(decrypt_ctx, digest_ctx, ciphertext, \
+ plaintext, req) ( \
+ (KCF_PROV_DUAL_OPS(pd) && \
+ KCF_PROV_DUAL_OPS(pd)->decrypt_digest_update) ? \
+ KCF_PROV_DUAL_OPS(pd)->decrypt_digest_update( \
+ decrypt_ctx, digest_ctx, ciphertext, plaintext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SIGN_ENCRYPT_UPDATE(sign_ctx, encrypt_ctx, plaintext, \
+ ciphertext, req) ( \
+ (KCF_PROV_DUAL_OPS(pd) && \
+ KCF_PROV_DUAL_OPS(pd)->sign_encrypt_update) ? \
+ KCF_PROV_DUAL_OPS(pd)->sign_encrypt_update( \
+ sign_ctx, encrypt_ctx, plaintext, ciphertext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_DECRYPT_VERIFY_UPDATE(decrypt_ctx, verify_ctx, ciphertext, \
+ plaintext, req) ( \
+ (KCF_PROV_DUAL_OPS(pd) && \
+ KCF_PROV_DUAL_OPS(pd)->decrypt_verify_update) ? \
+ KCF_PROV_DUAL_OPS(pd)->decrypt_verify_update( \
+ decrypt_ctx, verify_ctx, ciphertext, plaintext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_dual_cipher_mac_ops(9S) entry points.
+ */
+
+#define KCF_PROV_ENCRYPT_MAC_INIT(pd, ctx, encr_mech, encr_key, mac_mech, \
+ mac_key, encr_ctx_template, mac_ctx_template, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_init) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_init( \
+ ctx, encr_mech, encr_key, mac_mech, mac_key, encr_ctx_template, \
+ mac_ctx_template, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_ENCRYPT_MAC(pd, ctx, plaintext, ciphertext, mac, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac( \
+ ctx, plaintext, ciphertext, mac, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_ENCRYPT_MAC_UPDATE(pd, ctx, plaintext, ciphertext, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_update) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_update( \
+ ctx, plaintext, ciphertext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_ENCRYPT_MAC_FINAL(pd, ctx, ciphertext, mac, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_final) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_final( \
+ ctx, ciphertext, mac, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_ENCRYPT_MAC_ATOMIC(pd, session, encr_mech, encr_key, \
+ mac_mech, mac_key, plaintext, ciphertext, mac, \
+ encr_ctx_template, mac_ctx_template, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_atomic) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_atomic( \
+ (pd)->pd_prov_handle, session, encr_mech, encr_key, \
+ mac_mech, mac_key, plaintext, ciphertext, mac, \
+ encr_ctx_template, mac_ctx_template, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_MAC_DECRYPT_INIT(pd, ctx, mac_mech, mac_key, decr_mech, \
+ decr_key, mac_ctx_template, decr_ctx_template, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_init) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_init( \
+ ctx, mac_mech, mac_key, decr_mech, decr_key, mac_ctx_template, \
+ decr_ctx_template, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_MAC_DECRYPT(pd, ctx, ciphertext, mac, plaintext, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt( \
+ ctx, ciphertext, mac, plaintext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_MAC_DECRYPT_UPDATE(pd, ctx, ciphertext, plaintext, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_update) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_update( \
+ ctx, ciphertext, plaintext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_MAC_DECRYPT_FINAL(pd, ctx, mac, plaintext, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_final) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_final( \
+ ctx, mac, plaintext, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_MAC_DECRYPT_ATOMIC(pd, session, mac_mech, mac_key, \
+ decr_mech, decr_key, ciphertext, mac, plaintext, \
+ mac_ctx_template, decr_ctx_template, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_atomic) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_atomic( \
+ (pd)->pd_prov_handle, session, mac_mech, mac_key, \
+ decr_mech, decr_key, ciphertext, mac, plaintext, \
+ mac_ctx_template, decr_ctx_template, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_MAC_VERIFY_DECRYPT_ATOMIC(pd, session, mac_mech, mac_key, \
+ decr_mech, decr_key, ciphertext, mac, plaintext, \
+ mac_ctx_template, decr_ctx_template, req) ( \
+ (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_verify_decrypt_atomic \
+ != NULL) ? \
+ KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_verify_decrypt_atomic( \
+ (pd)->pd_prov_handle, session, mac_mech, mac_key, \
+ decr_mech, decr_key, ciphertext, mac, plaintext, \
+ mac_ctx_template, decr_ctx_template, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_random_number_ops(9S) entry points.
+ */
+
+#define KCF_PROV_SEED_RANDOM(pd, session, buf, len, est, flags, req) ( \
+ (KCF_PROV_RANDOM_OPS(pd) && KCF_PROV_RANDOM_OPS(pd)->seed_random) ? \
+ KCF_PROV_RANDOM_OPS(pd)->seed_random((pd)->pd_prov_handle, \
+ session, buf, len, est, flags, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_GENERATE_RANDOM(pd, session, buf, len, req) ( \
+ (KCF_PROV_RANDOM_OPS(pd) && \
+ KCF_PROV_RANDOM_OPS(pd)->generate_random) ? \
+ KCF_PROV_RANDOM_OPS(pd)->generate_random((pd)->pd_prov_handle, \
+ session, buf, len, req) : CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_session_ops(9S) entry points.
+ *
+ * ops_pd is the provider descriptor that supplies the ops_vector.
+ * pd is the descriptor that supplies the provider handle.
+ * Only session open/close needs two handles.
+ */
+
+#define KCF_PROV_SESSION_OPEN(ops_pd, session, req, pd) ( \
+ (KCF_PROV_SESSION_OPS(ops_pd) && \
+ KCF_PROV_SESSION_OPS(ops_pd)->session_open) ? \
+ KCF_PROV_SESSION_OPS(ops_pd)->session_open((pd)->pd_prov_handle, \
+ session, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SESSION_CLOSE(ops_pd, session, req, pd) ( \
+ (KCF_PROV_SESSION_OPS(ops_pd) && \
+ KCF_PROV_SESSION_OPS(ops_pd)->session_close) ? \
+ KCF_PROV_SESSION_OPS(ops_pd)->session_close((pd)->pd_prov_handle, \
+ session, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SESSION_LOGIN(pd, session, user_type, pin, len, req) ( \
+ (KCF_PROV_SESSION_OPS(pd) && \
+ KCF_PROV_SESSION_OPS(pd)->session_login) ? \
+ KCF_PROV_SESSION_OPS(pd)->session_login((pd)->pd_prov_handle, \
+ session, user_type, pin, len, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SESSION_LOGOUT(pd, session, req) ( \
+ (KCF_PROV_SESSION_OPS(pd) && \
+ KCF_PROV_SESSION_OPS(pd)->session_logout) ? \
+ KCF_PROV_SESSION_OPS(pd)->session_logout((pd)->pd_prov_handle, \
+ session, req) : CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_object_ops(9S) entry points.
+ */
+
+#define KCF_PROV_OBJECT_CREATE(pd, session, template, count, object, req) ( \
+ (KCF_PROV_OBJECT_OPS(pd) && KCF_PROV_OBJECT_OPS(pd)->object_create) ? \
+ KCF_PROV_OBJECT_OPS(pd)->object_create((pd)->pd_prov_handle, \
+ session, template, count, object, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_OBJECT_COPY(pd, session, object, template, count, \
+ new_object, req) ( \
+ (KCF_PROV_OBJECT_OPS(pd) && KCF_PROV_OBJECT_OPS(pd)->object_copy) ? \
+ KCF_PROV_OBJECT_OPS(pd)->object_copy((pd)->pd_prov_handle, \
+ session, object, template, count, new_object, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_OBJECT_DESTROY(pd, session, object, req) ( \
+ (KCF_PROV_OBJECT_OPS(pd) && KCF_PROV_OBJECT_OPS(pd)->object_destroy) ? \
+ KCF_PROV_OBJECT_OPS(pd)->object_destroy((pd)->pd_prov_handle, \
+ session, object, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_OBJECT_GET_SIZE(pd, session, object, size, req) ( \
+ (KCF_PROV_OBJECT_OPS(pd) && \
+ KCF_PROV_OBJECT_OPS(pd)->object_get_size) ? \
+ KCF_PROV_OBJECT_OPS(pd)->object_get_size((pd)->pd_prov_handle, \
+ session, object, size, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_OBJECT_GET_ATTRIBUTE_VALUE(pd, session, object, template, \
+ count, req) ( \
+ (KCF_PROV_OBJECT_OPS(pd) && \
+ KCF_PROV_OBJECT_OPS(pd)->object_get_attribute_value) ? \
+ KCF_PROV_OBJECT_OPS(pd)->object_get_attribute_value( \
+ (pd)->pd_prov_handle, session, object, template, count, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_OBJECT_SET_ATTRIBUTE_VALUE(pd, session, object, template, \
+ count, req) ( \
+ (KCF_PROV_OBJECT_OPS(pd) && \
+ KCF_PROV_OBJECT_OPS(pd)->object_set_attribute_value) ? \
+ KCF_PROV_OBJECT_OPS(pd)->object_set_attribute_value( \
+ (pd)->pd_prov_handle, session, object, template, count, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_OBJECT_FIND_INIT(pd, session, template, count, ppriv, \
+ req) ( \
+ (KCF_PROV_OBJECT_OPS(pd) && \
+ KCF_PROV_OBJECT_OPS(pd)->object_find_init) ? \
+ KCF_PROV_OBJECT_OPS(pd)->object_find_init((pd)->pd_prov_handle, \
+ session, template, count, ppriv, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_OBJECT_FIND(pd, ppriv, objects, max_objects, object_count, \
+ req) ( \
+ (KCF_PROV_OBJECT_OPS(pd) && KCF_PROV_OBJECT_OPS(pd)->object_find) ? \
+ KCF_PROV_OBJECT_OPS(pd)->object_find( \
+ (pd)->pd_prov_handle, ppriv, objects, max_objects, object_count, \
+ req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_OBJECT_FIND_FINAL(pd, ppriv, req) ( \
+ (KCF_PROV_OBJECT_OPS(pd) && \
+ KCF_PROV_OBJECT_OPS(pd)->object_find_final) ? \
+ KCF_PROV_OBJECT_OPS(pd)->object_find_final( \
+ (pd)->pd_prov_handle, ppriv, req) : CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_key_ops(9S) entry points.
+ */
+
+#define KCF_PROV_KEY_GENERATE(pd, session, mech, template, count, object, \
+ req) ( \
+ (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_generate) ? \
+ KCF_PROV_KEY_OPS(pd)->key_generate((pd)->pd_prov_handle, \
+ session, mech, template, count, object, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_KEY_GENERATE_PAIR(pd, session, mech, pub_template, \
+ pub_count, priv_template, priv_count, pub_key, priv_key, req) ( \
+ (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_generate_pair) ? \
+ KCF_PROV_KEY_OPS(pd)->key_generate_pair((pd)->pd_prov_handle, \
+ session, mech, pub_template, pub_count, priv_template, \
+ priv_count, pub_key, priv_key, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_KEY_WRAP(pd, session, mech, wrapping_key, key, wrapped_key, \
+ wrapped_key_len, req) ( \
+ (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_wrap) ? \
+ KCF_PROV_KEY_OPS(pd)->key_wrap((pd)->pd_prov_handle, \
+ session, mech, wrapping_key, key, wrapped_key, wrapped_key_len, \
+ req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_KEY_UNWRAP(pd, session, mech, unwrapping_key, wrapped_key, \
+ wrapped_key_len, template, count, key, req) ( \
+ (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_unwrap) ? \
+ KCF_PROV_KEY_OPS(pd)->key_unwrap((pd)->pd_prov_handle, \
+ session, mech, unwrapping_key, wrapped_key, wrapped_key_len, \
+ template, count, key, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_KEY_DERIVE(pd, session, mech, base_key, template, count, \
+ key, req) ( \
+ (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_derive) ? \
+ KCF_PROV_KEY_OPS(pd)->key_derive((pd)->pd_prov_handle, \
+ session, mech, base_key, template, count, key, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_KEY_CHECK(pd, mech, key) ( \
+ (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_check) ? \
+ KCF_PROV_KEY_OPS(pd)->key_check((pd)->pd_prov_handle, mech, key) : \
+ CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_provider_management_ops(9S) entry points.
+ *
+ * ops_pd is the provider descriptor that supplies the ops_vector.
+ * pd is the descriptor that supplies the provider handle.
+ * Only ext_info needs two handles.
+ */
+
+#define KCF_PROV_EXT_INFO(ops_pd, provext_info, req, pd) ( \
+ (KCF_PROV_PROVIDER_OPS(ops_pd) && \
+ KCF_PROV_PROVIDER_OPS(ops_pd)->ext_info) ? \
+ KCF_PROV_PROVIDER_OPS(ops_pd)->ext_info((pd)->pd_prov_handle, \
+ provext_info, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_INIT_TOKEN(pd, pin, pin_len, label, req) ( \
+ (KCF_PROV_PROVIDER_OPS(pd) && KCF_PROV_PROVIDER_OPS(pd)->init_token) ? \
+ KCF_PROV_PROVIDER_OPS(pd)->init_token((pd)->pd_prov_handle, \
+ pin, pin_len, label, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_INIT_PIN(pd, session, pin, pin_len, req) ( \
+ (KCF_PROV_PROVIDER_OPS(pd) && KCF_PROV_PROVIDER_OPS(pd)->init_pin) ? \
+ KCF_PROV_PROVIDER_OPS(pd)->init_pin((pd)->pd_prov_handle, \
+ session, pin, pin_len, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_SET_PIN(pd, session, old_pin, old_len, new_pin, new_len, \
+ req) ( \
+ (KCF_PROV_PROVIDER_OPS(pd) && KCF_PROV_PROVIDER_OPS(pd)->set_pin) ? \
+ KCF_PROV_PROVIDER_OPS(pd)->set_pin((pd)->pd_prov_handle, \
+ session, old_pin, old_len, new_pin, new_len, req) : \
+ CRYPTO_NOT_SUPPORTED)
+
+/*
+ * Wrappers for crypto_nostore_key_ops(9S) entry points.
+ */
+
+#define KCF_PROV_NOSTORE_KEY_GENERATE(pd, session, mech, template, count, \
+ out_template, out_count, req) ( \
+ (KCF_PROV_NOSTORE_KEY_OPS(pd) && \
+ KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_generate) ? \
+ KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_generate( \
+ (pd)->pd_prov_handle, session, mech, template, count, \
+ out_template, out_count, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_NOSTORE_KEY_GENERATE_PAIR(pd, session, mech, pub_template, \
+ pub_count, priv_template, priv_count, out_pub_template, \
+ out_pub_count, out_priv_template, out_priv_count, req) ( \
+ (KCF_PROV_NOSTORE_KEY_OPS(pd) && \
+ KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_generate_pair) ? \
+ KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_generate_pair( \
+ (pd)->pd_prov_handle, session, mech, pub_template, pub_count, \
+ priv_template, priv_count, out_pub_template, out_pub_count, \
+ out_priv_template, out_priv_count, req) : CRYPTO_NOT_SUPPORTED)
+
+#define KCF_PROV_NOSTORE_KEY_DERIVE(pd, session, mech, base_key, template, \
+ count, out_template, out_count, req) ( \
+ (KCF_PROV_NOSTORE_KEY_OPS(pd) && \
+ KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_derive) ? \
+ KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_derive( \
+ (pd)->pd_prov_handle, session, mech, base_key, template, count, \
+ out_template, out_count, req) : CRYPTO_NOT_SUPPORTED)
+
+/*
+ * The following routines are exported by the kcf module (/kernel/misc/kcf)
+ * to the crypto and cryptoadmin modules.
+ */
+
+/* Digest/mac/cipher entry points that take a provider descriptor and session */
+extern int crypto_digest_single(crypto_context_t, crypto_data_t *,
+ crypto_data_t *, crypto_call_req_t *);
+
+extern int crypto_mac_single(crypto_context_t, crypto_data_t *,
+ crypto_data_t *, crypto_call_req_t *);
+
+extern int crypto_encrypt_single(crypto_context_t, crypto_data_t *,
+ crypto_data_t *, crypto_call_req_t *);
+
+extern int crypto_decrypt_single(crypto_context_t, crypto_data_t *,
+ crypto_data_t *, crypto_call_req_t *);
+
+
+/* Other private digest/mac/cipher entry points not exported through k-API */
+extern int crypto_digest_key_prov(crypto_context_t, crypto_key_t *,
+ crypto_call_req_t *);
+
+/* Private sign entry points exported by KCF */
+extern int crypto_sign_single(crypto_context_t, crypto_data_t *,
+ crypto_data_t *, crypto_call_req_t *);
+
+extern int crypto_sign_recover_single(crypto_context_t, crypto_data_t *,
+ crypto_data_t *, crypto_call_req_t *);
+
+/* Private verify entry points exported by KCF */
+extern int crypto_verify_single(crypto_context_t, crypto_data_t *,
+ crypto_data_t *, crypto_call_req_t *);
+
+extern int crypto_verify_recover_single(crypto_context_t, crypto_data_t *,
+ crypto_data_t *, crypto_call_req_t *);
+
+/* Private dual operations entry points exported by KCF */
+extern int crypto_digest_encrypt_update(crypto_context_t, crypto_context_t,
+ crypto_data_t *, crypto_data_t *, crypto_call_req_t *);
+extern int crypto_decrypt_digest_update(crypto_context_t, crypto_context_t,
+ crypto_data_t *, crypto_data_t *, crypto_call_req_t *);
+extern int crypto_sign_encrypt_update(crypto_context_t, crypto_context_t,
+ crypto_data_t *, crypto_data_t *, crypto_call_req_t *);
+extern int crypto_decrypt_verify_update(crypto_context_t, crypto_context_t,
+ crypto_data_t *, crypto_data_t *, crypto_call_req_t *);
+
+/* Random Number Generation */
+int crypto_seed_random(crypto_provider_handle_t provider, uchar_t *buf,
+ size_t len, crypto_call_req_t *req);
+int crypto_generate_random(crypto_provider_handle_t provider, uchar_t *buf,
+ size_t len, crypto_call_req_t *req);
+
+/* Provider Management */
+int crypto_get_provider_info(crypto_provider_id_t id,
+ crypto_provider_info_t **info, crypto_call_req_t *req);
+int crypto_get_provider_mechanisms(crypto_minor_t *, crypto_provider_id_t id,
+ uint_t *count, crypto_mech_name_t **list);
+int crypto_init_token(crypto_provider_handle_t provider, char *pin,
+ size_t pin_len, char *label, crypto_call_req_t *);
+int crypto_init_pin(crypto_provider_handle_t provider, char *pin,
+ size_t pin_len, crypto_call_req_t *req);
+int crypto_set_pin(crypto_provider_handle_t provider, char *old_pin,
+ size_t old_len, char *new_pin, size_t new_len, crypto_call_req_t *req);
+void crypto_free_provider_list(crypto_provider_entry_t *list, uint_t count);
+void crypto_free_provider_info(crypto_provider_info_t *info);
+
+/* Administrative */
+int crypto_get_dev_list(uint_t *count, crypto_dev_list_entry_t **list);
+int crypto_get_soft_list(uint_t *count, char **list, size_t *len);
+int crypto_get_dev_info(char *name, uint_t instance, uint_t *count,
+ crypto_mech_name_t **list);
+int crypto_get_soft_info(caddr_t name, uint_t *count,
+ crypto_mech_name_t **list);
+int crypto_load_dev_disabled(char *name, uint_t instance, uint_t count,
+ crypto_mech_name_t *list);
+int crypto_load_soft_disabled(caddr_t name, uint_t count,
+ crypto_mech_name_t *list);
+int crypto_unload_soft_module(caddr_t path);
+int crypto_load_soft_config(caddr_t name, uint_t count,
+ crypto_mech_name_t *list);
+int crypto_load_door(uint_t did);
+void crypto_free_mech_list(crypto_mech_name_t *list, uint_t count);
+void crypto_free_dev_list(crypto_dev_list_entry_t *list, uint_t count);
+
+/* Miscellaneous */
+int crypto_get_mechanism_number(caddr_t name, crypto_mech_type_t *number);
+int crypto_get_function_list(crypto_provider_id_t id,
+ crypto_function_list_t **list, int kmflag);
+void crypto_free_function_list(crypto_function_list_t *list);
+int crypto_build_permitted_mech_names(kcf_provider_desc_t *,
+ crypto_mech_name_t **, uint_t *, int);
+extern void kcf_destroy_mech_tabs(void);
+extern void kcf_init_mech_tabs(void);
+extern int kcf_add_mech_provider(short, kcf_provider_desc_t *,
+ kcf_prov_mech_desc_t **);
+extern void kcf_remove_mech_provider(char *, kcf_provider_desc_t *);
+extern int kcf_get_mech_entry(crypto_mech_type_t, kcf_mech_entry_t **);
+extern kcf_provider_desc_t *kcf_alloc_provider_desc(crypto_provider_info_t *);
+extern void kcf_provider_zero_refcnt(kcf_provider_desc_t *);
+extern void kcf_free_provider_desc(kcf_provider_desc_t *);
+extern void kcf_soft_config_init(void);
+extern int get_sw_provider_for_mech(crypto_mech_name_t, char **);
+extern crypto_mech_type_t crypto_mech2id_common(char *, boolean_t);
+extern void undo_register_provider(kcf_provider_desc_t *, boolean_t);
+extern void redo_register_provider(kcf_provider_desc_t *);
+extern void kcf_rnd_init(void);
+extern boolean_t kcf_rngprov_check(void);
+extern int kcf_rnd_get_pseudo_bytes(uint8_t *, size_t);
+extern int kcf_rnd_get_bytes(uint8_t *, size_t, boolean_t, boolean_t);
+extern int random_add_pseudo_entropy(uint8_t *, size_t, uint_t);
+extern void kcf_rnd_schedule_timeout(boolean_t);
+extern int crypto_uio_data(crypto_data_t *, uchar_t *, int, cmd_type_t,
+ void *, void (*update)(void));
+extern int crypto_mblk_data(crypto_data_t *, uchar_t *, int, cmd_type_t,
+ void *, void (*update)(void));
+extern int crypto_put_output_data(uchar_t *, crypto_data_t *, int);
+extern int crypto_get_input_data(crypto_data_t *, uchar_t **, uchar_t *);
+extern int crypto_copy_key_to_ctx(crypto_key_t *, crypto_key_t **, size_t *,
+ int kmflag);
+extern int crypto_digest_data(crypto_data_t *, void *, uchar_t *,
+ void (*update)(void), void (*final)(void), uchar_t);
+extern int crypto_update_iov(void *, crypto_data_t *, crypto_data_t *,
+ int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
+ void (*copy_block)(uint8_t *, uint64_t *));
+extern int crypto_update_uio(void *, crypto_data_t *, crypto_data_t *,
+ int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
+ void (*copy_block)(uint8_t *, uint64_t *));
+extern int crypto_update_mp(void *, crypto_data_t *, crypto_data_t *,
+ int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
+ void (*copy_block)(uint8_t *, uint64_t *));
+extern int crypto_get_key_attr(crypto_key_t *, crypto_attr_type_t, uchar_t **,
+ ssize_t *);
+
+/* Access to the provider's table */
+extern void kcf_prov_tab_destroy(void);
+extern void kcf_prov_tab_init(void);
+extern int kcf_prov_tab_add_provider(kcf_provider_desc_t *);
+extern int kcf_prov_tab_rem_provider(crypto_provider_id_t);
+extern kcf_provider_desc_t *kcf_prov_tab_lookup_by_name(char *);
+extern kcf_provider_desc_t *kcf_prov_tab_lookup_by_dev(char *, uint_t);
+extern int kcf_get_hw_prov_tab(uint_t *, kcf_provider_desc_t ***, int,
+ char *, uint_t, boolean_t);
+extern int kcf_get_slot_list(uint_t *, kcf_provider_desc_t ***, boolean_t);
+extern void kcf_free_provider_tab(uint_t, kcf_provider_desc_t **);
+extern kcf_provider_desc_t *kcf_prov_tab_lookup(crypto_provider_id_t);
+extern int kcf_get_sw_prov(crypto_mech_type_t, kcf_provider_desc_t **,
+ kcf_mech_entry_t **, boolean_t);
+
+/* Access to the policy table */
+extern boolean_t is_mech_disabled(kcf_provider_desc_t *, crypto_mech_name_t);
+extern boolean_t is_mech_disabled_byname(crypto_provider_type_t, char *,
+ uint_t, crypto_mech_name_t);
+extern void kcf_policy_tab_init(void);
+extern void kcf_policy_free_desc(kcf_policy_desc_t *);
+extern void kcf_policy_remove_by_name(char *, uint_t *, crypto_mech_name_t **);
+extern void kcf_policy_remove_by_dev(char *, uint_t, uint_t *,
+ crypto_mech_name_t **);
+extern kcf_policy_desc_t *kcf_policy_lookup_by_name(char *);
+extern kcf_policy_desc_t *kcf_policy_lookup_by_dev(char *, uint_t);
+extern int kcf_policy_load_soft_disabled(char *, uint_t, crypto_mech_name_t *,
+ uint_t *, crypto_mech_name_t **);
+extern int kcf_policy_load_dev_disabled(char *, uint_t, uint_t,
+ crypto_mech_name_t *, uint_t *, crypto_mech_name_t **);
+extern boolean_t in_soft_config_list(char *);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_CRYPTO_IMPL_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_CRYPTO_IOCTL_H
+#define _SYS_CRYPTO_IOCTL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/spi.h>
+#include <sys/crypto/common.h>
+
+#define CRYPTO_MAX_ATTRIBUTE_COUNT 128
+
+#define CRYPTO_IOFLAGS_RW_SESSION 0x00000001
+
+#define CRYPTO(x) (('y' << 8) | (x))
+
+#define MAX_NUM_THRESHOLD 7
+
+/* the PKCS11 Mechanisms */
+#define CKM_RC4 0x00000111
+#define CKM_DES3_ECB 0x00000132
+#define CKM_DES3_CBC 0x00000133
+#define CKM_MD5 0x00000210
+#define CKM_SHA_1 0x00000220
+#define CKM_AES_ECB 0x00001081
+#define CKM_AES_CBC 0x00001082
+
+/*
+ * General Purpose Ioctls
+ */
+
+typedef struct fl_mechs_threshold {
+ int mech_type;
+ uint32_t mech_threshold;
+} fl_mechs_threshold_t;
+
+typedef struct crypto_function_list {
+ boolean_t fl_digest_init;
+ boolean_t fl_digest;
+ boolean_t fl_digest_update;
+ boolean_t fl_digest_key;
+ boolean_t fl_digest_final;
+
+ boolean_t fl_encrypt_init;
+ boolean_t fl_encrypt;
+ boolean_t fl_encrypt_update;
+ boolean_t fl_encrypt_final;
+
+ boolean_t fl_decrypt_init;
+ boolean_t fl_decrypt;
+ boolean_t fl_decrypt_update;
+ boolean_t fl_decrypt_final;
+
+ boolean_t fl_mac_init;
+ boolean_t fl_mac;
+ boolean_t fl_mac_update;
+ boolean_t fl_mac_final;
+
+ boolean_t fl_sign_init;
+ boolean_t fl_sign;
+ boolean_t fl_sign_update;
+ boolean_t fl_sign_final;
+ boolean_t fl_sign_recover_init;
+ boolean_t fl_sign_recover;
+
+ boolean_t fl_verify_init;
+ boolean_t fl_verify;
+ boolean_t fl_verify_update;
+ boolean_t fl_verify_final;
+ boolean_t fl_verify_recover_init;
+ boolean_t fl_verify_recover;
+
+ boolean_t fl_digest_encrypt_update;
+ boolean_t fl_decrypt_digest_update;
+ boolean_t fl_sign_encrypt_update;
+ boolean_t fl_decrypt_verify_update;
+
+ boolean_t fl_seed_random;
+ boolean_t fl_generate_random;
+
+ boolean_t fl_session_open;
+ boolean_t fl_session_close;
+ boolean_t fl_session_login;
+ boolean_t fl_session_logout;
+
+ boolean_t fl_object_create;
+ boolean_t fl_object_copy;
+ boolean_t fl_object_destroy;
+ boolean_t fl_object_get_size;
+ boolean_t fl_object_get_attribute_value;
+ boolean_t fl_object_set_attribute_value;
+ boolean_t fl_object_find_init;
+ boolean_t fl_object_find;
+ boolean_t fl_object_find_final;
+
+ boolean_t fl_key_generate;
+ boolean_t fl_key_generate_pair;
+ boolean_t fl_key_wrap;
+ boolean_t fl_key_unwrap;
+ boolean_t fl_key_derive;
+
+ boolean_t fl_init_token;
+ boolean_t fl_init_pin;
+ boolean_t fl_set_pin;
+
+ boolean_t prov_is_limited;
+ uint32_t prov_hash_threshold;
+ uint32_t prov_hash_limit;
+
+ int total_threshold_count;
+ fl_mechs_threshold_t fl_threshold[MAX_NUM_THRESHOLD];
+} crypto_function_list_t;
+
+typedef struct crypto_get_function_list {
+ uint_t fl_return_value;
+ crypto_provider_id_t fl_provider_id;
+ crypto_function_list_t fl_list;
+} crypto_get_function_list_t;
+
+typedef struct crypto_get_mechanism_number {
+ uint_t pn_return_value;
+ caddr_t pn_mechanism_string;
+ size_t pn_mechanism_len;
+ crypto_mech_type_t pn_internal_number;
+} crypto_get_mechanism_number_t;
+
+#ifdef _KERNEL
+#ifdef _SYSCALL32
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack(4)
+#endif
+
+typedef struct crypto_get_mechanism_number32 {
+ uint32_t pn_return_value;
+ caddr32_t pn_mechanism_string;
+ size32_t pn_mechanism_len;
+ crypto_mech_type_t pn_internal_number;
+} crypto_get_mechanism_number32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack()
+#endif
+
+#endif /* _SYSCALL32 */
+#endif /* _KERNEL */
+
+#define CRYPTO_GET_FUNCTION_LIST CRYPTO(20)
+#define CRYPTO_GET_MECHANISM_NUMBER CRYPTO(21)
+
+/*
+ * Session Ioctls
+ */
+
+typedef uint32_t crypto_flags_t;
+
+typedef struct crypto_open_session {
+ uint_t os_return_value;
+ crypto_session_id_t os_session;
+ crypto_flags_t os_flags;
+ crypto_provider_id_t os_provider_id;
+} crypto_open_session_t;
+
+typedef struct crypto_close_session {
+ uint_t cs_return_value;
+ crypto_session_id_t cs_session;
+} crypto_close_session_t;
+
+typedef struct crypto_close_all_sessions {
+ uint_t as_return_value;
+ crypto_provider_id_t as_provider_id;
+} crypto_close_all_sessions_t;
+
+#define CRYPTO_OPEN_SESSION CRYPTO(30)
+#define CRYPTO_CLOSE_SESSION CRYPTO(31)
+#define CRYPTO_CLOSE_ALL_SESSIONS CRYPTO(32)
+
+/*
+ * Login Ioctls
+ */
+typedef struct crypto_login {
+ uint_t co_return_value;
+ crypto_session_id_t co_session;
+ uint_t co_user_type;
+ uint_t co_pin_len;
+ caddr_t co_pin;
+} crypto_login_t;
+
+typedef struct crypto_logout {
+ uint_t cl_return_value;
+ crypto_session_id_t cl_session;
+} crypto_logout_t;
+
+#ifdef _KERNEL
+#ifdef _SYSCALL32
+
+typedef struct crypto_login32 {
+ uint32_t co_return_value;
+ crypto_session_id_t co_session;
+ uint32_t co_user_type;
+ uint32_t co_pin_len;
+ caddr32_t co_pin;
+} crypto_login32_t;
+
+typedef struct crypto_logout32 {
+ uint32_t cl_return_value;
+ crypto_session_id_t cl_session;
+} crypto_logout32_t;
+
+#endif /* _SYSCALL32 */
+#endif /* _KERNEL */
+
+#define CRYPTO_LOGIN CRYPTO(40)
+#define CRYPTO_LOGOUT CRYPTO(41)
+
+/* flag for encrypt and decrypt operations */
+#define CRYPTO_INPLACE_OPERATION 0x00000001
+
+/*
+ * Cryptographic Ioctls
+ */
+typedef struct crypto_encrypt {
+ uint_t ce_return_value;
+ crypto_session_id_t ce_session;
+ size_t ce_datalen;
+ caddr_t ce_databuf;
+ size_t ce_encrlen;
+ caddr_t ce_encrbuf;
+ uint_t ce_flags;
+} crypto_encrypt_t;
+
+typedef struct crypto_encrypt_init {
+ uint_t ei_return_value;
+ crypto_session_id_t ei_session;
+ crypto_mechanism_t ei_mech;
+ crypto_key_t ei_key;
+} crypto_encrypt_init_t;
+
+typedef struct crypto_encrypt_update {
+ uint_t eu_return_value;
+ crypto_session_id_t eu_session;
+ size_t eu_datalen;
+ caddr_t eu_databuf;
+ size_t eu_encrlen;
+ caddr_t eu_encrbuf;
+} crypto_encrypt_update_t;
+
+typedef struct crypto_encrypt_final {
+ uint_t ef_return_value;
+ crypto_session_id_t ef_session;
+ size_t ef_encrlen;
+ caddr_t ef_encrbuf;
+} crypto_encrypt_final_t;
+
+typedef struct crypto_decrypt {
+ uint_t cd_return_value;
+ crypto_session_id_t cd_session;
+ size_t cd_encrlen;
+ caddr_t cd_encrbuf;
+ size_t cd_datalen;
+ caddr_t cd_databuf;
+ uint_t cd_flags;
+} crypto_decrypt_t;
+
+typedef struct crypto_decrypt_init {
+ uint_t di_return_value;
+ crypto_session_id_t di_session;
+ crypto_mechanism_t di_mech;
+ crypto_key_t di_key;
+} crypto_decrypt_init_t;
+
+typedef struct crypto_decrypt_update {
+ uint_t du_return_value;
+ crypto_session_id_t du_session;
+ size_t du_encrlen;
+ caddr_t du_encrbuf;
+ size_t du_datalen;
+ caddr_t du_databuf;
+} crypto_decrypt_update_t;
+
+typedef struct crypto_decrypt_final {
+ uint_t df_return_value;
+ crypto_session_id_t df_session;
+ size_t df_datalen;
+ caddr_t df_databuf;
+} crypto_decrypt_final_t;
+
+typedef struct crypto_digest {
+ uint_t cd_return_value;
+ crypto_session_id_t cd_session;
+ size_t cd_datalen;
+ caddr_t cd_databuf;
+ size_t cd_digestlen;
+ caddr_t cd_digestbuf;
+} crypto_digest_t;
+
+typedef struct crypto_digest_init {
+ uint_t di_return_value;
+ crypto_session_id_t di_session;
+ crypto_mechanism_t di_mech;
+} crypto_digest_init_t;
+
+typedef struct crypto_digest_update {
+ uint_t du_return_value;
+ crypto_session_id_t du_session;
+ size_t du_datalen;
+ caddr_t du_databuf;
+} crypto_digest_update_t;
+
+typedef struct crypto_digest_key {
+ uint_t dk_return_value;
+ crypto_session_id_t dk_session;
+ crypto_key_t dk_key;
+} crypto_digest_key_t;
+
+typedef struct crypto_digest_final {
+ uint_t df_return_value;
+ crypto_session_id_t df_session;
+ size_t df_digestlen;
+ caddr_t df_digestbuf;
+} crypto_digest_final_t;
+
+typedef struct crypto_mac {
+ uint_t cm_return_value;
+ crypto_session_id_t cm_session;
+ size_t cm_datalen;
+ caddr_t cm_databuf;
+ size_t cm_maclen;
+ caddr_t cm_macbuf;
+} crypto_mac_t;
+
+typedef struct crypto_mac_init {
+ uint_t mi_return_value;
+ crypto_session_id_t mi_session;
+ crypto_mechanism_t mi_mech;
+ crypto_key_t mi_key;
+} crypto_mac_init_t;
+
+typedef struct crypto_mac_update {
+ uint_t mu_return_value;
+ crypto_session_id_t mu_session;
+ size_t mu_datalen;
+ caddr_t mu_databuf;
+} crypto_mac_update_t;
+
+typedef struct crypto_mac_final {
+ uint_t mf_return_value;
+ crypto_session_id_t mf_session;
+ size_t mf_maclen;
+ caddr_t mf_macbuf;
+} crypto_mac_final_t;
+
+typedef struct crypto_sign {
+ uint_t cs_return_value;
+ crypto_session_id_t cs_session;
+ size_t cs_datalen;
+ caddr_t cs_databuf;
+ size_t cs_signlen;
+ caddr_t cs_signbuf;
+} crypto_sign_t;
+
+typedef struct crypto_sign_init {
+ uint_t si_return_value;
+ crypto_session_id_t si_session;
+ crypto_mechanism_t si_mech;
+ crypto_key_t si_key;
+} crypto_sign_init_t;
+
+typedef struct crypto_sign_update {
+ uint_t su_return_value;
+ crypto_session_id_t su_session;
+ size_t su_datalen;
+ caddr_t su_databuf;
+} crypto_sign_update_t;
+
+typedef struct crypto_sign_final {
+ uint_t sf_return_value;
+ crypto_session_id_t sf_session;
+ size_t sf_signlen;
+ caddr_t sf_signbuf;
+} crypto_sign_final_t;
+
+typedef struct crypto_sign_recover_init {
+ uint_t ri_return_value;
+ crypto_session_id_t ri_session;
+ crypto_mechanism_t ri_mech;
+ crypto_key_t ri_key;
+} crypto_sign_recover_init_t;
+
+typedef struct crypto_sign_recover {
+ uint_t sr_return_value;
+ crypto_session_id_t sr_session;
+ size_t sr_datalen;
+ caddr_t sr_databuf;
+ size_t sr_signlen;
+ caddr_t sr_signbuf;
+} crypto_sign_recover_t;
+
+typedef struct crypto_verify {
+ uint_t cv_return_value;
+ crypto_session_id_t cv_session;
+ size_t cv_datalen;
+ caddr_t cv_databuf;
+ size_t cv_signlen;
+ caddr_t cv_signbuf;
+} crypto_verify_t;
+
+typedef struct crypto_verify_init {
+ uint_t vi_return_value;
+ crypto_session_id_t vi_session;
+ crypto_mechanism_t vi_mech;
+ crypto_key_t vi_key;
+} crypto_verify_init_t;
+
+typedef struct crypto_verify_update {
+ uint_t vu_return_value;
+ crypto_session_id_t vu_session;
+ size_t vu_datalen;
+ caddr_t vu_databuf;
+} crypto_verify_update_t;
+
+typedef struct crypto_verify_final {
+ uint_t vf_return_value;
+ crypto_session_id_t vf_session;
+ size_t vf_signlen;
+ caddr_t vf_signbuf;
+} crypto_verify_final_t;
+
+typedef struct crypto_verify_recover_init {
+ uint_t ri_return_value;
+ crypto_session_id_t ri_session;
+ crypto_mechanism_t ri_mech;
+ crypto_key_t ri_key;
+} crypto_verify_recover_init_t;
+
+typedef struct crypto_verify_recover {
+ uint_t vr_return_value;
+ crypto_session_id_t vr_session;
+ size_t vr_signlen;
+ caddr_t vr_signbuf;
+ size_t vr_datalen;
+ caddr_t vr_databuf;
+} crypto_verify_recover_t;
+
+typedef struct crypto_digest_encrypt_update {
+ uint_t eu_return_value;
+ crypto_session_id_t eu_session;
+ size_t eu_datalen;
+ caddr_t eu_databuf;
+ size_t eu_encrlen;
+ caddr_t eu_encrbuf;
+} crypto_digest_encrypt_update_t;
+
+typedef struct crypto_decrypt_digest_update {
+ uint_t du_return_value;
+ crypto_session_id_t du_session;
+ size_t du_encrlen;
+ caddr_t du_encrbuf;
+ size_t du_datalen;
+ caddr_t du_databuf;
+} crypto_decrypt_digest_update_t;
+
+typedef struct crypto_sign_encrypt_update {
+ uint_t eu_return_value;
+ crypto_session_id_t eu_session;
+ size_t eu_datalen;
+ caddr_t eu_databuf;
+ size_t eu_encrlen;
+ caddr_t eu_encrbuf;
+} crypto_sign_encrypt_update_t;
+
+typedef struct crypto_decrypt_verify_update {
+ uint_t vu_return_value;
+ crypto_session_id_t vu_session;
+ size_t vu_encrlen;
+ caddr_t vu_encrbuf;
+ size_t vu_datalen;
+ caddr_t vu_databuf;
+} crypto_decrypt_verify_update_t;
+
+#ifdef _KERNEL
+#ifdef _SYSCALL32
+
+typedef struct crypto_encrypt32 {
+ uint32_t ce_return_value;
+ crypto_session_id_t ce_session;
+ size32_t ce_datalen;
+ caddr32_t ce_databuf;
+ size32_t ce_encrlen;
+ caddr32_t ce_encrbuf;
+ uint32_t ce_flags;
+} crypto_encrypt32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack(4)
+#endif
+
+typedef struct crypto_encrypt_init32 {
+ uint32_t ei_return_value;
+ crypto_session_id_t ei_session;
+ crypto_mechanism32_t ei_mech;
+ crypto_key32_t ei_key;
+} crypto_encrypt_init32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack()
+#endif
+
+typedef struct crypto_encrypt_update32 {
+ uint32_t eu_return_value;
+ crypto_session_id_t eu_session;
+ size32_t eu_datalen;
+ caddr32_t eu_databuf;
+ size32_t eu_encrlen;
+ caddr32_t eu_encrbuf;
+} crypto_encrypt_update32_t;
+
+typedef struct crypto_encrypt_final32 {
+ uint32_t ef_return_value;
+ crypto_session_id_t ef_session;
+ size32_t ef_encrlen;
+ caddr32_t ef_encrbuf;
+} crypto_encrypt_final32_t;
+
+typedef struct crypto_decrypt32 {
+ uint32_t cd_return_value;
+ crypto_session_id_t cd_session;
+ size32_t cd_encrlen;
+ caddr32_t cd_encrbuf;
+ size32_t cd_datalen;
+ caddr32_t cd_databuf;
+ uint32_t cd_flags;
+} crypto_decrypt32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack(4)
+#endif
+
+typedef struct crypto_decrypt_init32 {
+ uint32_t di_return_value;
+ crypto_session_id_t di_session;
+ crypto_mechanism32_t di_mech;
+ crypto_key32_t di_key;
+} crypto_decrypt_init32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack()
+#endif
+
+typedef struct crypto_decrypt_update32 {
+ uint32_t du_return_value;
+ crypto_session_id_t du_session;
+ size32_t du_encrlen;
+ caddr32_t du_encrbuf;
+ size32_t du_datalen;
+ caddr32_t du_databuf;
+} crypto_decrypt_update32_t;
+
+typedef struct crypto_decrypt_final32 {
+ uint32_t df_return_value;
+ crypto_session_id_t df_session;
+ size32_t df_datalen;
+ caddr32_t df_databuf;
+} crypto_decrypt_final32_t;
+
+typedef struct crypto_digest32 {
+ uint32_t cd_return_value;
+ crypto_session_id_t cd_session;
+ size32_t cd_datalen;
+ caddr32_t cd_databuf;
+ size32_t cd_digestlen;
+ caddr32_t cd_digestbuf;
+} crypto_digest32_t;
+
+typedef struct crypto_digest_init32 {
+ uint32_t di_return_value;
+ crypto_session_id_t di_session;
+ crypto_mechanism32_t di_mech;
+} crypto_digest_init32_t;
+
+typedef struct crypto_digest_update32 {
+ uint32_t du_return_value;
+ crypto_session_id_t du_session;
+ size32_t du_datalen;
+ caddr32_t du_databuf;
+} crypto_digest_update32_t;
+
+typedef struct crypto_digest_key32 {
+ uint32_t dk_return_value;
+ crypto_session_id_t dk_session;
+ crypto_key32_t dk_key;
+} crypto_digest_key32_t;
+
+typedef struct crypto_digest_final32 {
+ uint32_t df_return_value;
+ crypto_session_id_t df_session;
+ size32_t df_digestlen;
+ caddr32_t df_digestbuf;
+} crypto_digest_final32_t;
+
+typedef struct crypto_mac32 {
+ uint32_t cm_return_value;
+ crypto_session_id_t cm_session;
+ size32_t cm_datalen;
+ caddr32_t cm_databuf;
+ size32_t cm_maclen;
+ caddr32_t cm_macbuf;
+} crypto_mac32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack(4)
+#endif
+
+typedef struct crypto_mac_init32 {
+ uint32_t mi_return_value;
+ crypto_session_id_t mi_session;
+ crypto_mechanism32_t mi_mech;
+ crypto_key32_t mi_key;
+} crypto_mac_init32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack()
+#endif
+
+typedef struct crypto_mac_update32 {
+ uint32_t mu_return_value;
+ crypto_session_id_t mu_session;
+ size32_t mu_datalen;
+ caddr32_t mu_databuf;
+} crypto_mac_update32_t;
+
+typedef struct crypto_mac_final32 {
+ uint32_t mf_return_value;
+ crypto_session_id_t mf_session;
+ size32_t mf_maclen;
+ caddr32_t mf_macbuf;
+} crypto_mac_final32_t;
+
+typedef struct crypto_sign32 {
+ uint32_t cs_return_value;
+ crypto_session_id_t cs_session;
+ size32_t cs_datalen;
+ caddr32_t cs_databuf;
+ size32_t cs_signlen;
+ caddr32_t cs_signbuf;
+} crypto_sign32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack(4)
+#endif
+
+typedef struct crypto_sign_init32 {
+ uint32_t si_return_value;
+ crypto_session_id_t si_session;
+ crypto_mechanism32_t si_mech;
+ crypto_key32_t si_key;
+} crypto_sign_init32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack()
+#endif
+
+typedef struct crypto_sign_update32 {
+ uint32_t su_return_value;
+ crypto_session_id_t su_session;
+ size32_t su_datalen;
+ caddr32_t su_databuf;
+} crypto_sign_update32_t;
+
+typedef struct crypto_sign_final32 {
+ uint32_t sf_return_value;
+ crypto_session_id_t sf_session;
+ size32_t sf_signlen;
+ caddr32_t sf_signbuf;
+} crypto_sign_final32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack(4)
+#endif
+
+typedef struct crypto_sign_recover_init32 {
+ uint32_t ri_return_value;
+ crypto_session_id_t ri_session;
+ crypto_mechanism32_t ri_mech;
+ crypto_key32_t ri_key;
+} crypto_sign_recover_init32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack()
+#endif
+
+typedef struct crypto_sign_recover32 {
+ uint32_t sr_return_value;
+ crypto_session_id_t sr_session;
+ size32_t sr_datalen;
+ caddr32_t sr_databuf;
+ size32_t sr_signlen;
+ caddr32_t sr_signbuf;
+} crypto_sign_recover32_t;
+
+typedef struct crypto_verify32 {
+ uint32_t cv_return_value;
+ crypto_session_id_t cv_session;
+ size32_t cv_datalen;
+ caddr32_t cv_databuf;
+ size32_t cv_signlen;
+ caddr32_t cv_signbuf;
+} crypto_verify32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack(4)
+#endif
+
+typedef struct crypto_verify_init32 {
+ uint32_t vi_return_value;
+ crypto_session_id_t vi_session;
+ crypto_mechanism32_t vi_mech;
+ crypto_key32_t vi_key;
+} crypto_verify_init32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack()
+#endif
+
+typedef struct crypto_verify_update32 {
+ uint32_t vu_return_value;
+ crypto_session_id_t vu_session;
+ size32_t vu_datalen;
+ caddr32_t vu_databuf;
+} crypto_verify_update32_t;
+
+typedef struct crypto_verify_final32 {
+ uint32_t vf_return_value;
+ crypto_session_id_t vf_session;
+ size32_t vf_signlen;
+ caddr32_t vf_signbuf;
+} crypto_verify_final32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack(4)
+#endif
+
+typedef struct crypto_verify_recover_init32 {
+ uint32_t ri_return_value;
+ crypto_session_id_t ri_session;
+ crypto_mechanism32_t ri_mech;
+ crypto_key32_t ri_key;
+} crypto_verify_recover_init32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack()
+#endif
+
+typedef struct crypto_verify_recover32 {
+ uint32_t vr_return_value;
+ crypto_session_id_t vr_session;
+ size32_t vr_signlen;
+ caddr32_t vr_signbuf;
+ size32_t vr_datalen;
+ caddr32_t vr_databuf;
+} crypto_verify_recover32_t;
+
+typedef struct crypto_digest_encrypt_update32 {
+ uint32_t eu_return_value;
+ crypto_session_id_t eu_session;
+ size32_t eu_datalen;
+ caddr32_t eu_databuf;
+ size32_t eu_encrlen;
+ caddr32_t eu_encrbuf;
+} crypto_digest_encrypt_update32_t;
+
+typedef struct crypto_decrypt_digest_update32 {
+ uint32_t du_return_value;
+ crypto_session_id_t du_session;
+ size32_t du_encrlen;
+ caddr32_t du_encrbuf;
+ size32_t du_datalen;
+ caddr32_t du_databuf;
+} crypto_decrypt_digest_update32_t;
+
+typedef struct crypto_sign_encrypt_update32 {
+ uint32_t eu_return_value;
+ crypto_session_id_t eu_session;
+ size32_t eu_datalen;
+ caddr32_t eu_databuf;
+ size32_t eu_encrlen;
+ caddr32_t eu_encrbuf;
+} crypto_sign_encrypt_update32_t;
+
+typedef struct crypto_decrypt_verify_update32 {
+ uint32_t vu_return_value;
+ crypto_session_id_t vu_session;
+ size32_t vu_encrlen;
+ caddr32_t vu_encrbuf;
+ size32_t vu_datalen;
+ caddr32_t vu_databuf;
+} crypto_decrypt_verify_update32_t;
+
+#endif /* _SYSCALL32 */
+#endif /* _KERNEL */
+
+#define CRYPTO_ENCRYPT CRYPTO(50)
+#define CRYPTO_ENCRYPT_INIT CRYPTO(51)
+#define CRYPTO_ENCRYPT_UPDATE CRYPTO(52)
+#define CRYPTO_ENCRYPT_FINAL CRYPTO(53)
+#define CRYPTO_DECRYPT CRYPTO(54)
+#define CRYPTO_DECRYPT_INIT CRYPTO(55)
+#define CRYPTO_DECRYPT_UPDATE CRYPTO(56)
+#define CRYPTO_DECRYPT_FINAL CRYPTO(57)
+
+#define CRYPTO_DIGEST CRYPTO(58)
+#define CRYPTO_DIGEST_INIT CRYPTO(59)
+#define CRYPTO_DIGEST_UPDATE CRYPTO(60)
+#define CRYPTO_DIGEST_KEY CRYPTO(61)
+#define CRYPTO_DIGEST_FINAL CRYPTO(62)
+#define CRYPTO_MAC CRYPTO(63)
+#define CRYPTO_MAC_INIT CRYPTO(64)
+#define CRYPTO_MAC_UPDATE CRYPTO(65)
+#define CRYPTO_MAC_FINAL CRYPTO(66)
+
+#define CRYPTO_SIGN CRYPTO(67)
+#define CRYPTO_SIGN_INIT CRYPTO(68)
+#define CRYPTO_SIGN_UPDATE CRYPTO(69)
+#define CRYPTO_SIGN_FINAL CRYPTO(70)
+#define CRYPTO_SIGN_RECOVER_INIT CRYPTO(71)
+#define CRYPTO_SIGN_RECOVER CRYPTO(72)
+#define CRYPTO_VERIFY CRYPTO(73)
+#define CRYPTO_VERIFY_INIT CRYPTO(74)
+#define CRYPTO_VERIFY_UPDATE CRYPTO(75)
+#define CRYPTO_VERIFY_FINAL CRYPTO(76)
+#define CRYPTO_VERIFY_RECOVER_INIT CRYPTO(77)
+#define CRYPTO_VERIFY_RECOVER CRYPTO(78)
+
+#define CRYPTO_DIGEST_ENCRYPT_UPDATE CRYPTO(79)
+#define CRYPTO_DECRYPT_DIGEST_UPDATE CRYPTO(80)
+#define CRYPTO_SIGN_ENCRYPT_UPDATE CRYPTO(81)
+#define CRYPTO_DECRYPT_VERIFY_UPDATE CRYPTO(82)
+
+/*
+ * Random Number Ioctls
+ */
+typedef struct crypto_seed_random {
+ uint_t sr_return_value;
+ crypto_session_id_t sr_session;
+ size_t sr_seedlen;
+ caddr_t sr_seedbuf;
+} crypto_seed_random_t;
+
+typedef struct crypto_generate_random {
+ uint_t gr_return_value;
+ crypto_session_id_t gr_session;
+ caddr_t gr_buf;
+ size_t gr_buflen;
+} crypto_generate_random_t;
+
+#ifdef _KERNEL
+#ifdef _SYSCALL32
+
+typedef struct crypto_seed_random32 {
+ uint32_t sr_return_value;
+ crypto_session_id_t sr_session;
+ size32_t sr_seedlen;
+ caddr32_t sr_seedbuf;
+} crypto_seed_random32_t;
+
+typedef struct crypto_generate_random32 {
+ uint32_t gr_return_value;
+ crypto_session_id_t gr_session;
+ caddr32_t gr_buf;
+ size32_t gr_buflen;
+} crypto_generate_random32_t;
+
+#endif /* _SYSCALL32 */
+#endif /* _KERNEL */
+
+#define CRYPTO_SEED_RANDOM CRYPTO(90)
+#define CRYPTO_GENERATE_RANDOM CRYPTO(91)
+
+/*
+ * Object Management Ioctls
+ */
+typedef struct crypto_object_create {
+ uint_t oc_return_value;
+ crypto_session_id_t oc_session;
+ crypto_object_id_t oc_handle;
+ uint_t oc_count;
+ caddr_t oc_attributes;
+} crypto_object_create_t;
+
+typedef struct crypto_object_copy {
+ uint_t oc_return_value;
+ crypto_session_id_t oc_session;
+ crypto_object_id_t oc_handle;
+ crypto_object_id_t oc_new_handle;
+ uint_t oc_count;
+ caddr_t oc_new_attributes;
+} crypto_object_copy_t;
+
+typedef struct crypto_object_destroy {
+ uint_t od_return_value;
+ crypto_session_id_t od_session;
+ crypto_object_id_t od_handle;
+} crypto_object_destroy_t;
+
+typedef struct crypto_object_get_attribute_value {
+ uint_t og_return_value;
+ crypto_session_id_t og_session;
+ crypto_object_id_t og_handle;
+ uint_t og_count;
+ caddr_t og_attributes;
+} crypto_object_get_attribute_value_t;
+
+typedef struct crypto_object_get_size {
+ uint_t gs_return_value;
+ crypto_session_id_t gs_session;
+ crypto_object_id_t gs_handle;
+ size_t gs_size;
+} crypto_object_get_size_t;
+
+typedef struct crypto_object_set_attribute_value {
+ uint_t sa_return_value;
+ crypto_session_id_t sa_session;
+ crypto_object_id_t sa_handle;
+ uint_t sa_count;
+ caddr_t sa_attributes;
+} crypto_object_set_attribute_value_t;
+
+typedef struct crypto_object_find_init {
+ uint_t fi_return_value;
+ crypto_session_id_t fi_session;
+ uint_t fi_count;
+ caddr_t fi_attributes;
+} crypto_object_find_init_t;
+
+typedef struct crypto_object_find_update {
+ uint_t fu_return_value;
+ crypto_session_id_t fu_session;
+ uint_t fu_max_count;
+ uint_t fu_count;
+ caddr_t fu_handles;
+} crypto_object_find_update_t;
+
+typedef struct crypto_object_find_final {
+ uint_t ff_return_value;
+ crypto_session_id_t ff_session;
+} crypto_object_find_final_t;
+
+#ifdef _KERNEL
+#ifdef _SYSCALL32
+
+typedef struct crypto_object_create32 {
+ uint32_t oc_return_value;
+ crypto_session_id_t oc_session;
+ crypto_object_id_t oc_handle;
+ uint32_t oc_count;
+ caddr32_t oc_attributes;
+} crypto_object_create32_t;
+
+typedef struct crypto_object_copy32 {
+ uint32_t oc_return_value;
+ crypto_session_id_t oc_session;
+ crypto_object_id_t oc_handle;
+ crypto_object_id_t oc_new_handle;
+ uint32_t oc_count;
+ caddr32_t oc_new_attributes;
+} crypto_object_copy32_t;
+
+typedef struct crypto_object_destroy32 {
+ uint32_t od_return_value;
+ crypto_session_id_t od_session;
+ crypto_object_id_t od_handle;
+} crypto_object_destroy32_t;
+
+typedef struct crypto_object_get_attribute_value32 {
+ uint32_t og_return_value;
+ crypto_session_id_t og_session;
+ crypto_object_id_t og_handle;
+ uint32_t og_count;
+ caddr32_t og_attributes;
+} crypto_object_get_attribute_value32_t;
+
+typedef struct crypto_object_get_size32 {
+ uint32_t gs_return_value;
+ crypto_session_id_t gs_session;
+ crypto_object_id_t gs_handle;
+ size32_t gs_size;
+} crypto_object_get_size32_t;
+
+typedef struct crypto_object_set_attribute_value32 {
+ uint32_t sa_return_value;
+ crypto_session_id_t sa_session;
+ crypto_object_id_t sa_handle;
+ uint32_t sa_count;
+ caddr32_t sa_attributes;
+} crypto_object_set_attribute_value32_t;
+
+typedef struct crypto_object_find_init32 {
+ uint32_t fi_return_value;
+ crypto_session_id_t fi_session;
+ uint32_t fi_count;
+ caddr32_t fi_attributes;
+} crypto_object_find_init32_t;
+
+typedef struct crypto_object_find_update32 {
+ uint32_t fu_return_value;
+ crypto_session_id_t fu_session;
+ uint32_t fu_max_count;
+ uint32_t fu_count;
+ caddr32_t fu_handles;
+} crypto_object_find_update32_t;
+
+typedef struct crypto_object_find_final32 {
+ uint32_t ff_return_value;
+ crypto_session_id_t ff_session;
+} crypto_object_find_final32_t;
+
+#endif /* _SYSCALL32 */
+#endif /* _KERNEL */
+
+#define CRYPTO_OBJECT_CREATE CRYPTO(100)
+#define CRYPTO_OBJECT_COPY CRYPTO(101)
+#define CRYPTO_OBJECT_DESTROY CRYPTO(102)
+#define CRYPTO_OBJECT_GET_ATTRIBUTE_VALUE CRYPTO(103)
+#define CRYPTO_OBJECT_GET_SIZE CRYPTO(104)
+#define CRYPTO_OBJECT_SET_ATTRIBUTE_VALUE CRYPTO(105)
+#define CRYPTO_OBJECT_FIND_INIT CRYPTO(106)
+#define CRYPTO_OBJECT_FIND_UPDATE CRYPTO(107)
+#define CRYPTO_OBJECT_FIND_FINAL CRYPTO(108)
+
+/*
+ * Key Generation Ioctls
+ */
+typedef struct crypto_object_generate_key {
+ uint_t gk_return_value;
+ crypto_session_id_t gk_session;
+ crypto_object_id_t gk_handle;
+ crypto_mechanism_t gk_mechanism;
+ uint_t gk_count;
+ caddr_t gk_attributes;
+} crypto_object_generate_key_t;
+
+typedef struct crypto_object_generate_key_pair {
+ uint_t kp_return_value;
+ crypto_session_id_t kp_session;
+ crypto_object_id_t kp_public_handle;
+ crypto_object_id_t kp_private_handle;
+ uint_t kp_public_count;
+ uint_t kp_private_count;
+ caddr_t kp_public_attributes;
+ caddr_t kp_private_attributes;
+ crypto_mechanism_t kp_mechanism;
+} crypto_object_generate_key_pair_t;
+
+typedef struct crypto_object_wrap_key {
+ uint_t wk_return_value;
+ crypto_session_id_t wk_session;
+ crypto_mechanism_t wk_mechanism;
+ crypto_key_t wk_wrapping_key;
+ crypto_object_id_t wk_object_handle;
+ size_t wk_wrapped_key_len;
+ caddr_t wk_wrapped_key;
+} crypto_object_wrap_key_t;
+
+typedef struct crypto_object_unwrap_key {
+ uint_t uk_return_value;
+ crypto_session_id_t uk_session;
+ crypto_mechanism_t uk_mechanism;
+ crypto_key_t uk_unwrapping_key;
+ crypto_object_id_t uk_object_handle;
+ size_t uk_wrapped_key_len;
+ caddr_t uk_wrapped_key;
+ uint_t uk_count;
+ caddr_t uk_attributes;
+} crypto_object_unwrap_key_t;
+
+typedef struct crypto_derive_key {
+ uint_t dk_return_value;
+ crypto_session_id_t dk_session;
+ crypto_mechanism_t dk_mechanism;
+ crypto_key_t dk_base_key;
+ crypto_object_id_t dk_object_handle;
+ uint_t dk_count;
+ caddr_t dk_attributes;
+} crypto_derive_key_t;
+
+#ifdef _KERNEL
+#ifdef _SYSCALL32
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack(4)
+#endif
+
+typedef struct crypto_object_generate_key32 {
+ uint32_t gk_return_value;
+ crypto_session_id_t gk_session;
+ crypto_object_id_t gk_handle;
+ crypto_mechanism32_t gk_mechanism;
+ uint32_t gk_count;
+ caddr32_t gk_attributes;
+} crypto_object_generate_key32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack()
+#endif
+
+typedef struct crypto_object_generate_key_pair32 {
+ uint32_t kp_return_value;
+ crypto_session_id_t kp_session;
+ crypto_object_id_t kp_public_handle;
+ crypto_object_id_t kp_private_handle;
+ uint32_t kp_public_count;
+ uint32_t kp_private_count;
+ caddr32_t kp_public_attributes;
+ caddr32_t kp_private_attributes;
+ crypto_mechanism32_t kp_mechanism;
+} crypto_object_generate_key_pair32_t;
+
+typedef struct crypto_object_wrap_key32 {
+ uint32_t wk_return_value;
+ crypto_session_id_t wk_session;
+ crypto_mechanism32_t wk_mechanism;
+ crypto_key32_t wk_wrapping_key;
+ crypto_object_id_t wk_object_handle;
+ size32_t wk_wrapped_key_len;
+ caddr32_t wk_wrapped_key;
+} crypto_object_wrap_key32_t;
+
+typedef struct crypto_object_unwrap_key32 {
+ uint32_t uk_return_value;
+ crypto_session_id_t uk_session;
+ crypto_mechanism32_t uk_mechanism;
+ crypto_key32_t uk_unwrapping_key;
+ crypto_object_id_t uk_object_handle;
+ size32_t uk_wrapped_key_len;
+ caddr32_t uk_wrapped_key;
+ uint32_t uk_count;
+ caddr32_t uk_attributes;
+} crypto_object_unwrap_key32_t;
+
+typedef struct crypto_derive_key32 {
+ uint32_t dk_return_value;
+ crypto_session_id_t dk_session;
+ crypto_mechanism32_t dk_mechanism;
+ crypto_key32_t dk_base_key;
+ crypto_object_id_t dk_object_handle;
+ uint32_t dk_count;
+ caddr32_t dk_attributes;
+} crypto_derive_key32_t;
+
+#endif /* _SYSCALL32 */
+#endif /* _KERNEL */
+
+#define CRYPTO_GENERATE_KEY CRYPTO(110)
+#define CRYPTO_GENERATE_KEY_PAIR CRYPTO(111)
+#define CRYPTO_WRAP_KEY CRYPTO(112)
+#define CRYPTO_UNWRAP_KEY CRYPTO(113)
+#define CRYPTO_DERIVE_KEY CRYPTO(114)
+
+/*
+ * Provider Management Ioctls
+ */
+
+typedef struct crypto_get_provider_list {
+ uint_t pl_return_value;
+ uint_t pl_count;
+ crypto_provider_entry_t pl_list[1];
+} crypto_get_provider_list_t;
+
+typedef struct crypto_provider_data {
+ uchar_t pd_prov_desc[CRYPTO_PROVIDER_DESCR_MAX_LEN];
+ uchar_t pd_label[CRYPTO_EXT_SIZE_LABEL];
+ uchar_t pd_manufacturerID[CRYPTO_EXT_SIZE_MANUF];
+ uchar_t pd_model[CRYPTO_EXT_SIZE_MODEL];
+ uchar_t pd_serial_number[CRYPTO_EXT_SIZE_SERIAL];
+ ulong_t pd_flags;
+ ulong_t pd_max_session_count;
+ ulong_t pd_session_count;
+ ulong_t pd_max_rw_session_count;
+ ulong_t pd_rw_session_count;
+ ulong_t pd_max_pin_len;
+ ulong_t pd_min_pin_len;
+ ulong_t pd_total_public_memory;
+ ulong_t pd_free_public_memory;
+ ulong_t pd_total_private_memory;
+ ulong_t pd_free_private_memory;
+ crypto_version_t pd_hardware_version;
+ crypto_version_t pd_firmware_version;
+ uchar_t pd_time[CRYPTO_EXT_SIZE_TIME];
+} crypto_provider_data_t;
+
+typedef struct crypto_get_provider_info {
+ uint_t gi_return_value;
+ crypto_provider_id_t gi_provider_id;
+ crypto_provider_data_t gi_provider_data;
+} crypto_get_provider_info_t;
+
+typedef struct crypto_get_provider_mechanisms {
+ uint_t pm_return_value;
+ crypto_provider_id_t pm_provider_id;
+ uint_t pm_count;
+ crypto_mech_name_t pm_list[1];
+} crypto_get_provider_mechanisms_t;
+
+typedef struct crypto_get_provider_mechanism_info {
+ uint_t mi_return_value;
+ crypto_provider_id_t mi_provider_id;
+ crypto_mech_name_t mi_mechanism_name;
+ uint32_t mi_min_key_size;
+ uint32_t mi_max_key_size;
+ uint32_t mi_flags;
+} crypto_get_provider_mechanism_info_t;
+
+typedef struct crypto_init_token {
+ uint_t it_return_value;
+ crypto_provider_id_t it_provider_id;
+ caddr_t it_pin;
+ size_t it_pin_len;
+ caddr_t it_label;
+} crypto_init_token_t;
+
+typedef struct crypto_init_pin {
+ uint_t ip_return_value;
+ crypto_session_id_t ip_session;
+ caddr_t ip_pin;
+ size_t ip_pin_len;
+} crypto_init_pin_t;
+
+typedef struct crypto_set_pin {
+ uint_t sp_return_value;
+ crypto_session_id_t sp_session;
+ caddr_t sp_old_pin;
+ size_t sp_old_len;
+ caddr_t sp_new_pin;
+ size_t sp_new_len;
+} crypto_set_pin_t;
+
+#ifdef _KERNEL
+#ifdef _SYSCALL32
+
+typedef struct crypto_get_provider_list32 {
+ uint32_t pl_return_value;
+ uint32_t pl_count;
+ crypto_provider_entry_t pl_list[1];
+} crypto_get_provider_list32_t;
+
+typedef struct crypto_version32 {
+ uchar_t cv_major;
+ uchar_t cv_minor;
+} crypto_version32_t;
+
+typedef struct crypto_provider_data32 {
+ uchar_t pd_prov_desc[CRYPTO_PROVIDER_DESCR_MAX_LEN];
+ uchar_t pd_label[CRYPTO_EXT_SIZE_LABEL];
+ uchar_t pd_manufacturerID[CRYPTO_EXT_SIZE_MANUF];
+ uchar_t pd_model[CRYPTO_EXT_SIZE_MODEL];
+ uchar_t pd_serial_number[CRYPTO_EXT_SIZE_SERIAL];
+ uint32_t pd_flags;
+ uint32_t pd_max_session_count;
+ uint32_t pd_session_count;
+ uint32_t pd_max_rw_session_count;
+ uint32_t pd_rw_session_count;
+ uint32_t pd_max_pin_len;
+ uint32_t pd_min_pin_len;
+ uint32_t pd_total_public_memory;
+ uint32_t pd_free_public_memory;
+ uint32_t pd_total_private_memory;
+ uint32_t pd_free_private_memory;
+ crypto_version32_t pd_hardware_version;
+ crypto_version32_t pd_firmware_version;
+ uchar_t pd_time[CRYPTO_EXT_SIZE_TIME];
+} crypto_provider_data32_t;
+
+typedef struct crypto_get_provider_info32 {
+ uint32_t gi_return_value;
+ crypto_provider_id_t gi_provider_id;
+ crypto_provider_data32_t gi_provider_data;
+} crypto_get_provider_info32_t;
+
+typedef struct crypto_get_provider_mechanisms32 {
+ uint32_t pm_return_value;
+ crypto_provider_id_t pm_provider_id;
+ uint32_t pm_count;
+ crypto_mech_name_t pm_list[1];
+} crypto_get_provider_mechanisms32_t;
+
+typedef struct crypto_init_token32 {
+ uint32_t it_return_value;
+ crypto_provider_id_t it_provider_id;
+ caddr32_t it_pin;
+ size32_t it_pin_len;
+ caddr32_t it_label;
+} crypto_init_token32_t;
+
+typedef struct crypto_init_pin32 {
+ uint32_t ip_return_value;
+ crypto_session_id_t ip_session;
+ caddr32_t ip_pin;
+ size32_t ip_pin_len;
+} crypto_init_pin32_t;
+
+typedef struct crypto_set_pin32 {
+ uint32_t sp_return_value;
+ crypto_session_id_t sp_session;
+ caddr32_t sp_old_pin;
+ size32_t sp_old_len;
+ caddr32_t sp_new_pin;
+ size32_t sp_new_len;
+} crypto_set_pin32_t;
+
+#endif /* _SYSCALL32 */
+#endif /* _KERNEL */
+
+#define CRYPTO_GET_PROVIDER_LIST CRYPTO(120)
+#define CRYPTO_GET_PROVIDER_INFO CRYPTO(121)
+#define CRYPTO_GET_PROVIDER_MECHANISMS CRYPTO(122)
+#define CRYPTO_GET_PROVIDER_MECHANISM_INFO CRYPTO(123)
+#define CRYPTO_INIT_TOKEN CRYPTO(124)
+#define CRYPTO_INIT_PIN CRYPTO(125)
+#define CRYPTO_SET_PIN CRYPTO(126)
+
+/*
+ * No (Key) Store Key Generation Ioctls
+ */
+typedef struct crypto_nostore_generate_key {
+ uint_t ngk_return_value;
+ crypto_session_id_t ngk_session;
+ crypto_mechanism_t ngk_mechanism;
+ uint_t ngk_in_count;
+ uint_t ngk_out_count;
+ caddr_t ngk_in_attributes;
+ caddr_t ngk_out_attributes;
+} crypto_nostore_generate_key_t;
+
+typedef struct crypto_nostore_generate_key_pair {
+ uint_t nkp_return_value;
+ crypto_session_id_t nkp_session;
+ uint_t nkp_in_public_count;
+ uint_t nkp_in_private_count;
+ uint_t nkp_out_public_count;
+ uint_t nkp_out_private_count;
+ caddr_t nkp_in_public_attributes;
+ caddr_t nkp_in_private_attributes;
+ caddr_t nkp_out_public_attributes;
+ caddr_t nkp_out_private_attributes;
+ crypto_mechanism_t nkp_mechanism;
+} crypto_nostore_generate_key_pair_t;
+
+typedef struct crypto_nostore_derive_key {
+ uint_t ndk_return_value;
+ crypto_session_id_t ndk_session;
+ crypto_mechanism_t ndk_mechanism;
+ crypto_key_t ndk_base_key;
+ uint_t ndk_in_count;
+ uint_t ndk_out_count;
+ caddr_t ndk_in_attributes;
+ caddr_t ndk_out_attributes;
+} crypto_nostore_derive_key_t;
+
+#ifdef _KERNEL
+#ifdef _SYSCALL32
+
+typedef struct crypto_nostore_generate_key32 {
+ uint32_t ngk_return_value;
+ crypto_session_id_t ngk_session;
+ crypto_mechanism32_t ngk_mechanism;
+ uint32_t ngk_in_count;
+ uint32_t ngk_out_count;
+ caddr32_t ngk_in_attributes;
+ caddr32_t ngk_out_attributes;
+} crypto_nostore_generate_key32_t;
+
+typedef struct crypto_nostore_generate_key_pair32 {
+ uint32_t nkp_return_value;
+ crypto_session_id_t nkp_session;
+ uint32_t nkp_in_public_count;
+ uint32_t nkp_in_private_count;
+ uint32_t nkp_out_public_count;
+ uint32_t nkp_out_private_count;
+ caddr32_t nkp_in_public_attributes;
+ caddr32_t nkp_in_private_attributes;
+ caddr32_t nkp_out_public_attributes;
+ caddr32_t nkp_out_private_attributes;
+ crypto_mechanism32_t nkp_mechanism;
+} crypto_nostore_generate_key_pair32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack(4)
+#endif
+
+typedef struct crypto_nostore_derive_key32 {
+ uint32_t ndk_return_value;
+ crypto_session_id_t ndk_session;
+ crypto_mechanism32_t ndk_mechanism;
+ crypto_key32_t ndk_base_key;
+ uint32_t ndk_in_count;
+ uint32_t ndk_out_count;
+ caddr32_t ndk_in_attributes;
+ caddr32_t ndk_out_attributes;
+} crypto_nostore_derive_key32_t;
+
+#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4
+#pragma pack()
+#endif
+
+#endif /* _SYSCALL32 */
+#endif /* _KERNEL */
+
+#define CRYPTO_NOSTORE_GENERATE_KEY CRYPTO(127)
+#define CRYPTO_NOSTORE_GENERATE_KEY_PAIR CRYPTO(128)
+#define CRYPTO_NOSTORE_DERIVE_KEY CRYPTO(129)
+
+/*
+ * Mechanism Ioctls
+ */
+
+typedef struct crypto_get_mechanism_list {
+ uint_t ml_return_value;
+ uint_t ml_count;
+ crypto_mech_name_t ml_list[1];
+} crypto_get_mechanism_list_t;
+
+typedef struct crypto_get_all_mechanism_info {
+ uint_t mi_return_value;
+ crypto_mech_name_t mi_mechanism_name;
+ uint_t mi_count;
+ crypto_mechanism_info_t mi_list[1];
+} crypto_get_all_mechanism_info_t;
+
+#ifdef _KERNEL
+#ifdef _SYSCALL32
+
+typedef struct crypto_get_mechanism_list32 {
+ uint32_t ml_return_value;
+ uint32_t ml_count;
+ crypto_mech_name_t ml_list[1];
+} crypto_get_mechanism_list32_t;
+
+typedef struct crypto_get_all_mechanism_info32 {
+ uint32_t mi_return_value;
+ crypto_mech_name_t mi_mechanism_name;
+ uint32_t mi_count;
+ crypto_mechanism_info32_t mi_list[1];
+} crypto_get_all_mechanism_info32_t;
+
+#endif /* _SYSCALL32 */
+#endif /* _KERNEL */
+
+#define CRYPTO_GET_MECHANISM_LIST CRYPTO(140)
+#define CRYPTO_GET_ALL_MECHANISM_INFO CRYPTO(141)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_CRYPTO_IOCTL_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_CRYPTO_IOCTLADMIN_H
+#define _SYS_CRYPTO_IOCTLADMIN_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+
+#define ADMIN_IOCTL_DEVICE "/dev/cryptoadm"
+
+#define CRYPTOADMIN(x) (('y' << 8) | (x))
+
+/*
+ * Administrative IOCTLs
+ */
+
+typedef struct crypto_get_dev_list {
+ uint_t dl_return_value;
+ uint_t dl_dev_count;
+ crypto_dev_list_entry_t dl_devs[1];
+} crypto_get_dev_list_t;
+
+typedef struct crypto_get_soft_list {
+ uint_t sl_return_value;
+ uint_t sl_soft_count;
+ size_t sl_soft_len;
+ caddr_t sl_soft_names;
+} crypto_get_soft_list_t;
+
+typedef struct crypto_get_dev_info {
+ uint_t di_return_value;
+ char di_dev_name[MAXNAMELEN];
+ uint_t di_dev_instance;
+ uint_t di_count;
+ crypto_mech_name_t di_list[1];
+} crypto_get_dev_info_t;
+
+typedef struct crypto_get_soft_info {
+ uint_t si_return_value;
+ char si_name[MAXNAMELEN];
+ uint_t si_count;
+ crypto_mech_name_t si_list[1];
+} crypto_get_soft_info_t;
+
+typedef struct crypto_load_dev_disabled {
+ uint_t dd_return_value;
+ char dd_dev_name[MAXNAMELEN];
+ uint_t dd_dev_instance;
+ uint_t dd_count;
+ crypto_mech_name_t dd_list[1];
+} crypto_load_dev_disabled_t;
+
+typedef struct crypto_load_soft_disabled {
+ uint_t sd_return_value;
+ char sd_name[MAXNAMELEN];
+ uint_t sd_count;
+ crypto_mech_name_t sd_list[1];
+} crypto_load_soft_disabled_t;
+
+typedef struct crypto_unload_soft_module {
+ uint_t sm_return_value;
+ char sm_name[MAXNAMELEN];
+} crypto_unload_soft_module_t;
+
+typedef struct crypto_load_soft_config {
+ uint_t sc_return_value;
+ char sc_name[MAXNAMELEN];
+ uint_t sc_count;
+ crypto_mech_name_t sc_list[1];
+} crypto_load_soft_config_t;
+
+typedef struct crypto_load_door {
+ uint_t ld_return_value;
+ uint_t ld_did;
+} crypto_load_door_t;
+
+#ifdef _KERNEL
+#ifdef _SYSCALL32
+
+typedef struct crypto_get_soft_list32 {
+ uint32_t sl_return_value;
+ uint32_t sl_soft_count;
+ size32_t sl_soft_len;
+ caddr32_t sl_soft_names;
+} crypto_get_soft_list32_t;
+
+#endif /* _SYSCALL32 */
+#endif /* _KERNEL */
+
+#define CRYPTO_GET_VERSION CRYPTOADMIN(1)
+#define CRYPTO_GET_DEV_LIST CRYPTOADMIN(2)
+#define CRYPTO_GET_SOFT_LIST CRYPTOADMIN(3)
+#define CRYPTO_GET_DEV_INFO CRYPTOADMIN(4)
+#define CRYPTO_GET_SOFT_INFO CRYPTOADMIN(5)
+#define CRYPTO_LOAD_DEV_DISABLED CRYPTOADMIN(8)
+#define CRYPTO_LOAD_SOFT_DISABLED CRYPTOADMIN(9)
+#define CRYPTO_UNLOAD_SOFT_MODULE CRYPTOADMIN(10)
+#define CRYPTO_LOAD_SOFT_CONFIG CRYPTOADMIN(11)
+#define CRYPTO_POOL_CREATE CRYPTOADMIN(12)
+#define CRYPTO_POOL_WAIT CRYPTOADMIN(13)
+#define CRYPTO_POOL_RUN CRYPTOADMIN(14)
+#define CRYPTO_LOAD_DOOR CRYPTOADMIN(15)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_CRYPTO_IOCTLADMIN_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_CRYPTO_OPS_IMPL_H
+#define _SYS_CRYPTO_OPS_IMPL_H
+
+/*
+ * Scheduler internal structures.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/spi.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/common.h>
+
+/*
+ * The parameters needed for each function group are batched
+ * in one structure. This is much simpler than having a
+ * separate structure for each function.
+ *
+ * In some cases, a field is generically named to keep the
+ * structure small. The comments indicate these cases.
+ */
+typedef struct kcf_digest_ops_params {
+ crypto_session_id_t do_sid;
+ crypto_mech_type_t do_framework_mechtype;
+ crypto_mechanism_t do_mech;
+ crypto_data_t *do_data;
+ crypto_data_t *do_digest;
+ crypto_key_t *do_digest_key; /* Argument for digest_key() */
+} kcf_digest_ops_params_t;
+
+typedef struct kcf_mac_ops_params {
+ crypto_session_id_t mo_sid;
+ crypto_mech_type_t mo_framework_mechtype;
+ crypto_mechanism_t mo_mech;
+ crypto_key_t *mo_key;
+ crypto_data_t *mo_data;
+ crypto_data_t *mo_mac;
+ crypto_spi_ctx_template_t mo_templ;
+} kcf_mac_ops_params_t;
+
+typedef struct kcf_encrypt_ops_params {
+ crypto_session_id_t eo_sid;
+ crypto_mech_type_t eo_framework_mechtype;
+ crypto_mechanism_t eo_mech;
+ crypto_key_t *eo_key;
+ crypto_data_t *eo_plaintext;
+ crypto_data_t *eo_ciphertext;
+ crypto_spi_ctx_template_t eo_templ;
+} kcf_encrypt_ops_params_t;
+
+typedef struct kcf_decrypt_ops_params {
+ crypto_session_id_t dop_sid;
+ crypto_mech_type_t dop_framework_mechtype;
+ crypto_mechanism_t dop_mech;
+ crypto_key_t *dop_key;
+ crypto_data_t *dop_ciphertext;
+ crypto_data_t *dop_plaintext;
+ crypto_spi_ctx_template_t dop_templ;
+} kcf_decrypt_ops_params_t;
+
+typedef struct kcf_sign_ops_params {
+ crypto_session_id_t so_sid;
+ crypto_mech_type_t so_framework_mechtype;
+ crypto_mechanism_t so_mech;
+ crypto_key_t *so_key;
+ crypto_data_t *so_data;
+ crypto_data_t *so_signature;
+ crypto_spi_ctx_template_t so_templ;
+} kcf_sign_ops_params_t;
+
+typedef struct kcf_verify_ops_params {
+ crypto_session_id_t vo_sid;
+ crypto_mech_type_t vo_framework_mechtype;
+ crypto_mechanism_t vo_mech;
+ crypto_key_t *vo_key;
+ crypto_data_t *vo_data;
+ crypto_data_t *vo_signature;
+ crypto_spi_ctx_template_t vo_templ;
+} kcf_verify_ops_params_t;
+
+typedef struct kcf_encrypt_mac_ops_params {
+ crypto_session_id_t em_sid;
+ crypto_mech_type_t em_framework_encr_mechtype;
+ crypto_mechanism_t em_encr_mech;
+ crypto_key_t *em_encr_key;
+ crypto_mech_type_t em_framework_mac_mechtype;
+ crypto_mechanism_t em_mac_mech;
+ crypto_key_t *em_mac_key;
+ crypto_data_t *em_plaintext;
+ crypto_dual_data_t *em_ciphertext;
+ crypto_data_t *em_mac;
+ crypto_spi_ctx_template_t em_encr_templ;
+ crypto_spi_ctx_template_t em_mac_templ;
+} kcf_encrypt_mac_ops_params_t;
+
+typedef struct kcf_mac_decrypt_ops_params {
+ crypto_session_id_t md_sid;
+ crypto_mech_type_t md_framework_mac_mechtype;
+ crypto_mechanism_t md_mac_mech;
+ crypto_key_t *md_mac_key;
+ crypto_mech_type_t md_framework_decr_mechtype;
+ crypto_mechanism_t md_decr_mech;
+ crypto_key_t *md_decr_key;
+ crypto_dual_data_t *md_ciphertext;
+ crypto_data_t *md_mac;
+ crypto_data_t *md_plaintext;
+ crypto_spi_ctx_template_t md_mac_templ;
+ crypto_spi_ctx_template_t md_decr_templ;
+} kcf_mac_decrypt_ops_params_t;
+
+typedef struct kcf_random_number_ops_params {
+ crypto_session_id_t rn_sid;
+ uchar_t *rn_buf;
+ size_t rn_buflen;
+ uint_t rn_entropy_est;
+ uint32_t rn_flags;
+} kcf_random_number_ops_params_t;
+
+/*
+ * so_pd is useful when the provider descriptor (pd) supplying the
+ * provider handle is different from the pd supplying the ops vector.
+ * This is the case for session open/close where so_pd can be the pd
+ * of a logical provider. The pd supplying the ops vector is passed
+ * as an argument to kcf_submit_request().
+ */
+typedef struct kcf_session_ops_params {
+ crypto_session_id_t *so_sid_ptr;
+ crypto_session_id_t so_sid;
+ crypto_user_type_t so_user_type;
+ char *so_pin;
+ size_t so_pin_len;
+ kcf_provider_desc_t *so_pd;
+} kcf_session_ops_params_t;
+
+typedef struct kcf_object_ops_params {
+ crypto_session_id_t oo_sid;
+ crypto_object_id_t oo_object_id;
+ crypto_object_attribute_t *oo_template;
+ uint_t oo_attribute_count;
+ crypto_object_id_t *oo_object_id_ptr;
+ size_t *oo_object_size;
+ void **oo_find_init_pp_ptr;
+ void *oo_find_pp;
+ uint_t oo_max_object_count;
+ uint_t *oo_object_count_ptr;
+} kcf_object_ops_params_t;
+
+/*
+ * ko_key is used to encode wrapping key in key_wrap() and
+ * unwrapping key in key_unwrap(). ko_key_template and
+ * ko_key_attribute_count are used to encode public template
+ * and public template attr count in key_generate_pair().
+ * kops->ko_key_object_id_ptr is used to encode public key
+ * in key_generate_pair().
+ */
+typedef struct kcf_key_ops_params {
+ crypto_session_id_t ko_sid;
+ crypto_mech_type_t ko_framework_mechtype;
+ crypto_mechanism_t ko_mech;
+ crypto_object_attribute_t *ko_key_template;
+ uint_t ko_key_attribute_count;
+ crypto_object_id_t *ko_key_object_id_ptr;
+ crypto_object_attribute_t *ko_private_key_template;
+ uint_t ko_private_key_attribute_count;
+ crypto_object_id_t *ko_private_key_object_id_ptr;
+ crypto_key_t *ko_key;
+ uchar_t *ko_wrapped_key;
+ size_t *ko_wrapped_key_len_ptr;
+ crypto_object_attribute_t *ko_out_template1;
+ crypto_object_attribute_t *ko_out_template2;
+ uint_t ko_out_attribute_count1;
+ uint_t ko_out_attribute_count2;
+} kcf_key_ops_params_t;
+
+/*
+ * po_pin and po_pin_len are used to encode new_pin and new_pin_len
+ * when wrapping set_pin() function parameters.
+ *
+ * po_pd is useful when the provider descriptor (pd) supplying the
+ * provider handle is different from the pd supplying the ops vector.
+ * This is true for the ext_info provider entry point where po_pd
+ * can be the pd of a logical provider. The pd supplying the ops vector
+ * is passed as an argument to kcf_submit_request().
+ */
+typedef struct kcf_provmgmt_ops_params {
+ crypto_session_id_t po_sid;
+ char *po_pin;
+ size_t po_pin_len;
+ char *po_old_pin;
+ size_t po_old_pin_len;
+ char *po_label;
+ crypto_provider_ext_info_t *po_ext_info;
+ kcf_provider_desc_t *po_pd;
+} kcf_provmgmt_ops_params_t;
+
+/*
+ * The operation type within a function group.
+ */
+typedef enum kcf_op_type {
+ /* common ops for all mechanisms */
+ KCF_OP_INIT = 1,
+ KCF_OP_SINGLE, /* pkcs11 sense. So, INIT is already done */
+ KCF_OP_UPDATE,
+ KCF_OP_FINAL,
+ KCF_OP_ATOMIC,
+
+ /* digest_key op */
+ KCF_OP_DIGEST_KEY,
+
+ /* mac specific op */
+ KCF_OP_MAC_VERIFY_ATOMIC,
+
+ /* mac/cipher specific op */
+ KCF_OP_MAC_VERIFY_DECRYPT_ATOMIC,
+
+ /* sign_recover ops */
+ KCF_OP_SIGN_RECOVER_INIT,
+ KCF_OP_SIGN_RECOVER,
+ KCF_OP_SIGN_RECOVER_ATOMIC,
+
+ /* verify_recover ops */
+ KCF_OP_VERIFY_RECOVER_INIT,
+ KCF_OP_VERIFY_RECOVER,
+ KCF_OP_VERIFY_RECOVER_ATOMIC,
+
+ /* random number ops */
+ KCF_OP_RANDOM_SEED,
+ KCF_OP_RANDOM_GENERATE,
+
+ /* session management ops */
+ KCF_OP_SESSION_OPEN,
+ KCF_OP_SESSION_CLOSE,
+ KCF_OP_SESSION_LOGIN,
+ KCF_OP_SESSION_LOGOUT,
+
+ /* object management ops */
+ KCF_OP_OBJECT_CREATE,
+ KCF_OP_OBJECT_COPY,
+ KCF_OP_OBJECT_DESTROY,
+ KCF_OP_OBJECT_GET_SIZE,
+ KCF_OP_OBJECT_GET_ATTRIBUTE_VALUE,
+ KCF_OP_OBJECT_SET_ATTRIBUTE_VALUE,
+ KCF_OP_OBJECT_FIND_INIT,
+ KCF_OP_OBJECT_FIND,
+ KCF_OP_OBJECT_FIND_FINAL,
+
+ /* key management ops */
+ KCF_OP_KEY_GENERATE,
+ KCF_OP_KEY_GENERATE_PAIR,
+ KCF_OP_KEY_WRAP,
+ KCF_OP_KEY_UNWRAP,
+ KCF_OP_KEY_DERIVE,
+ KCF_OP_KEY_CHECK,
+
+ /* provider management ops */
+ KCF_OP_MGMT_EXTINFO,
+ KCF_OP_MGMT_INITTOKEN,
+ KCF_OP_MGMT_INITPIN,
+ KCF_OP_MGMT_SETPIN
+} kcf_op_type_t;
+
+/*
+ * The operation groups that need wrapping of parameters. This is somewhat
+ * similar to the function group type in spi.h except that this also includes
+ * all the functions that don't have a mechanism.
+ *
+ * The wrapper macros should never take these enum values as an argument.
+ * Rather, they are assigned in the macro itself since they are known
+ * from the macro name.
+ */
+typedef enum kcf_op_group {
+ KCF_OG_DIGEST = 1,
+ KCF_OG_MAC,
+ KCF_OG_ENCRYPT,
+ KCF_OG_DECRYPT,
+ KCF_OG_SIGN,
+ KCF_OG_VERIFY,
+ KCF_OG_ENCRYPT_MAC,
+ KCF_OG_MAC_DECRYPT,
+ KCF_OG_RANDOM,
+ KCF_OG_SESSION,
+ KCF_OG_OBJECT,
+ KCF_OG_KEY,
+ KCF_OG_PROVMGMT,
+ KCF_OG_NOSTORE_KEY
+} kcf_op_group_t;
+
+/*
+ * The kcf_op_type_t enum values used here should be only for those
+ * operations for which there is a k-api routine in sys/crypto/api.h.
+ */
+#define IS_INIT_OP(ftype) ((ftype) == KCF_OP_INIT)
+#define IS_SINGLE_OP(ftype) ((ftype) == KCF_OP_SINGLE)
+#define IS_UPDATE_OP(ftype) ((ftype) == KCF_OP_UPDATE)
+#define IS_FINAL_OP(ftype) ((ftype) == KCF_OP_FINAL)
+#define IS_ATOMIC_OP(ftype) ( \
+ (ftype) == KCF_OP_ATOMIC || (ftype) == KCF_OP_MAC_VERIFY_ATOMIC || \
+ (ftype) == KCF_OP_MAC_VERIFY_DECRYPT_ATOMIC || \
+ (ftype) == KCF_OP_SIGN_RECOVER_ATOMIC || \
+ (ftype) == KCF_OP_VERIFY_RECOVER_ATOMIC)
+
+/*
+ * Keep the parameters associated with a request around.
+ * We need to pass them to the SPI.
+ */
+typedef struct kcf_req_params {
+ kcf_op_group_t rp_opgrp;
+ kcf_op_type_t rp_optype;
+
+ union {
+ kcf_digest_ops_params_t digest_params;
+ kcf_mac_ops_params_t mac_params;
+ kcf_encrypt_ops_params_t encrypt_params;
+ kcf_decrypt_ops_params_t decrypt_params;
+ kcf_sign_ops_params_t sign_params;
+ kcf_verify_ops_params_t verify_params;
+ kcf_encrypt_mac_ops_params_t encrypt_mac_params;
+ kcf_mac_decrypt_ops_params_t mac_decrypt_params;
+ kcf_random_number_ops_params_t random_number_params;
+ kcf_session_ops_params_t session_params;
+ kcf_object_ops_params_t object_params;
+ kcf_key_ops_params_t key_params;
+ kcf_provmgmt_ops_params_t provmgmt_params;
+ } rp_u;
+} kcf_req_params_t;
+
+
+/*
+ * The ioctl/k-api code should bundle the parameters into a kcf_req_params_t
+ * structure before calling a scheduler routine. The following macros are
+ * available for that purpose.
+ *
+ * For the most part, the macro arguments closely correspond to the
+ * function parameters. In some cases, we use generic names. The comments
+ * for the structure should indicate these cases.
+ */
+#define KCF_WRAP_DIGEST_OPS_PARAMS(req, ftype, _sid, _mech, _key, \
+ _data, _digest) { \
+ kcf_digest_ops_params_t *dops = &(req)->rp_u.digest_params; \
+ crypto_mechanism_t *mechp = _mech; \
+ \
+ (req)->rp_opgrp = KCF_OG_DIGEST; \
+ (req)->rp_optype = ftype; \
+ dops->do_sid = _sid; \
+ if (mechp != NULL) { \
+ dops->do_mech = *mechp; \
+ dops->do_framework_mechtype = mechp->cm_type; \
+ } \
+ dops->do_digest_key = _key; \
+ dops->do_data = _data; \
+ dops->do_digest = _digest; \
+}
+
+#define KCF_WRAP_MAC_OPS_PARAMS(req, ftype, _sid, _mech, _key, \
+ _data, _mac, _templ) { \
+ kcf_mac_ops_params_t *mops = &(req)->rp_u.mac_params; \
+ crypto_mechanism_t *mechp = _mech; \
+ \
+ (req)->rp_opgrp = KCF_OG_MAC; \
+ (req)->rp_optype = ftype; \
+ mops->mo_sid = _sid; \
+ if (mechp != NULL) { \
+ mops->mo_mech = *mechp; \
+ mops->mo_framework_mechtype = mechp->cm_type; \
+ } \
+ mops->mo_key = _key; \
+ mops->mo_data = _data; \
+ mops->mo_mac = _mac; \
+ mops->mo_templ = _templ; \
+}
+
+#define KCF_WRAP_ENCRYPT_OPS_PARAMS(req, ftype, _sid, _mech, _key, \
+ _plaintext, _ciphertext, _templ) { \
+ kcf_encrypt_ops_params_t *cops = &(req)->rp_u.encrypt_params; \
+ crypto_mechanism_t *mechp = _mech; \
+ \
+ (req)->rp_opgrp = KCF_OG_ENCRYPT; \
+ (req)->rp_optype = ftype; \
+ cops->eo_sid = _sid; \
+ if (mechp != NULL) { \
+ cops->eo_mech = *mechp; \
+ cops->eo_framework_mechtype = mechp->cm_type; \
+ } \
+ cops->eo_key = _key; \
+ cops->eo_plaintext = _plaintext; \
+ cops->eo_ciphertext = _ciphertext; \
+ cops->eo_templ = _templ; \
+}
+
+#define KCF_WRAP_DECRYPT_OPS_PARAMS(req, ftype, _sid, _mech, _key, \
+ _ciphertext, _plaintext, _templ) { \
+ kcf_decrypt_ops_params_t *cops = &(req)->rp_u.decrypt_params; \
+ crypto_mechanism_t *mechp = _mech; \
+ \
+ (req)->rp_opgrp = KCF_OG_DECRYPT; \
+ (req)->rp_optype = ftype; \
+ cops->dop_sid = _sid; \
+ if (mechp != NULL) { \
+ cops->dop_mech = *mechp; \
+ cops->dop_framework_mechtype = mechp->cm_type; \
+ } \
+ cops->dop_key = _key; \
+ cops->dop_ciphertext = _ciphertext; \
+ cops->dop_plaintext = _plaintext; \
+ cops->dop_templ = _templ; \
+}
+
+#define KCF_WRAP_SIGN_OPS_PARAMS(req, ftype, _sid, _mech, _key, \
+ _data, _signature, _templ) { \
+ kcf_sign_ops_params_t *sops = &(req)->rp_u.sign_params; \
+ crypto_mechanism_t *mechp = _mech; \
+ \
+ (req)->rp_opgrp = KCF_OG_SIGN; \
+ (req)->rp_optype = ftype; \
+ sops->so_sid = _sid; \
+ if (mechp != NULL) { \
+ sops->so_mech = *mechp; \
+ sops->so_framework_mechtype = mechp->cm_type; \
+ } \
+ sops->so_key = _key; \
+ sops->so_data = _data; \
+ sops->so_signature = _signature; \
+ sops->so_templ = _templ; \
+}
+
+#define KCF_WRAP_VERIFY_OPS_PARAMS(req, ftype, _sid, _mech, _key, \
+ _data, _signature, _templ) { \
+ kcf_verify_ops_params_t *vops = &(req)->rp_u.verify_params; \
+ crypto_mechanism_t *mechp = _mech; \
+ \
+ (req)->rp_opgrp = KCF_OG_VERIFY; \
+ (req)->rp_optype = ftype; \
+ vops->vo_sid = _sid; \
+ if (mechp != NULL) { \
+ vops->vo_mech = *mechp; \
+ vops->vo_framework_mechtype = mechp->cm_type; \
+ } \
+ vops->vo_key = _key; \
+ vops->vo_data = _data; \
+ vops->vo_signature = _signature; \
+ vops->vo_templ = _templ; \
+}
+
+#define KCF_WRAP_ENCRYPT_MAC_OPS_PARAMS(req, ftype, _sid, _encr_key, \
+ _mac_key, _plaintext, _ciphertext, _mac, _encr_templ, _mac_templ) { \
+ kcf_encrypt_mac_ops_params_t *cmops = &(req)->rp_u.encrypt_mac_params; \
+ \
+ (req)->rp_opgrp = KCF_OG_ENCRYPT_MAC; \
+ (req)->rp_optype = ftype; \
+ cmops->em_sid = _sid; \
+ cmops->em_encr_key = _encr_key; \
+ cmops->em_mac_key = _mac_key; \
+ cmops->em_plaintext = _plaintext; \
+ cmops->em_ciphertext = _ciphertext; \
+ cmops->em_mac = _mac; \
+ cmops->em_encr_templ = _encr_templ; \
+ cmops->em_mac_templ = _mac_templ; \
+}
+
+#define KCF_WRAP_MAC_DECRYPT_OPS_PARAMS(req, ftype, _sid, _mac_key, \
+ _decr_key, _ciphertext, _mac, _plaintext, _mac_templ, _decr_templ) { \
+ kcf_mac_decrypt_ops_params_t *cmops = &(req)->rp_u.mac_decrypt_params; \
+ \
+ (req)->rp_opgrp = KCF_OG_MAC_DECRYPT; \
+ (req)->rp_optype = ftype; \
+ cmops->md_sid = _sid; \
+ cmops->md_mac_key = _mac_key; \
+ cmops->md_decr_key = _decr_key; \
+ cmops->md_ciphertext = _ciphertext; \
+ cmops->md_mac = _mac; \
+ cmops->md_plaintext = _plaintext; \
+ cmops->md_mac_templ = _mac_templ; \
+ cmops->md_decr_templ = _decr_templ; \
+}
+
+#define KCF_WRAP_RANDOM_OPS_PARAMS(req, ftype, _sid, _buf, _buflen, \
+ _est, _flags) { \
+ kcf_random_number_ops_params_t *rops = \
+ &(req)->rp_u.random_number_params; \
+ \
+ (req)->rp_opgrp = KCF_OG_RANDOM; \
+ (req)->rp_optype = ftype; \
+ rops->rn_sid = _sid; \
+ rops->rn_buf = _buf; \
+ rops->rn_buflen = _buflen; \
+ rops->rn_entropy_est = _est; \
+ rops->rn_flags = _flags; \
+}
+
+#define KCF_WRAP_SESSION_OPS_PARAMS(req, ftype, _sid_ptr, _sid, \
+ _user_type, _pin, _pin_len, _pd) { \
+ kcf_session_ops_params_t *sops = &(req)->rp_u.session_params; \
+ \
+ (req)->rp_opgrp = KCF_OG_SESSION; \
+ (req)->rp_optype = ftype; \
+ sops->so_sid_ptr = _sid_ptr; \
+ sops->so_sid = _sid; \
+ sops->so_user_type = _user_type; \
+ sops->so_pin = _pin; \
+ sops->so_pin_len = _pin_len; \
+ sops->so_pd = _pd; \
+}
+
+#define KCF_WRAP_OBJECT_OPS_PARAMS(req, ftype, _sid, _object_id, \
+ _template, _attribute_count, _object_id_ptr, _object_size, \
+ _find_init_pp_ptr, _find_pp, _max_object_count, _object_count_ptr) { \
+ kcf_object_ops_params_t *jops = &(req)->rp_u.object_params; \
+ \
+ (req)->rp_opgrp = KCF_OG_OBJECT; \
+ (req)->rp_optype = ftype; \
+ jops->oo_sid = _sid; \
+ jops->oo_object_id = _object_id; \
+ jops->oo_template = _template; \
+ jops->oo_attribute_count = _attribute_count; \
+ jops->oo_object_id_ptr = _object_id_ptr; \
+ jops->oo_object_size = _object_size; \
+ jops->oo_find_init_pp_ptr = _find_init_pp_ptr; \
+ jops->oo_find_pp = _find_pp; \
+ jops->oo_max_object_count = _max_object_count; \
+ jops->oo_object_count_ptr = _object_count_ptr; \
+}
+
+#define KCF_WRAP_KEY_OPS_PARAMS(req, ftype, _sid, _mech, _key_template, \
+ _key_attribute_count, _key_object_id_ptr, _private_key_template, \
+ _private_key_attribute_count, _private_key_object_id_ptr, \
+ _key, _wrapped_key, _wrapped_key_len_ptr) { \
+ kcf_key_ops_params_t *kops = &(req)->rp_u.key_params; \
+ crypto_mechanism_t *mechp = _mech; \
+ \
+ (req)->rp_opgrp = KCF_OG_KEY; \
+ (req)->rp_optype = ftype; \
+ kops->ko_sid = _sid; \
+ if (mechp != NULL) { \
+ kops->ko_mech = *mechp; \
+ kops->ko_framework_mechtype = mechp->cm_type; \
+ } \
+ kops->ko_key_template = _key_template; \
+ kops->ko_key_attribute_count = _key_attribute_count; \
+ kops->ko_key_object_id_ptr = _key_object_id_ptr; \
+ kops->ko_private_key_template = _private_key_template; \
+ kops->ko_private_key_attribute_count = _private_key_attribute_count; \
+ kops->ko_private_key_object_id_ptr = _private_key_object_id_ptr; \
+ kops->ko_key = _key; \
+ kops->ko_wrapped_key = _wrapped_key; \
+ kops->ko_wrapped_key_len_ptr = _wrapped_key_len_ptr; \
+}
+
+#define KCF_WRAP_PROVMGMT_OPS_PARAMS(req, ftype, _sid, _old_pin, \
+ _old_pin_len, _pin, _pin_len, _label, _ext_info, _pd) { \
+ kcf_provmgmt_ops_params_t *pops = &(req)->rp_u.provmgmt_params; \
+ \
+ (req)->rp_opgrp = KCF_OG_PROVMGMT; \
+ (req)->rp_optype = ftype; \
+ pops->po_sid = _sid; \
+ pops->po_pin = _pin; \
+ pops->po_pin_len = _pin_len; \
+ pops->po_old_pin = _old_pin; \
+ pops->po_old_pin_len = _old_pin_len; \
+ pops->po_label = _label; \
+ pops->po_ext_info = _ext_info; \
+ pops->po_pd = _pd; \
+}
+
+#define KCF_WRAP_NOSTORE_KEY_OPS_PARAMS(req, ftype, _sid, _mech, \
+ _key_template, _key_attribute_count, _private_key_template, \
+ _private_key_attribute_count, _key, _out_template1, \
+ _out_attribute_count1, _out_template2, _out_attribute_count2) { \
+ kcf_key_ops_params_t *kops = &(req)->rp_u.key_params; \
+ crypto_mechanism_t *mechp = _mech; \
+ \
+ (req)->rp_opgrp = KCF_OG_NOSTORE_KEY; \
+ (req)->rp_optype = ftype; \
+ kops->ko_sid = _sid; \
+ if (mechp != NULL) { \
+ kops->ko_mech = *mechp; \
+ kops->ko_framework_mechtype = mechp->cm_type; \
+ } \
+ kops->ko_key_template = _key_template; \
+ kops->ko_key_attribute_count = _key_attribute_count; \
+ kops->ko_key_object_id_ptr = NULL; \
+ kops->ko_private_key_template = _private_key_template; \
+ kops->ko_private_key_attribute_count = _private_key_attribute_count; \
+ kops->ko_private_key_object_id_ptr = NULL; \
+ kops->ko_key = _key; \
+ kops->ko_wrapped_key = NULL; \
+ kops->ko_wrapped_key_len_ptr = 0; \
+ kops->ko_out_template1 = _out_template1; \
+ kops->ko_out_template2 = _out_template2; \
+ kops->ko_out_attribute_count1 = _out_attribute_count1; \
+ kops->ko_out_attribute_count2 = _out_attribute_count2; \
+}
+
+#define KCF_SET_PROVIDER_MECHNUM(fmtype, pd, mechp) \
+ (mechp)->cm_type = \
+ KCF_TO_PROV_MECHNUM(pd, fmtype);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_CRYPTO_OPS_IMPL_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_CRYPTO_SCHED_IMPL_H
+#define _SYS_CRYPTO_SCHED_IMPL_H
+
+/*
+ * Scheduler internal structures.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/api.h>
+#include <sys/crypto/spi.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/ops_impl.h>
+
+typedef void (kcf_func_t)(void *, int);
+
+typedef enum kcf_req_status {
+ REQ_ALLOCATED = 1,
+ REQ_WAITING, /* At the framework level */
+ REQ_INPROGRESS, /* At the provider level */
+ REQ_DONE,
+ REQ_CANCELED
+} kcf_req_status_t;
+
+typedef enum kcf_call_type {
+ CRYPTO_SYNCH = 1,
+ CRYPTO_ASYNCH
+} kcf_call_type_t;
+
+#define CHECK_RESTRICT(crq) (crq != NULL && \
+ ((crq)->cr_flag & CRYPTO_RESTRICTED))
+
+#define CHECK_RESTRICT_FALSE B_FALSE
+
+#define CHECK_FASTPATH(crq, pd) ((crq) == NULL || \
+ !((crq)->cr_flag & CRYPTO_ALWAYS_QUEUE)) && \
+ (pd)->pd_prov_type == CRYPTO_SW_PROVIDER
+
+#define KCF_KMFLAG(crq) (((crq) == NULL) ? KM_SLEEP : KM_NOSLEEP)
+
+/*
+ * The framework keeps an internal handle to use in the adaptive
+ * asynchronous case. This is the case when a client has the
+ * CRYPTO_ALWAYS_QUEUE bit clear and a software provider is used for
+ * the request. The request is completed in the context of the calling
+ * thread and kernel memory must be allocated with KM_NOSLEEP.
+ *
+ * The framework passes a pointer to the handle in crypto_req_handle_t
+ * argument when it calls the SPI of the software provider. The macros
+ * KCF_RHNDL() and KCF_SWFP_RHNDL() are used to do this.
+ *
+ * When a provider asks the framework for kmflag value via
+ * crypto_kmflag(9S) we use REQHNDL2_KMFLAG() macro.
+ */
+extern ulong_t kcf_swprov_hndl;
+#define KCF_RHNDL(kmflag) (((kmflag) == KM_SLEEP) ? NULL : &kcf_swprov_hndl)
+#define KCF_SWFP_RHNDL(crq) (((crq) == NULL) ? NULL : &kcf_swprov_hndl)
+#define REQHNDL2_KMFLAG(rhndl) \
+ ((rhndl == &kcf_swprov_hndl) ? KM_NOSLEEP : KM_SLEEP)
+
+/* Internal call_req flags. They start after the public ones in api.h */
+
+#define CRYPTO_SETDUAL 0x00001000 /* Set the 'cont' boolean before */
+ /* submitting the request */
+#define KCF_ISDUALREQ(crq) \
+ (((crq) == NULL) ? B_FALSE : (crq->cr_flag & CRYPTO_SETDUAL))
+
+typedef struct kcf_prov_tried {
+ kcf_provider_desc_t *pt_pd;
+ struct kcf_prov_tried *pt_next;
+} kcf_prov_tried_t;
+
+#define IS_FG_SUPPORTED(mdesc, fg) \
+ (((mdesc)->pm_mech_info.cm_func_group_mask & (fg)) != 0)
+
+#define IS_PROVIDER_TRIED(pd, tlist) \
+ (tlist != NULL && is_in_triedlist(pd, tlist))
+
+#define IS_RECOVERABLE(error) \
+ (error == CRYPTO_BUFFER_TOO_BIG || \
+ error == CRYPTO_BUSY || \
+ error == CRYPTO_DEVICE_ERROR || \
+ error == CRYPTO_DEVICE_MEMORY || \
+ error == CRYPTO_KEY_SIZE_RANGE || \
+ error == CRYPTO_NO_PERMISSION)
+
+#define KCF_ATOMIC_INCR(x) atomic_add_32(&(x), 1)
+#define KCF_ATOMIC_DECR(x) atomic_add_32(&(x), -1)
+
+/*
+ * Node structure for synchronous requests.
+ */
+typedef struct kcf_sreq_node {
+ /* Should always be the first field in this structure */
+ kcf_call_type_t sn_type;
+ /*
+ * sn_cv and sr_lock are used to wait for the
+ * operation to complete. sn_lock also protects
+ * the sn_state field.
+ */
+ kcondvar_t sn_cv;
+ kmutex_t sn_lock;
+ kcf_req_status_t sn_state;
+
+ /*
+ * Return value from the operation. This will be
+ * one of the CRYPTO_* errors defined in common.h.
+ */
+ int sn_rv;
+
+ /*
+ * parameters to call the SPI with. This can be
+ * a pointer as we know the caller context/stack stays.
+ */
+ struct kcf_req_params *sn_params;
+
+ /* Internal context for this request */
+ struct kcf_context *sn_context;
+
+ /* Provider handling this request */
+ kcf_provider_desc_t *sn_provider;
+} kcf_sreq_node_t;
+
+/*
+ * Node structure for asynchronous requests. A node can be on
+ * on a chain of requests hanging of the internal context
+ * structure and can be in the global software provider queue.
+ */
+typedef struct kcf_areq_node {
+ /* Should always be the first field in this structure */
+ kcf_call_type_t an_type;
+
+ /* an_lock protects the field an_state */
+ kmutex_t an_lock;
+ kcf_req_status_t an_state;
+ crypto_call_req_t an_reqarg;
+
+ /*
+ * parameters to call the SPI with. We need to
+ * save the params since the caller stack can go away.
+ */
+ struct kcf_req_params an_params;
+
+ /*
+ * The next two fields should be NULL for operations that
+ * don't need a context.
+ */
+ /* Internal context for this request */
+ struct kcf_context *an_context;
+
+ /* next in chain of requests for context */
+ struct kcf_areq_node *an_ctxchain_next;
+
+ kcondvar_t an_turn_cv;
+ boolean_t an_is_my_turn;
+ boolean_t an_isdual; /* for internal reuse */
+
+ /*
+ * Next and previous nodes in the global software
+ * queue. These fields are NULL for a hardware
+ * provider since we use a taskq there.
+ */
+ struct kcf_areq_node *an_next;
+ struct kcf_areq_node *an_prev;
+
+ /* Provider handling this request */
+ kcf_provider_desc_t *an_provider;
+ kcf_prov_tried_t *an_tried_plist;
+
+ struct kcf_areq_node *an_idnext; /* Next in ID hash */
+ struct kcf_areq_node *an_idprev; /* Prev in ID hash */
+ kcondvar_t an_done; /* Signal request completion */
+ uint_t an_refcnt;
+} kcf_areq_node_t;
+
+#define KCF_AREQ_REFHOLD(areq) { \
+ atomic_add_32(&(areq)->an_refcnt, 1); \
+ ASSERT((areq)->an_refcnt != 0); \
+}
+
+#define KCF_AREQ_REFRELE(areq) { \
+ ASSERT((areq)->an_refcnt != 0); \
+ membar_exit(); \
+ if (atomic_add_32_nv(&(areq)->an_refcnt, -1) == 0) \
+ kcf_free_req(areq); \
+}
+
+#define GET_REQ_TYPE(arg) *((kcf_call_type_t *)(arg))
+
+#define NOTIFY_CLIENT(areq, err) (*(areq)->an_reqarg.cr_callback_func)(\
+ (areq)->an_reqarg.cr_callback_arg, err);
+
+/* For internally generated call requests for dual operations */
+typedef struct kcf_call_req {
+ crypto_call_req_t kr_callreq; /* external client call req */
+ kcf_req_params_t kr_params; /* Params saved for next call */
+ kcf_areq_node_t *kr_areq; /* Use this areq */
+ off_t kr_saveoffset;
+ size_t kr_savelen;
+} kcf_dual_req_t;
+
+/*
+ * The following are some what similar to macros in callo.h, which implement
+ * callout tables.
+ *
+ * The lower four bits of the ID are used to encode the table ID to
+ * index in to. The REQID_COUNTER_HIGH bit is used to avoid any check for
+ * wrap around when generating ID. We assume that there won't be a request
+ * which takes more time than 2^^(sizeof (long) - 5) other requests submitted
+ * after it. This ensures there won't be any ID collision.
+ */
+#define REQID_COUNTER_HIGH (1UL << (8 * sizeof (long) - 1))
+#define REQID_COUNTER_SHIFT 4
+#define REQID_COUNTER_LOW (1 << REQID_COUNTER_SHIFT)
+#define REQID_TABLES 16
+#define REQID_TABLE_MASK (REQID_TABLES - 1)
+
+#define REQID_BUCKETS 512
+#define REQID_BUCKET_MASK (REQID_BUCKETS - 1)
+#define REQID_HASH(id) (((id) >> REQID_COUNTER_SHIFT) & REQID_BUCKET_MASK)
+
+#define GET_REQID(areq) (areq)->an_reqarg.cr_reqid
+#define SET_REQID(areq, val) GET_REQID(areq) = val
+
+/*
+ * Hash table for async requests.
+ */
+typedef struct kcf_reqid_table {
+ kmutex_t rt_lock;
+ crypto_req_id_t rt_curid;
+ kcf_areq_node_t *rt_idhash[REQID_BUCKETS];
+} kcf_reqid_table_t;
+
+/*
+ * Global software provider queue structure. Requests to be
+ * handled by a SW provider and have the ALWAYS_QUEUE flag set
+ * get queued here.
+ */
+typedef struct kcf_global_swq {
+ /*
+ * gs_cv and gs_lock are used to wait for new requests.
+ * gs_lock protects the changes to the queue.
+ */
+ kcondvar_t gs_cv;
+ kmutex_t gs_lock;
+ uint_t gs_njobs;
+ uint_t gs_maxjobs;
+ kcf_areq_node_t *gs_first;
+ kcf_areq_node_t *gs_last;
+} kcf_global_swq_t;
+
+
+/*
+ * Internal representation of a canonical context. We contain crypto_ctx_t
+ * structure in order to have just one memory allocation. The SPI
+ * ((crypto_ctx_t *)ctx)->cc_framework_private maps to this structure.
+ */
+typedef struct kcf_context {
+ crypto_ctx_t kc_glbl_ctx;
+ uint_t kc_refcnt;
+ kmutex_t kc_in_use_lock;
+ /*
+ * kc_req_chain_first and kc_req_chain_last are used to chain
+ * multiple async requests using the same context. They should be
+ * NULL for sync requests.
+ */
+ kcf_areq_node_t *kc_req_chain_first;
+ kcf_areq_node_t *kc_req_chain_last;
+ kcf_provider_desc_t *kc_prov_desc; /* Prov. descriptor */
+ kcf_provider_desc_t *kc_sw_prov_desc; /* Prov. descriptor */
+ kcf_mech_entry_t *kc_mech;
+ struct kcf_context *kc_secondctx; /* for dual contexts */
+} kcf_context_t;
+
+/*
+ * Bump up the reference count on the framework private context. A
+ * global context or a request that references this structure should
+ * do a hold.
+ */
+#define KCF_CONTEXT_REFHOLD(ictx) { \
+ atomic_add_32(&(ictx)->kc_refcnt, 1); \
+ ASSERT((ictx)->kc_refcnt != 0); \
+}
+
+/*
+ * Decrement the reference count on the framework private context.
+ * When the last reference is released, the framework private
+ * context structure is freed along with the global context.
+ */
+#define KCF_CONTEXT_REFRELE(ictx) { \
+ ASSERT((ictx)->kc_refcnt != 0); \
+ membar_exit(); \
+ if (atomic_add_32_nv(&(ictx)->kc_refcnt, -1) == 0) \
+ kcf_free_context(ictx); \
+}
+
+/*
+ * Check if we can release the context now. In case of CRYPTO_QUEUED
+ * we do not release it as we can do it only after the provider notified
+ * us. In case of CRYPTO_BUSY, the client can retry the request using
+ * the context, so we do not release the context.
+ *
+ * This macro should be called only from the final routine in
+ * an init/update/final sequence. We do not release the context in case
+ * of update operations. We require the consumer to free it
+ * explicitly, in case it wants to abandon the operation. This is done
+ * as there may be mechanisms in ECB mode that can continue even if
+ * an operation on a block fails.
+ */
+#define KCF_CONTEXT_COND_RELEASE(rv, kcf_ctx) { \
+ if (KCF_CONTEXT_DONE(rv)) \
+ KCF_CONTEXT_REFRELE(kcf_ctx); \
+}
+
+/*
+ * This macro determines whether we're done with a context.
+ */
+#define KCF_CONTEXT_DONE(rv) \
+ ((rv) != CRYPTO_QUEUED && (rv) != CRYPTO_BUSY && \
+ (rv) != CRYPTO_BUFFER_TOO_SMALL)
+
+/*
+ * A crypto_ctx_template_t is internally a pointer to this struct
+ */
+typedef struct kcf_ctx_template {
+ crypto_kcf_provider_handle_t ct_prov_handle; /* provider handle */
+ uint_t ct_generation; /* generation # */
+ size_t ct_size; /* for freeing */
+ crypto_spi_ctx_template_t ct_prov_tmpl; /* context template */
+ /* from the SW prov */
+} kcf_ctx_template_t;
+
+/*
+ * Structure for pool of threads working on global software queue.
+ */
+typedef struct kcf_pool {
+ uint32_t kp_threads; /* Number of threads in pool */
+ uint32_t kp_idlethreads; /* Idle threads in pool */
+ uint32_t kp_blockedthreads; /* Blocked threads in pool */
+
+ /*
+ * cv & lock to monitor the condition when no threads
+ * are around. In this case the failover thread kicks in.
+ */
+ kcondvar_t kp_nothr_cv;
+ kmutex_t kp_thread_lock;
+
+ /* Userspace thread creator variables. */
+ boolean_t kp_signal_create_thread; /* Create requested flag */
+ int kp_nthrs; /* # of threads to create */
+ boolean_t kp_user_waiting; /* Thread waiting for work */
+
+ /*
+ * cv & lock for the condition where more threads need to be
+ * created. kp_user_lock also protects the three fileds above.
+ */
+ kcondvar_t kp_user_cv; /* Creator cond. variable */
+ kmutex_t kp_user_lock; /* Creator lock */
+} kcf_pool_t;
+
+
+/*
+ * State of a crypto bufcall element.
+ */
+typedef enum cbuf_state {
+ CBUF_FREE = 1,
+ CBUF_WAITING,
+ CBUF_RUNNING
+} cbuf_state_t;
+
+/*
+ * Structure of a crypto bufcall element.
+ */
+typedef struct kcf_cbuf_elem {
+ /*
+ * lock and cv to wait for CBUF_RUNNING to be done
+ * kc_lock also protects kc_state.
+ */
+ kmutex_t kc_lock;
+ kcondvar_t kc_cv;
+ cbuf_state_t kc_state;
+
+ struct kcf_cbuf_elem *kc_next;
+ struct kcf_cbuf_elem *kc_prev;
+
+ void (*kc_func)(void *arg);
+ void *kc_arg;
+} kcf_cbuf_elem_t;
+
+/*
+ * State of a notify element.
+ */
+typedef enum ntfy_elem_state {
+ NTFY_WAITING = 1,
+ NTFY_RUNNING
+} ntfy_elem_state_t;
+
+/*
+ * Structure of a notify list element.
+ */
+typedef struct kcf_ntfy_elem {
+ /*
+ * lock and cv to wait for NTFY_RUNNING to be done.
+ * kn_lock also protects kn_state.
+ */
+ kmutex_t kn_lock;
+ kcondvar_t kn_cv;
+ ntfy_elem_state_t kn_state;
+
+ struct kcf_ntfy_elem *kn_next;
+ struct kcf_ntfy_elem *kn_prev;
+
+ crypto_notify_callback_t kn_func;
+ uint32_t kn_event_mask;
+} kcf_ntfy_elem_t;
+
+
+/*
+ * The following values are based on the assumption that it would
+ * take around eight cpus to load a hardware provider (This is true for
+ * at least one product) and a kernel client may come from different
+ * low-priority interrupt levels. We will have CYRPTO_TASKQ_MIN number
+ * of cached taskq entries. The CRYPTO_TASKQ_MAX number is based on
+ * a throughput of 1GB/s using 512-byte buffers. These are just
+ * reasonable estimates and might need to change in future.
+ */
+#define CRYPTO_TASKQ_THREADS 8
+#define CYRPTO_TASKQ_MIN 64
+#define CRYPTO_TASKQ_MAX 2 * 1024 * 1024
+
+extern int crypto_taskq_threads;
+extern int crypto_taskq_minalloc;
+extern int crypto_taskq_maxalloc;
+extern kcf_global_swq_t *gswq;
+extern int kcf_maxthreads;
+extern int kcf_minthreads;
+
+/*
+ * All pending crypto bufcalls are put on a list. cbuf_list_lock
+ * protects changes to this list.
+ */
+extern kmutex_t cbuf_list_lock;
+extern kcondvar_t cbuf_list_cv;
+
+/*
+ * All event subscribers are put on a list. kcf_notify_list_lock
+ * protects changes to this list.
+ */
+extern kmutex_t ntfy_list_lock;
+extern kcondvar_t ntfy_list_cv;
+
+boolean_t kcf_get_next_logical_provider_member(kcf_provider_desc_t *,
+ kcf_provider_desc_t *, kcf_provider_desc_t **);
+extern int kcf_get_hardware_provider(crypto_mech_type_t, crypto_mech_type_t,
+ boolean_t, kcf_provider_desc_t *, kcf_provider_desc_t **,
+ crypto_func_group_t);
+extern int kcf_get_hardware_provider_nomech(offset_t, offset_t,
+ boolean_t, kcf_provider_desc_t *, kcf_provider_desc_t **);
+extern void kcf_free_triedlist(kcf_prov_tried_t *);
+extern kcf_prov_tried_t *kcf_insert_triedlist(kcf_prov_tried_t **,
+ kcf_provider_desc_t *, int);
+extern kcf_provider_desc_t *kcf_get_mech_provider(crypto_mech_type_t,
+ kcf_mech_entry_t **, int *, kcf_prov_tried_t *, crypto_func_group_t,
+ boolean_t, size_t);
+extern kcf_provider_desc_t *kcf_get_dual_provider(crypto_mechanism_t *,
+ crypto_mechanism_t *, kcf_mech_entry_t **, crypto_mech_type_t *,
+ crypto_mech_type_t *, int *, kcf_prov_tried_t *,
+ crypto_func_group_t, crypto_func_group_t, boolean_t, size_t);
+extern crypto_ctx_t *kcf_new_ctx(crypto_call_req_t *, kcf_provider_desc_t *,
+ crypto_session_id_t);
+extern int kcf_submit_request(kcf_provider_desc_t *, crypto_ctx_t *,
+ crypto_call_req_t *, kcf_req_params_t *, boolean_t);
+extern void kcf_sched_destroy(void);
+extern void kcf_sched_init(void);
+extern void kcf_sched_start(void);
+extern void kcf_sop_done(kcf_sreq_node_t *, int);
+extern void kcf_aop_done(kcf_areq_node_t *, int);
+extern int common_submit_request(kcf_provider_desc_t *,
+ crypto_ctx_t *, kcf_req_params_t *, crypto_req_handle_t);
+extern void kcf_free_context(kcf_context_t *);
+
+extern int kcf_svc_wait(int *);
+extern int kcf_svc_do_run(void);
+extern int kcf_need_signature_verification(kcf_provider_desc_t *);
+extern void kcf_verify_signature(void *);
+extern struct modctl *kcf_get_modctl(crypto_provider_info_t *);
+extern void verify_unverified_providers(void);
+extern void kcf_free_req(kcf_areq_node_t *areq);
+extern void crypto_bufcall_service(void);
+
+extern void kcf_walk_ntfylist(uint32_t, void *);
+extern void kcf_do_notify(kcf_provider_desc_t *, boolean_t);
+
+extern kcf_dual_req_t *kcf_alloc_req(crypto_call_req_t *);
+extern void kcf_next_req(void *, int);
+extern void kcf_last_req(void *, int);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_CRYPTO_SCHED_IMPL_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_CRYPTO_SPI_H
+#define _SYS_CRYPTO_SPI_H
+
+/*
+ * CSPI: Cryptographic Service Provider Interface.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#define CRYPTO_SPI_VERSION_1 1
+#define CRYPTO_SPI_VERSION_2 2
+#define CRYPTO_SPI_VERSION_3 3
+
+/*
+ * Provider-private handle. This handle is specified by a provider
+ * when it registers by means of the pi_provider_handle field of
+ * the crypto_provider_info structure, and passed to the provider
+ * when its entry points are invoked.
+ */
+typedef void *crypto_provider_handle_t;
+
+/*
+ * Context templates can be used to by software providers to pre-process
+ * keying material, such as key schedules. They are allocated by
+ * a software provider create_ctx_template(9E) entry point, and passed
+ * as argument to initialization and atomic provider entry points.
+ */
+typedef void *crypto_spi_ctx_template_t;
+
+/*
+ * Request handles are used by the kernel to identify an asynchronous
+ * request being processed by a provider. It is passed by the kernel
+ * to a hardware provider when submitting a request, and must be
+ * specified by a provider when calling crypto_op_notification(9F)
+ */
+typedef void *crypto_req_handle_t;
+
+/* Values for cc_flags field */
+#define CRYPTO_INIT_OPSTATE 0x00000001 /* allocate and init cc_opstate */
+#define CRYPTO_USE_OPSTATE 0x00000002 /* .. start using it as context */
+
+/*
+ * The context structure is passed from the kernel to a provider.
+ * It contains the information needed to process a multi-part or
+ * single part operation. The context structure is not used
+ * by atomic operations.
+ *
+ * Parameters needed to perform a cryptographic operation, such
+ * as keys, mechanisms, input and output buffers, are passed
+ * as separate arguments to Provider routines.
+ */
+typedef struct crypto_ctx {
+ crypto_provider_handle_t cc_provider;
+ crypto_session_id_t cc_session;
+ void *cc_provider_private; /* owned by provider */
+ void *cc_framework_private; /* owned by framework */
+ uint32_t cc_flags; /* flags */
+ void *cc_opstate; /* state */
+} crypto_ctx_t;
+
+/*
+ * Extended provider information.
+ */
+
+/*
+ * valid values for ei_flags field of extended info structure
+ * They match the RSA Security, Inc PKCS#11 tokenInfo flags.
+ */
+#define CRYPTO_EXTF_RNG 0x00000001
+#define CRYPTO_EXTF_WRITE_PROTECTED 0x00000002
+#define CRYPTO_EXTF_LOGIN_REQUIRED 0x00000004
+#define CRYPTO_EXTF_USER_PIN_INITIALIZED 0x00000008
+#define CRYPTO_EXTF_CLOCK_ON_TOKEN 0x00000040
+#define CRYPTO_EXTF_PROTECTED_AUTHENTICATION_PATH 0x00000100
+#define CRYPTO_EXTF_DUAL_CRYPTO_OPERATIONS 0x00000200
+#define CRYPTO_EXTF_TOKEN_INITIALIZED 0x00000400
+#define CRYPTO_EXTF_USER_PIN_COUNT_LOW 0x00010000
+#define CRYPTO_EXTF_USER_PIN_FINAL_TRY 0x00020000
+#define CRYPTO_EXTF_USER_PIN_LOCKED 0x00040000
+#define CRYPTO_EXTF_USER_PIN_TO_BE_CHANGED 0x00080000
+#define CRYPTO_EXTF_SO_PIN_COUNT_LOW 0x00100000
+#define CRYPTO_EXTF_SO_PIN_FINAL_TRY 0x00200000
+#define CRYPTO_EXTF_SO_PIN_LOCKED 0x00400000
+#define CRYPTO_EXTF_SO_PIN_TO_BE_CHANGED 0x00800000
+
+/*
+ * The crypto_control_ops structure contains pointers to control
+ * operations for cryptographic providers. It is passed through
+ * the crypto_ops(9S) structure when providers register with the
+ * kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_control_ops {
+ void (*provider_status)(crypto_provider_handle_t, uint_t *);
+} crypto_control_ops_t;
+
+/*
+ * The crypto_ctx_ops structure contains points to context and context
+ * templates management operations for cryptographic providers. It is
+ * passed through the crypto_ops(9S) structure when providers register
+ * with the kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_ctx_ops {
+ int (*create_ctx_template)(crypto_provider_handle_t,
+ crypto_mechanism_t *, crypto_key_t *,
+ crypto_spi_ctx_template_t *, size_t *, crypto_req_handle_t);
+ int (*free_context)(crypto_ctx_t *);
+} crypto_ctx_ops_t;
+
+/*
+ * The crypto_digest_ops structure contains pointers to digest
+ * operations for cryptographic providers. It is passed through
+ * the crypto_ops(9S) structure when providers register with the
+ * kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_digest_ops {
+ int (*digest_init)(crypto_ctx_t *, crypto_mechanism_t *,
+ crypto_req_handle_t);
+ int (*digest)(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
+ crypto_req_handle_t);
+ int (*digest_update)(crypto_ctx_t *, crypto_data_t *,
+ crypto_req_handle_t);
+ int (*digest_key)(crypto_ctx_t *, crypto_key_t *, crypto_req_handle_t);
+ int (*digest_final)(crypto_ctx_t *, crypto_data_t *,
+ crypto_req_handle_t);
+ int (*digest_atomic)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_data_t *,
+ crypto_data_t *, crypto_req_handle_t);
+} crypto_digest_ops_t;
+
+/*
+ * The crypto_cipher_ops structure contains pointers to encryption
+ * and decryption operations for cryptographic providers. It is
+ * passed through the crypto_ops(9S) structure when providers register
+ * with the kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_cipher_ops {
+ int (*encrypt_init)(crypto_ctx_t *,
+ crypto_mechanism_t *, crypto_key_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+ int (*encrypt)(crypto_ctx_t *,
+ crypto_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*encrypt_update)(crypto_ctx_t *,
+ crypto_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*encrypt_final)(crypto_ctx_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*encrypt_atomic)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
+
+ int (*decrypt_init)(crypto_ctx_t *,
+ crypto_mechanism_t *, crypto_key_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+ int (*decrypt)(crypto_ctx_t *,
+ crypto_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*decrypt_update)(crypto_ctx_t *,
+ crypto_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*decrypt_final)(crypto_ctx_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*decrypt_atomic)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
+} crypto_cipher_ops_t;
+
+/*
+ * The crypto_mac_ops structure contains pointers to MAC
+ * operations for cryptographic providers. It is passed through
+ * the crypto_ops(9S) structure when providers register with the
+ * kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_mac_ops {
+ int (*mac_init)(crypto_ctx_t *,
+ crypto_mechanism_t *, crypto_key_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+ int (*mac)(crypto_ctx_t *,
+ crypto_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*mac_update)(crypto_ctx_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*mac_final)(crypto_ctx_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*mac_atomic)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_data_t *, crypto_spi_ctx_template_t,
+ crypto_req_handle_t);
+ int (*mac_verify_atomic)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_data_t *, crypto_spi_ctx_template_t,
+ crypto_req_handle_t);
+} crypto_mac_ops_t;
+
+/*
+ * The crypto_sign_ops structure contains pointers to signing
+ * operations for cryptographic providers. It is passed through
+ * the crypto_ops(9S) structure when providers register with the
+ * kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_sign_ops {
+ int (*sign_init)(crypto_ctx_t *,
+ crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t,
+ crypto_req_handle_t);
+ int (*sign)(crypto_ctx_t *,
+ crypto_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*sign_update)(crypto_ctx_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*sign_final)(crypto_ctx_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*sign_atomic)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_data_t *, crypto_spi_ctx_template_t,
+ crypto_req_handle_t);
+ int (*sign_recover_init)(crypto_ctx_t *, crypto_mechanism_t *,
+ crypto_key_t *, crypto_spi_ctx_template_t,
+ crypto_req_handle_t);
+ int (*sign_recover)(crypto_ctx_t *,
+ crypto_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*sign_recover_atomic)(crypto_provider_handle_t,
+ crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
+ crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
+ crypto_req_handle_t);
+} crypto_sign_ops_t;
+
+/*
+ * The crypto_verify_ops structure contains pointers to verify
+ * operations for cryptographic providers. It is passed through
+ * the crypto_ops(9S) structure when providers register with the
+ * kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_verify_ops {
+ int (*verify_init)(crypto_ctx_t *,
+ crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t,
+ crypto_req_handle_t);
+ int (*do_verify)(crypto_ctx_t *,
+ crypto_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*verify_update)(crypto_ctx_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*verify_final)(crypto_ctx_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*verify_atomic)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_data_t *, crypto_spi_ctx_template_t,
+ crypto_req_handle_t);
+ int (*verify_recover_init)(crypto_ctx_t *, crypto_mechanism_t *,
+ crypto_key_t *, crypto_spi_ctx_template_t,
+ crypto_req_handle_t);
+ int (*verify_recover)(crypto_ctx_t *,
+ crypto_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*verify_recover_atomic)(crypto_provider_handle_t,
+ crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
+ crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
+ crypto_req_handle_t);
+} crypto_verify_ops_t;
+
+/*
+ * The crypto_dual_ops structure contains pointers to dual
+ * cipher and sign/verify operations for cryptographic providers.
+ * It is passed through the crypto_ops(9S) structure when
+ * providers register with the kernel using
+ * crypto_register_provider(9F).
+ */
+typedef struct crypto_dual_ops {
+ int (*digest_encrypt_update)(
+ crypto_ctx_t *, crypto_ctx_t *, crypto_data_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*decrypt_digest_update)(
+ crypto_ctx_t *, crypto_ctx_t *, crypto_data_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*sign_encrypt_update)(
+ crypto_ctx_t *, crypto_ctx_t *, crypto_data_t *,
+ crypto_data_t *, crypto_req_handle_t);
+ int (*decrypt_verify_update)(
+ crypto_ctx_t *, crypto_ctx_t *, crypto_data_t *,
+ crypto_data_t *, crypto_req_handle_t);
+} crypto_dual_ops_t;
+
+/*
+ * The crypto_dual_cipher_mac_ops structure contains pointers to dual
+ * cipher and MAC operations for cryptographic providers.
+ * It is passed through the crypto_ops(9S) structure when
+ * providers register with the kernel using
+ * crypto_register_provider(9F).
+ */
+typedef struct crypto_dual_cipher_mac_ops {
+ int (*encrypt_mac_init)(crypto_ctx_t *,
+ crypto_mechanism_t *, crypto_key_t *, crypto_mechanism_t *,
+ crypto_key_t *, crypto_spi_ctx_template_t,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+ int (*encrypt_mac)(crypto_ctx_t *,
+ crypto_data_t *, crypto_dual_data_t *, crypto_data_t *,
+ crypto_req_handle_t);
+ int (*encrypt_mac_update)(crypto_ctx_t *,
+ crypto_data_t *, crypto_dual_data_t *, crypto_req_handle_t);
+ int (*encrypt_mac_final)(crypto_ctx_t *,
+ crypto_dual_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*encrypt_mac_atomic)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_mechanism_t *,
+ crypto_key_t *, crypto_data_t *, crypto_dual_data_t *,
+ crypto_data_t *, crypto_spi_ctx_template_t,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+
+ int (*mac_decrypt_init)(crypto_ctx_t *,
+ crypto_mechanism_t *, crypto_key_t *, crypto_mechanism_t *,
+ crypto_key_t *, crypto_spi_ctx_template_t,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+ int (*mac_decrypt)(crypto_ctx_t *,
+ crypto_dual_data_t *, crypto_data_t *, crypto_data_t *,
+ crypto_req_handle_t);
+ int (*mac_decrypt_update)(crypto_ctx_t *,
+ crypto_dual_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*mac_decrypt_final)(crypto_ctx_t *,
+ crypto_data_t *, crypto_data_t *, crypto_req_handle_t);
+ int (*mac_decrypt_atomic)(crypto_provider_handle_t,
+ crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
+ crypto_mechanism_t *, crypto_key_t *, crypto_dual_data_t *,
+ crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+ int (*mac_verify_decrypt_atomic)(crypto_provider_handle_t,
+ crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
+ crypto_mechanism_t *, crypto_key_t *, crypto_dual_data_t *,
+ crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+} crypto_dual_cipher_mac_ops_t;
+
+/*
+ * The crypto_random_number_ops structure contains pointers to random
+ * number operations for cryptographic providers. It is passed through
+ * the crypto_ops(9S) structure when providers register with the
+ * kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_random_number_ops {
+ int (*seed_random)(crypto_provider_handle_t, crypto_session_id_t,
+ uchar_t *, size_t, uint_t, uint32_t, crypto_req_handle_t);
+ int (*generate_random)(crypto_provider_handle_t, crypto_session_id_t,
+ uchar_t *, size_t, crypto_req_handle_t);
+} crypto_random_number_ops_t;
+
+/*
+ * Flag values for seed_random.
+ */
+#define CRYPTO_SEED_NOW 0x00000001
+
+/*
+ * The crypto_session_ops structure contains pointers to session
+ * operations for cryptographic providers. It is passed through
+ * the crypto_ops(9S) structure when providers register with the
+ * kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_session_ops {
+ int (*session_open)(crypto_provider_handle_t, crypto_session_id_t *,
+ crypto_req_handle_t);
+ int (*session_close)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_req_handle_t);
+ int (*session_login)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_user_type_t, char *, size_t, crypto_req_handle_t);
+ int (*session_logout)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_req_handle_t);
+} crypto_session_ops_t;
+
+/*
+ * The crypto_object_ops structure contains pointers to object
+ * operations for cryptographic providers. It is passed through
+ * the crypto_ops(9S) structure when providers register with the
+ * kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_object_ops {
+ int (*object_create)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_object_attribute_t *, uint_t, crypto_object_id_t *,
+ crypto_req_handle_t);
+ int (*object_copy)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_object_id_t, crypto_object_attribute_t *, uint_t,
+ crypto_object_id_t *, crypto_req_handle_t);
+ int (*object_destroy)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_object_id_t, crypto_req_handle_t);
+ int (*object_get_size)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_object_id_t, size_t *, crypto_req_handle_t);
+ int (*object_get_attribute_value)(crypto_provider_handle_t,
+ crypto_session_id_t, crypto_object_id_t,
+ crypto_object_attribute_t *, uint_t, crypto_req_handle_t);
+ int (*object_set_attribute_value)(crypto_provider_handle_t,
+ crypto_session_id_t, crypto_object_id_t,
+ crypto_object_attribute_t *, uint_t, crypto_req_handle_t);
+ int (*object_find_init)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_object_attribute_t *, uint_t, void **,
+ crypto_req_handle_t);
+ int (*object_find)(crypto_provider_handle_t, void *,
+ crypto_object_id_t *, uint_t, uint_t *, crypto_req_handle_t);
+ int (*object_find_final)(crypto_provider_handle_t, void *,
+ crypto_req_handle_t);
+} crypto_object_ops_t;
+
+/*
+ * The crypto_key_ops structure contains pointers to key
+ * operations for cryptographic providers. It is passed through
+ * the crypto_ops(9S) structure when providers register with the
+ * kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_key_ops {
+ int (*key_generate)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_object_attribute_t *, uint_t,
+ crypto_object_id_t *, crypto_req_handle_t);
+ int (*key_generate_pair)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_object_attribute_t *, uint_t,
+ crypto_object_attribute_t *, uint_t, crypto_object_id_t *,
+ crypto_object_id_t *, crypto_req_handle_t);
+ int (*key_wrap)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_object_id_t *,
+ uchar_t *, size_t *, crypto_req_handle_t);
+ int (*key_unwrap)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, uchar_t *, size_t *,
+ crypto_object_attribute_t *, uint_t,
+ crypto_object_id_t *, crypto_req_handle_t);
+ int (*key_derive)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_object_attribute_t *,
+ uint_t, crypto_object_id_t *, crypto_req_handle_t);
+ int (*key_check)(crypto_provider_handle_t, crypto_mechanism_t *,
+ crypto_key_t *);
+} crypto_key_ops_t;
+
+/*
+ * The crypto_provider_management_ops structure contains pointers
+ * to management operations for cryptographic providers. It is passed
+ * through the crypto_ops(9S) structure when providers register with the
+ * kernel using crypto_register_provider(9F).
+ */
+typedef struct crypto_provider_management_ops {
+ int (*ext_info)(crypto_provider_handle_t,
+ crypto_provider_ext_info_t *, crypto_req_handle_t);
+ int (*init_token)(crypto_provider_handle_t, char *, size_t,
+ char *, crypto_req_handle_t);
+ int (*init_pin)(crypto_provider_handle_t, crypto_session_id_t,
+ char *, size_t, crypto_req_handle_t);
+ int (*set_pin)(crypto_provider_handle_t, crypto_session_id_t,
+ char *, size_t, char *, size_t, crypto_req_handle_t);
+} crypto_provider_management_ops_t;
+
+typedef struct crypto_mech_ops {
+ int (*copyin_mechanism)(crypto_provider_handle_t,
+ crypto_mechanism_t *, crypto_mechanism_t *, int *, int);
+ int (*copyout_mechanism)(crypto_provider_handle_t,
+ crypto_mechanism_t *, crypto_mechanism_t *, int *, int);
+ int (*free_mechanism)(crypto_provider_handle_t, crypto_mechanism_t *);
+} crypto_mech_ops_t;
+
+typedef struct crypto_nostore_key_ops {
+ int (*nostore_key_generate)(crypto_provider_handle_t,
+ crypto_session_id_t, crypto_mechanism_t *,
+ crypto_object_attribute_t *, uint_t, crypto_object_attribute_t *,
+ uint_t, crypto_req_handle_t);
+ int (*nostore_key_generate_pair)(crypto_provider_handle_t,
+ crypto_session_id_t, crypto_mechanism_t *,
+ crypto_object_attribute_t *, uint_t, crypto_object_attribute_t *,
+ uint_t, crypto_object_attribute_t *, uint_t,
+ crypto_object_attribute_t *, uint_t, crypto_req_handle_t);
+ int (*nostore_key_derive)(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_object_attribute_t *,
+ uint_t, crypto_object_attribute_t *, uint_t, crypto_req_handle_t);
+} crypto_nostore_key_ops_t;
+
+/*
+ * The crypto_ops(9S) structure contains the structures containing
+ * the pointers to functions implemented by cryptographic providers.
+ * It is specified as part of the crypto_provider_info(9S)
+ * supplied by a provider when it registers with the kernel
+ * by calling crypto_register_provider(9F).
+ */
+typedef struct crypto_ops_v1 {
+ crypto_control_ops_t *co_control_ops;
+ crypto_digest_ops_t *co_digest_ops;
+ crypto_cipher_ops_t *co_cipher_ops;
+ crypto_mac_ops_t *co_mac_ops;
+ crypto_sign_ops_t *co_sign_ops;
+ crypto_verify_ops_t *co_verify_ops;
+ crypto_dual_ops_t *co_dual_ops;
+ crypto_dual_cipher_mac_ops_t *co_dual_cipher_mac_ops;
+ crypto_random_number_ops_t *co_random_ops;
+ crypto_session_ops_t *co_session_ops;
+ crypto_object_ops_t *co_object_ops;
+ crypto_key_ops_t *co_key_ops;
+ crypto_provider_management_ops_t *co_provider_ops;
+ crypto_ctx_ops_t *co_ctx_ops;
+} crypto_ops_v1_t;
+
+typedef struct crypto_ops_v2 {
+ crypto_ops_v1_t v1_ops;
+ crypto_mech_ops_t *co_mech_ops;
+} crypto_ops_v2_t;
+
+typedef struct crypto_ops_v3 {
+ crypto_ops_v2_t v2_ops;
+ crypto_nostore_key_ops_t *co_nostore_key_ops;
+} crypto_ops_v3_t;
+
+typedef struct crypto_ops {
+ union {
+ crypto_ops_v3_t cou_v3;
+ crypto_ops_v2_t cou_v2;
+ crypto_ops_v1_t cou_v1;
+ } cou;
+} crypto_ops_t;
+
+#define co_control_ops cou.cou_v1.co_control_ops
+#define co_digest_ops cou.cou_v1.co_digest_ops
+#define co_cipher_ops cou.cou_v1.co_cipher_ops
+#define co_mac_ops cou.cou_v1.co_mac_ops
+#define co_sign_ops cou.cou_v1.co_sign_ops
+#define co_verify_ops cou.cou_v1.co_verify_ops
+#define co_dual_ops cou.cou_v1.co_dual_ops
+#define co_dual_cipher_mac_ops cou.cou_v1.co_dual_cipher_mac_ops
+#define co_random_ops cou.cou_v1.co_random_ops
+#define co_session_ops cou.cou_v1.co_session_ops
+#define co_object_ops cou.cou_v1.co_object_ops
+#define co_key_ops cou.cou_v1.co_key_ops
+#define co_provider_ops cou.cou_v1.co_provider_ops
+#define co_ctx_ops cou.cou_v1.co_ctx_ops
+#define co_mech_ops cou.cou_v2.co_mech_ops
+#define co_nostore_key_ops cou.cou_v3.co_nostore_key_ops
+
+/*
+ * The mechanism info structure crypto_mech_info_t contains a function group
+ * bit mask cm_func_group_mask. This field, of type crypto_func_group_t,
+ * specifies the provider entry point that can be used a particular
+ * mechanism. The function group mask is a combination of the following values.
+ */
+
+typedef uint32_t crypto_func_group_t;
+
+
+#define CRYPTO_FG_ENCRYPT 0x00000001 /* encrypt_init() */
+#define CRYPTO_FG_DECRYPT 0x00000002 /* decrypt_init() */
+#define CRYPTO_FG_DIGEST 0x00000004 /* digest_init() */
+#define CRYPTO_FG_SIGN 0x00000008 /* sign_init() */
+#define CRYPTO_FG_SIGN_RECOVER 0x00000010 /* sign_recover_init() */
+#define CRYPTO_FG_VERIFY 0x00000020 /* verify_init() */
+#define CRYPTO_FG_VERIFY_RECOVER 0x00000040 /* verify_recover_init() */
+#define CRYPTO_FG_GENERATE 0x00000080 /* key_generate() */
+#define CRYPTO_FG_GENERATE_KEY_PAIR 0x00000100 /* key_generate_pair() */
+#define CRYPTO_FG_WRAP 0x00000200 /* key_wrap() */
+#define CRYPTO_FG_UNWRAP 0x00000400 /* key_unwrap() */
+#define CRYPTO_FG_DERIVE 0x00000800 /* key_derive() */
+#define CRYPTO_FG_MAC 0x00001000 /* mac_init() */
+#define CRYPTO_FG_ENCRYPT_MAC 0x00002000 /* encrypt_mac_init() */
+#define CRYPTO_FG_MAC_DECRYPT 0x00004000 /* decrypt_mac_init() */
+#define CRYPTO_FG_ENCRYPT_ATOMIC 0x00008000 /* encrypt_atomic() */
+#define CRYPTO_FG_DECRYPT_ATOMIC 0x00010000 /* decrypt_atomic() */
+#define CRYPTO_FG_MAC_ATOMIC 0x00020000 /* mac_atomic() */
+#define CRYPTO_FG_DIGEST_ATOMIC 0x00040000 /* digest_atomic() */
+#define CRYPTO_FG_SIGN_ATOMIC 0x00080000 /* sign_atomic() */
+#define CRYPTO_FG_SIGN_RECOVER_ATOMIC 0x00100000 /* sign_recover_atomic() */
+#define CRYPTO_FG_VERIFY_ATOMIC 0x00200000 /* verify_atomic() */
+#define CRYPTO_FG_VERIFY_RECOVER_ATOMIC 0x00400000 /* verify_recover_atomic() */
+#define CRYPTO_FG_ENCRYPT_MAC_ATOMIC 0x00800000 /* encrypt_mac_atomic() */
+#define CRYPTO_FG_MAC_DECRYPT_ATOMIC 0x01000000 /* mac_decrypt_atomic() */
+#define CRYPTO_FG_RESERVED 0x80000000
+
+/*
+ * Maximum length of the pi_provider_description field of the
+ * crypto_provider_info structure.
+ */
+#define CRYPTO_PROVIDER_DESCR_MAX_LEN 64
+
+
+/* Bit mask for all the simple operations */
+#define CRYPTO_FG_SIMPLEOP_MASK (CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | \
+ CRYPTO_FG_DIGEST | CRYPTO_FG_SIGN | CRYPTO_FG_VERIFY | CRYPTO_FG_MAC | \
+ CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC | \
+ CRYPTO_FG_MAC_ATOMIC | CRYPTO_FG_DIGEST_ATOMIC | CRYPTO_FG_SIGN_ATOMIC | \
+ CRYPTO_FG_VERIFY_ATOMIC)
+
+/* Bit mask for all the dual operations */
+#define CRYPTO_FG_MAC_CIPHER_MASK (CRYPTO_FG_ENCRYPT_MAC | \
+ CRYPTO_FG_MAC_DECRYPT | CRYPTO_FG_ENCRYPT_MAC_ATOMIC | \
+ CRYPTO_FG_MAC_DECRYPT_ATOMIC)
+
+/* Add other combos to CRYPTO_FG_DUAL_MASK */
+#define CRYPTO_FG_DUAL_MASK CRYPTO_FG_MAC_CIPHER_MASK
+
+/*
+ * The crypto_mech_info structure specifies one of the mechanisms
+ * supported by a cryptographic provider. The pi_mechanisms field of
+ * the crypto_provider_info structure contains a pointer to an array
+ * of crypto_mech_info's.
+ */
+typedef struct crypto_mech_info {
+ crypto_mech_name_t cm_mech_name;
+ crypto_mech_type_t cm_mech_number;
+ crypto_func_group_t cm_func_group_mask;
+ ssize_t cm_min_key_length;
+ ssize_t cm_max_key_length;
+ uint32_t cm_mech_flags;
+} crypto_mech_info_t;
+
+/* Alias the old name to the new name for compatibility. */
+#define cm_keysize_unit cm_mech_flags
+
+/*
+ * The following is used by a provider that sets
+ * CRYPTO_HASH_NO_UPDATE. It needs to specify the maximum
+ * input data size it can digest in this field.
+ */
+#define cm_max_input_length cm_max_key_length
+
+/*
+ * crypto_kcf_provider_handle_t is a handle allocated by the kernel.
+ * It is returned after the provider registers with
+ * crypto_register_provider(), and must be specified by the provider
+ * when calling crypto_unregister_provider(), and
+ * crypto_provider_notification().
+ */
+typedef uint_t crypto_kcf_provider_handle_t;
+
+/*
+ * Provider information. Passed as argument to crypto_register_provider(9F).
+ * Describes the provider and its capabilities. Multiple providers can
+ * register for the same device instance. In this case, the same
+ * pi_provider_dev must be specified with a different pi_provider_handle.
+ */
+typedef struct crypto_provider_info_v1 {
+ uint_t pi_interface_version;
+ char *pi_provider_description;
+ crypto_provider_type_t pi_provider_type;
+ crypto_provider_handle_t pi_provider_handle;
+ crypto_ops_t *pi_ops_vector;
+ uint_t pi_mech_list_count;
+ crypto_mech_info_t *pi_mechanisms;
+ uint_t pi_logical_provider_count;
+ crypto_kcf_provider_handle_t *pi_logical_providers;
+} crypto_provider_info_v1_t;
+
+typedef struct crypto_provider_info_v2 {
+ crypto_provider_info_v1_t v1_info;
+ uint_t pi_flags;
+} crypto_provider_info_v2_t;
+
+typedef struct crypto_provider_info {
+ union {
+ crypto_provider_info_v2_t piu_v2;
+ crypto_provider_info_v1_t piu_v1;
+ } piu;
+} crypto_provider_info_t;
+
+#define pi_interface_version piu.piu_v1.pi_interface_version
+#define pi_provider_description piu.piu_v1.pi_provider_description
+#define pi_provider_type piu.piu_v1.pi_provider_type
+#define pi_provider_handle piu.piu_v1.pi_provider_handle
+#define pi_ops_vector piu.piu_v1.pi_ops_vector
+#define pi_mech_list_count piu.piu_v1.pi_mech_list_count
+#define pi_mechanisms piu.piu_v1.pi_mechanisms
+#define pi_logical_provider_count piu.piu_v1.pi_logical_provider_count
+#define pi_logical_providers piu.piu_v1.pi_logical_providers
+#define pi_flags piu.piu_v2.pi_flags
+
+/* hidden providers can only be accessed via a logical provider */
+#define CRYPTO_HIDE_PROVIDER 0x00000001
+/*
+ * provider can not do multi-part digest (updates) and has a limit
+ * on maximum input data that it can digest.
+ */
+#define CRYPTO_HASH_NO_UPDATE 0x00000002
+
+/* provider can handle the request without returning a CRYPTO_QUEUED */
+#define CRYPTO_SYNCHRONOUS 0x00000004
+
+#define CRYPTO_PIFLAGS_RESERVED2 0x40000000
+#define CRYPTO_PIFLAGS_RESERVED1 0x80000000
+
+/*
+ * Provider status passed by a provider to crypto_provider_notification(9F)
+ * and returned by the provider_stauts(9E) entry point.
+ */
+#define CRYPTO_PROVIDER_READY 0
+#define CRYPTO_PROVIDER_BUSY 1
+#define CRYPTO_PROVIDER_FAILED 2
+
+/*
+ * Functions exported by Solaris to cryptographic providers. Providers
+ * call these functions to register and unregister, notify the kernel
+ * of state changes, and notify the kernel when a asynchronous request
+ * completed.
+ */
+extern int crypto_register_provider(crypto_provider_info_t *,
+ crypto_kcf_provider_handle_t *);
+extern int crypto_unregister_provider(crypto_kcf_provider_handle_t);
+extern void crypto_provider_notification(crypto_kcf_provider_handle_t, uint_t);
+extern void crypto_op_notification(crypto_req_handle_t, int);
+extern int crypto_kmflag(crypto_req_handle_t);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_CRYPTO_SPI_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _IA32_SYS_ASM_LINKAGE_H
+#define _IA32_SYS_ASM_LINKAGE_H
+
+#include <sys/stack.h>
+#include <sys/trap.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _ASM /* The remainder of this file is only for assembly files */
+
+/*
+ * make annoying differences in assembler syntax go away
+ */
+
+/*
+ * D16 and A16 are used to insert instructions prefixes; the
+ * macros help the assembler code be slightly more portable.
+ */
+#if !defined(__GNUC_AS__)
+/*
+ * /usr/ccs/bin/as prefixes are parsed as separate instructions
+ */
+#define D16 data16;
+#define A16 addr16;
+
+/*
+ * (There are some weird constructs in constant expressions)
+ */
+#define _CONST(const) [const]
+#define _BITNOT(const) -1!_CONST(const)
+#define _MUL(a, b) _CONST(a \* b)
+
+#else
+/*
+ * Why not use the 'data16' and 'addr16' prefixes .. well, the
+ * assembler doesn't quite believe in real mode, and thus argues with
+ * us about what we're trying to do.
+ */
+#define D16 .byte 0x66;
+#define A16 .byte 0x67;
+
+#define _CONST(const) (const)
+#define _BITNOT(const) ~_CONST(const)
+#define _MUL(a, b) _CONST(a * b)
+
+#endif
+
+/*
+ * C pointers are different sizes between i386 and amd64.
+ * These constants can be used to compute offsets into pointer arrays.
+ */
+#if defined(__amd64)
+#define CLONGSHIFT 3
+#define CLONGSIZE 8
+#define CLONGMASK 7
+#elif defined(__i386)
+#define CLONGSHIFT 2
+#define CLONGSIZE 4
+#define CLONGMASK 3
+#endif
+
+/*
+ * Since we know we're either ILP32 or LP64 ..
+ */
+#define CPTRSHIFT CLONGSHIFT
+#define CPTRSIZE CLONGSIZE
+#define CPTRMASK CLONGMASK
+
+#if CPTRSIZE != (1 << CPTRSHIFT) || CLONGSIZE != (1 << CLONGSHIFT)
+#error "inconsistent shift constants"
+#endif
+
+#if CPTRMASK != (CPTRSIZE - 1) || CLONGMASK != (CLONGSIZE - 1)
+#error "inconsistent mask constants"
+#endif
+
+#define ASM_ENTRY_ALIGN 16
+
+/*
+ * SSE register alignment and save areas
+ */
+
+#define XMM_SIZE 16
+#define XMM_ALIGN 16
+
+#if defined(__amd64)
+
+#define SAVE_XMM_PROLOG(sreg, nreg) \
+ subq $_CONST(_MUL(XMM_SIZE, nreg)), %rsp; \
+ movq %rsp, sreg
+
+#define RSTOR_XMM_EPILOG(sreg, nreg) \
+ addq $_CONST(_MUL(XMM_SIZE, nreg)), %rsp
+
+#elif defined(__i386)
+
+#define SAVE_XMM_PROLOG(sreg, nreg) \
+ subl $_CONST(_MUL(XMM_SIZE, nreg) + XMM_ALIGN), %esp; \
+ movl %esp, sreg; \
+ addl $XMM_ALIGN, sreg; \
+ andl $_BITNOT(XMM_ALIGN-1), sreg
+
+#define RSTOR_XMM_EPILOG(sreg, nreg) \
+ addl $_CONST(_MUL(XMM_SIZE, nreg) + XMM_ALIGN), %esp;
+
+#endif /* __i386 */
+
+/*
+ * profiling causes definitions of the MCOUNT and RTMCOUNT
+ * particular to the type
+ */
+#ifdef GPROF
+
+#define MCOUNT(x) \
+ pushl %ebp; \
+ movl %esp, %ebp; \
+ call _mcount; \
+ popl %ebp
+
+#endif /* GPROF */
+
+#ifdef PROF
+
+#define MCOUNT(x) \
+/* CSTYLED */ \
+ .lcomm .L_/**/x/**/1, 4, 4; \
+ pushl %ebp; \
+ movl %esp, %ebp; \
+/* CSTYLED */ \
+ movl $.L_/**/x/**/1, %edx; \
+ call _mcount; \
+ popl %ebp
+
+#endif /* PROF */
+
+/*
+ * if we are not profiling, MCOUNT should be defined to nothing
+ */
+#if !defined(PROF) && !defined(GPROF)
+#define MCOUNT(x)
+#endif /* !defined(PROF) && !defined(GPROF) */
+
+#define RTMCOUNT(x) MCOUNT(x)
+
+/*
+ * Macro to define weak symbol aliases. These are similar to the ANSI-C
+ * #pragma weak _name = name
+ * except a compiler can determine type. The assembler must be told. Hence,
+ * the second parameter must be the type of the symbol (i.e.: function,...)
+ */
+#define ANSI_PRAGMA_WEAK(sym, stype) \
+/* CSTYLED */ \
+ .weak _/**/sym; \
+/* CSTYLED */ \
+ .type _/**/sym, @stype; \
+/* CSTYLED */ \
+_/**/sym = sym
+
+/*
+ * Like ANSI_PRAGMA_WEAK(), but for unrelated names, as in:
+ * #pragma weak sym1 = sym2
+ */
+#define ANSI_PRAGMA_WEAK2(sym1, sym2, stype) \
+ .weak sym1; \
+ .type sym1, @stype; \
+sym1 = sym2
+
+/*
+ * ENTRY provides the standard procedure entry code and an easy way to
+ * insert the calls to mcount for profiling. ENTRY_NP is identical, but
+ * never calls mcount.
+ */
+#define ENTRY(x) \
+ .text; \
+ .align ASM_ENTRY_ALIGN; \
+ .globl x; \
+ .type x, @function; \
+x: MCOUNT(x)
+
+#define ENTRY_NP(x) \
+ .text; \
+ .align ASM_ENTRY_ALIGN; \
+ .globl x; \
+ .type x, @function; \
+x:
+
+#define RTENTRY(x) \
+ .text; \
+ .align ASM_ENTRY_ALIGN; \
+ .globl x; \
+ .type x, @function; \
+x: RTMCOUNT(x)
+
+/*
+ * ENTRY2 is identical to ENTRY but provides two labels for the entry point.
+ */
+#define ENTRY2(x, y) \
+ .text; \
+ .align ASM_ENTRY_ALIGN; \
+ .globl x, y; \
+ .type x, @function; \
+ .type y, @function; \
+/* CSTYLED */ \
+x: ; \
+y: MCOUNT(x)
+
+#define ENTRY_NP2(x, y) \
+ .text; \
+ .align ASM_ENTRY_ALIGN; \
+ .globl x, y; \
+ .type x, @function; \
+ .type y, @function; \
+/* CSTYLED */ \
+x: ; \
+y:
+
+
+/*
+ * ALTENTRY provides for additional entry points.
+ */
+#define ALTENTRY(x) \
+ .globl x; \
+ .type x, @function; \
+x:
+
+/*
+ * DGDEF and DGDEF2 provide global data declarations.
+ *
+ * DGDEF provides a word aligned word of storage.
+ *
+ * DGDEF2 allocates "sz" bytes of storage with **NO** alignment. This
+ * implies this macro is best used for byte arrays.
+ *
+ * DGDEF3 allocates "sz" bytes of storage with "algn" alignment.
+ */
+#define DGDEF2(name, sz) \
+ .data; \
+ .globl name; \
+ .type name, @object; \
+ .size name, sz; \
+name:
+
+#define DGDEF3(name, sz, algn) \
+ .data; \
+ .align algn; \
+ .globl name; \
+ .type name, @object; \
+ .size name, sz; \
+name:
+
+#define DGDEF(name) DGDEF3(name, 4, 4)
+
+/*
+ * SET_SIZE trails a function and set the size for the ELF symbol table.
+ */
+#define SET_SIZE(x) \
+ .size x, [.-x]
+
+/*
+ * NWORD provides native word value.
+ */
+#if defined(__amd64)
+
+/*CSTYLED*/
+#define NWORD quad
+
+#elif defined(__i386)
+
+#define NWORD long
+
+#endif /* __i386 */
+
+#endif /* _ASM */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _IA32_SYS_ASM_LINKAGE_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _IA32_SYS_STACK_H
+#define _IA32_SYS_STACK_H
+
+#if !defined(_ASM)
+
+#include <sys/types.h>
+
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * In the x86 world, a stack frame looks like this:
+ *
+ * |--------------------------|
+ * 4n+8(%ebp) ->| argument word n |
+ * | ... | (Previous frame)
+ * 8(%ebp) ->| argument word 0 |
+ * |--------------------------|--------------------
+ * 4(%ebp) ->| return address |
+ * |--------------------------|
+ * 0(%ebp) ->| previous %ebp (optional) |
+ * |--------------------------|
+ * -4(%ebp) ->| unspecified | (Current frame)
+ * | ... |
+ * 0(%esp) ->| variable size |
+ * |--------------------------|
+ */
+
+/*
+ * Stack alignment macros.
+ */
+
+#define STACK_ALIGN32 4
+#define STACK_ENTRY_ALIGN32 4
+#define STACK_BIAS32 0
+#define SA32(x) (((x)+(STACK_ALIGN32-1)) & ~(STACK_ALIGN32-1))
+#define STACK_RESERVE32 0
+#define MINFRAME32 0
+
+#if defined(__amd64)
+
+/*
+ * In the amd64 world, a stack frame looks like this:
+ *
+ * |--------------------------|
+ * 8n+16(%rbp)->| argument word n |
+ * | ... | (Previous frame)
+ * 16(%rbp) ->| argument word 0 |
+ * |--------------------------|--------------------
+ * 8(%rbp) ->| return address |
+ * |--------------------------|
+ * 0(%rbp) ->| previous %rbp |
+ * |--------------------------|
+ * -8(%rbp) ->| unspecified | (Current frame)
+ * | ... |
+ * 0(%rsp) ->| variable size |
+ * |--------------------------|
+ * -128(%rsp) ->| reserved for function |
+ * |--------------------------|
+ *
+ * The end of the input argument area must be aligned on a 16-byte
+ * boundary; i.e. (%rsp - 8) % 16 == 0 at function entry.
+ *
+ * The 128-byte location beyond %rsp is considered to be reserved for
+ * functions and is NOT modified by signal handlers. It can be used
+ * to store temporary data that is not needed across function calls.
+ */
+
+/*
+ * Stack alignment macros.
+ */
+
+#define STACK_ALIGN64 16
+#define STACK_ENTRY_ALIGN64 8
+#define STACK_BIAS64 0
+#define SA64(x) (((x)+(STACK_ALIGN64-1)) & ~(STACK_ALIGN64-1))
+#define STACK_RESERVE64 128
+#define MINFRAME64 0
+
+#define STACK_ALIGN STACK_ALIGN64
+#define STACK_ENTRY_ALIGN STACK_ENTRY_ALIGN64
+#define STACK_BIAS STACK_BIAS64
+#define SA(x) SA64(x)
+#define STACK_RESERVE STACK_RESERVE64
+#define MINFRAME MINFRAME64
+
+#elif defined(__i386)
+
+#define STACK_ALIGN STACK_ALIGN32
+#define STACK_ENTRY_ALIGN STACK_ENTRY_ALIGN32
+#define STACK_BIAS STACK_BIAS32
+#define SA(x) SA32(x)
+#define STACK_RESERVE STACK_RESERVE32
+#define MINFRAME MINFRAME32
+
+#endif /* __i386 */
+
+#if defined(_KERNEL) && !defined(_ASM)
+
+#if defined(DEBUG)
+#if STACK_ALIGN == 4
+#define ASSERT_STACK_ALIGNED() \
+ { \
+ uint32_t __tmp; \
+ ASSERT((((uintptr_t)&__tmp) & (STACK_ALIGN - 1)) == 0); \
+ }
+#elif (STACK_ALIGN == 16) && (_LONG_DOUBLE_ALIGNMENT == 16)
+#define ASSERT_STACK_ALIGNED() \
+ { \
+ long double __tmp; \
+ ASSERT((((uintptr_t)&__tmp) & (STACK_ALIGN - 1)) == 0); \
+ }
+#endif
+#else /* DEBUG */
+#define ASSERT_STACK_ALIGNED()
+#endif /* DEBUG */
+
+struct regs;
+
+void traceregs(struct regs *);
+void traceback(caddr_t);
+
+#endif /* defined(_KERNEL) && !defined(_ASM) */
+
+#define STACK_GROWTH_DOWN /* stacks grow from high to low addresses */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _IA32_SYS_STACK_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
+/* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
+/* All Rights Reserved */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _IA32_SYS_TRAP_H
+#define _IA32_SYS_TRAP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Trap type values
+ */
+
+#define T_ZERODIV 0x0 /* #de divide by 0 error */
+#define T_SGLSTP 0x1 /* #db single step */
+#define T_NMIFLT 0x2 /* NMI */
+#define T_BPTFLT 0x3 /* #bp breakpoint fault, INT3 insn */
+#define T_OVFLW 0x4 /* #of INTO overflow fault */
+#define T_BOUNDFLT 0x5 /* #br BOUND insn fault */
+#define T_ILLINST 0x6 /* #ud invalid opcode fault */
+#define T_NOEXTFLT 0x7 /* #nm device not available: x87 */
+#define T_DBLFLT 0x8 /* #df double fault */
+#define T_EXTOVRFLT 0x9 /* [not generated: 386 only] */
+#define T_TSSFLT 0xa /* #ts invalid TSS fault */
+#define T_SEGFLT 0xb /* #np segment not present fault */
+#define T_STKFLT 0xc /* #ss stack fault */
+#define T_GPFLT 0xd /* #gp general protection fault */
+#define T_PGFLT 0xe /* #pf page fault */
+#define T_EXTERRFLT 0x10 /* #mf x87 FPU error fault */
+#define T_ALIGNMENT 0x11 /* #ac alignment check error */
+#define T_MCE 0x12 /* #mc machine check exception */
+#define T_SIMDFPE 0x13 /* #xm SSE/SSE exception */
+#define T_DBGENTR 0x14 /* debugger entry */
+#define T_ENDPERR 0x21 /* emulated extension error flt */
+#define T_ENOEXTFLT 0x20 /* emulated ext not present */
+#define T_FASTTRAP 0xd2 /* fast system call */
+#define T_SYSCALLINT 0x91 /* general system call */
+#define T_DTRACE_RET 0x7f /* DTrace pid return */
+#define T_INT80 0x80 /* int80 handler for linux emulation */
+#define T_SOFTINT 0x50fd /* pseudo softint trap type */
+
+/*
+ * Pseudo traps.
+ */
+#define T_INTERRUPT 0x100
+#define T_FAULT 0x200
+#define T_AST 0x400
+#define T_SYSCALL 0x180
+
+
+/*
+ * Values of error code on stack in case of page fault
+ */
+
+#define PF_ERR_MASK 0x01 /* Mask for error bit */
+#define PF_ERR_PAGE 0x00 /* page not present */
+#define PF_ERR_PROT 0x01 /* protection error */
+#define PF_ERR_WRITE 0x02 /* fault caused by write (else read) */
+#define PF_ERR_USER 0x04 /* processor was in user mode */
+ /* (else supervisor) */
+#define PF_ERR_EXEC 0x10 /* attempt to execute a No eXec page (AMD) */
+
+/*
+ * Definitions for fast system call subfunctions
+ */
+#define T_FNULL 0 /* Null trap for testing */
+#define T_FGETFP 1 /* Get emulated FP context */
+#define T_FSETFP 2 /* Set emulated FP context */
+#define T_GETHRTIME 3 /* Get high resolution time */
+#define T_GETHRVTIME 4 /* Get high resolution virtual time */
+#define T_GETHRESTIME 5 /* Get high resolution time */
+#define T_GETLGRP 6 /* Get home lgrpid */
+
+#define T_LASTFAST 6 /* Last valid subfunction */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _IA32_SYS_TRAP_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_MODCTL_H
+#define _SYS_MODCTL_H
+
+/*
+ * loadable module support.
+ */
+
+#include <sys/zfs_context.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct modlmisc;
+struct modlinkage;
+
+/*
+ * The following structure defines the operations used by modctl
+ * to load and unload modules. Each supported loadable module type
+ * requires a set of mod_ops.
+ */
+struct mod_ops {
+ int (*modm_install)(struct modlmisc *, struct modlinkage *);
+ int (*modm_remove)(struct modlmisc *, struct modlinkage *);
+ int (*modm_info)(void *, struct modlinkage *, int *);
+};
+
+/*
+ * The defined set of mod_ops structures for each loadable module type
+ * Defined in modctl.c
+ */
+extern struct mod_ops mod_brandops;
+#if defined(__i386) || defined(__amd64)
+extern struct mod_ops mod_cpuops;
+#endif
+extern struct mod_ops mod_cryptoops;
+extern struct mod_ops mod_driverops;
+extern struct mod_ops mod_execops;
+extern struct mod_ops mod_fsops;
+extern struct mod_ops mod_miscops;
+extern struct mod_ops mod_schedops;
+extern struct mod_ops mod_strmodops;
+extern struct mod_ops mod_syscallops;
+extern struct mod_ops mod_sockmodops;
+#ifdef _SYSCALL32_IMPL
+extern struct mod_ops mod_syscallops32;
+#endif
+extern struct mod_ops mod_dacfops;
+extern struct mod_ops mod_ippops;
+extern struct mod_ops mod_pcbeops;
+extern struct mod_ops mod_devfsops;
+extern struct mod_ops mod_kiconvops;
+
+/*
+ * Definitions for the module specific linkage structures.
+ * The first two fields are the same in all of the structures.
+ * The linkinfo is for informational purposes only and is returned by
+ * modctl with the MODINFO cmd.
+ */
+
+/* For cryptographic providers */
+struct modlcrypto {
+ struct mod_ops *crypto_modops;
+ char *crypto_linkinfo;
+};
+
+/* For misc */
+struct modlmisc {
+ struct mod_ops *misc_modops;
+ char *misc_linkinfo;
+};
+
+/*
+ * Revision number of loadable modules support. This is the value
+ * that must be used in the modlinkage structure.
+ */
+#define MODREV_1 1
+
+/*
+ * The modlinkage structure is the structure that the module writer
+ * provides to the routines to install, remove, and stat a module.
+ * The ml_linkage element is an array of pointers to linkage structures.
+ * For most modules there is only one linkage structure. We allocate
+ * enough space for 3 linkage structures which happens to be the most
+ * we have in any sun supplied module. For those modules with more
+ * than 3 linkage structures (which is very unlikely), a modlinkage
+ * structure must be kmem_alloc'd in the module wrapper to be big enough
+ * for all of the linkage structures.
+ */
+struct modlinkage {
+ int ml_rev; /* rev of loadable modules system */
+#ifdef _LP64
+ void *ml_linkage[7]; /* more space in 64-bit OS */
+#else
+ void *ml_linkage[4]; /* NULL terminated list of */
+ /* linkage structures */
+#endif
+};
+
+/*
+ * commands. These are the commands supported by the modctl system call.
+ */
+#define MODLOAD 0
+#define MODUNLOAD 1
+#define MODINFO 2
+#define MODRESERVED 3
+#define MODSETMINIROOT 4
+#define MODADDMAJBIND 5
+#define MODGETPATH 6
+#define MODREADSYSBIND 7
+#define MODGETMAJBIND 8
+#define MODGETNAME 9
+#define MODSIZEOF_DEVID 10
+#define MODGETDEVID 11
+#define MODSIZEOF_MINORNAME 12
+#define MODGETMINORNAME 13
+#define MODGETPATHLEN 14
+#define MODEVENTS 15
+#define MODGETFBNAME 16
+#define MODREREADDACF 17
+#define MODLOADDRVCONF 18
+#define MODUNLOADDRVCONF 19
+#define MODREMMAJBIND 20
+#define MODDEVT2INSTANCE 21
+#define MODGETDEVFSPATH_LEN 22
+#define MODGETDEVFSPATH 23
+#define MODDEVID2PATHS 24
+#define MODSETDEVPOLICY 26
+#define MODGETDEVPOLICY 27
+#define MODALLOCPRIV 28
+#define MODGETDEVPOLICYBYNAME 29
+#define MODLOADMINORPERM 31
+#define MODADDMINORPERM 32
+#define MODREMMINORPERM 33
+#define MODREMDRVCLEANUP 34
+#define MODDEVEXISTS 35
+#define MODDEVREADDIR 36
+#define MODDEVNAME 37
+#define MODGETDEVFSPATH_MI_LEN 38
+#define MODGETDEVFSPATH_MI 39
+#define MODRETIRE 40
+#define MODUNRETIRE 41
+#define MODISRETIRED 42
+#define MODDEVEMPTYDIR 43
+#define MODREMDRVALIAS 44
+
+/*
+ * sub cmds for MODEVENTS
+ */
+#define MODEVENTS_FLUSH 0
+#define MODEVENTS_FLUSH_DUMP 1
+#define MODEVENTS_SET_DOOR_UPCALL_FILENAME 2
+#define MODEVENTS_GETDATA 3
+#define MODEVENTS_FREEDATA 4
+#define MODEVENTS_POST_EVENT 5
+#define MODEVENTS_REGISTER_EVENT 6
+
+/*
+ * devname subcmds for MODDEVNAME
+ */
+#define MODDEVNAME_LOOKUPDOOR 0
+#define MODDEVNAME_DEVFSADMNODE 1
+#define MODDEVNAME_NSMAPS 2
+#define MODDEVNAME_PROFILE 3
+#define MODDEVNAME_RECONFIG 4
+#define MODDEVNAME_SYSAVAIL 5
+
+
+/*
+ * Data structure passed to modconfig command in kernel to build devfs tree
+ */
+
+struct aliases {
+ struct aliases *a_next;
+ char *a_name;
+ int a_len;
+};
+
+#define MAXMODCONFNAME 256
+
+struct modconfig {
+ char drvname[MAXMODCONFNAME];
+ char drvclass[MAXMODCONFNAME];
+ int major;
+ int flags;
+ int num_aliases;
+ struct aliases *ap;
+};
+
+#if defined(_SYSCALL32)
+
+struct aliases32 {
+ caddr32_t a_next;
+ caddr32_t a_name;
+ int32_t a_len;
+};
+
+struct modconfig32 {
+ char drvname[MAXMODCONFNAME];
+ char drvclass[MAXMODCONFNAME];
+ int32_t major;
+ int32_t flags;
+ int32_t num_aliases;
+ caddr32_t ap;
+};
+
+#endif /* _SYSCALL32 */
+
+/* flags for modconfig */
+#define MOD_UNBIND_OVERRIDE 0x01 /* fail unbind if in use */
+
+/*
+ * Max module path length
+ */
+#define MOD_MAXPATH 256
+
+/*
+ * Default search path for modules ADDITIONAL to the directory
+ * where the kernel components we booted from are.
+ *
+ * Most often, this will be "/platform/{platform}/kernel /kernel /usr/kernel",
+ * but we don't wire it down here.
+ */
+#define MOD_DEFPATH "/kernel /usr/kernel"
+
+/*
+ * Default file name extension for autoloading modules.
+ */
+#define MOD_DEFEXT ""
+
+/*
+ * Parameters for modinfo
+ */
+#define MODMAXNAMELEN 32 /* max module name length */
+#define MODMAXLINKINFOLEN 32 /* max link info length */
+
+/*
+ * Module specific information.
+ */
+struct modspecific_info {
+ char msi_linkinfo[MODMAXLINKINFOLEN]; /* name in linkage struct */
+ int msi_p0; /* module specific information */
+};
+
+/*
+ * Structure returned by modctl with MODINFO command.
+ */
+#define MODMAXLINK 10 /* max linkages modinfo can handle */
+
+struct modinfo {
+ int mi_info; /* Flags for info wanted */
+ int mi_state; /* Flags for module state */
+ int mi_id; /* id of this loaded module */
+ int mi_nextid; /* id of next module or -1 */
+ caddr_t mi_base; /* virtual addr of text */
+ size_t mi_size; /* size of module in bytes */
+ int mi_rev; /* loadable modules rev */
+ int mi_loadcnt; /* # of times loaded */
+ char mi_name[MODMAXNAMELEN]; /* name of module */
+ struct modspecific_info mi_msinfo[MODMAXLINK];
+ /* mod specific info */
+};
+
+
+#if defined(_SYSCALL32)
+
+#define MODMAXNAMELEN32 32 /* max module name length */
+#define MODMAXLINKINFOLEN32 32 /* max link info length */
+#define MODMAXLINK32 10 /* max linkages modinfo can handle */
+
+struct modspecific_info32 {
+ char msi_linkinfo[MODMAXLINKINFOLEN32]; /* name in linkage struct */
+ int32_t msi_p0; /* module specific information */
+};
+
+struct modinfo32 {
+ int32_t mi_info; /* Flags for info wanted */
+ int32_t mi_state; /* Flags for module state */
+ int32_t mi_id; /* id of this loaded module */
+ int32_t mi_nextid; /* id of next module or -1 */
+ caddr32_t mi_base; /* virtual addr of text */
+ uint32_t mi_size; /* size of module in bytes */
+ int32_t mi_rev; /* loadable modules rev */
+ int32_t mi_loadcnt; /* # of times loaded */
+ char mi_name[MODMAXNAMELEN32]; /* name of module */
+ struct modspecific_info32 mi_msinfo[MODMAXLINK32];
+ /* mod specific info */
+};
+
+#endif /* _SYSCALL32 */
+
+/* Values for mi_info flags */
+#define MI_INFO_ONE 1
+#define MI_INFO_ALL 2
+#define MI_INFO_CNT 4
+#define MI_INFO_LINKAGE 8 /* used internally to extract modlinkage */
+/*
+ * MI_INFO_NOBASE indicates caller does not need mi_base. Failure to use this
+ * flag may lead 32-bit apps to receive an EOVERFLOW error from modctl(MODINFO)
+ * when used with a 64-bit kernel.
+ */
+#define MI_INFO_NOBASE 16
+
+/* Values for mi_state */
+#define MI_LOADED 1
+#define MI_INSTALLED 2
+
+/*
+ * Macros to vector to the appropriate module specific routine.
+ */
+#define MODL_INSTALL(MODL, MODLP) \
+ (*(MODL)->misc_modops->modm_install)(MODL, MODLP)
+#define MODL_REMOVE(MODL, MODLP) \
+ (*(MODL)->misc_modops->modm_remove)(MODL, MODLP)
+#define MODL_INFO(MODL, MODLP, P0) \
+ (*(MODL)->misc_modops->modm_info)(MODL, MODLP, P0)
+
+/*
+ * Definitions for stubs
+ */
+struct mod_stub_info {
+ uintptr_t mods_func_adr;
+ struct mod_modinfo *mods_modinfo;
+ uintptr_t mods_stub_adr;
+ int (*mods_errfcn)(void);
+ int mods_flag; /* flags defined below */
+};
+
+/*
+ * Definitions for mods_flag.
+ */
+#define MODS_WEAK 0x01 /* weak stub (not loaded if called) */
+#define MODS_NOUNLOAD 0x02 /* module not unloadable (no _fini()) */
+#define MODS_INSTALLED 0x10 /* module installed */
+
+struct mod_modinfo {
+ char *modm_module_name;
+ struct modctl *mp;
+ struct mod_stub_info modm_stubs[1];
+};
+
+struct modctl_list {
+ struct modctl_list *modl_next;
+ struct modctl *modl_modp;
+};
+
+/*
+ * Structure to manage a loadable module.
+ * Note: the module (mod_mp) structure's "text" and "text_size" information
+ * are replicated in the modctl structure so that mod_containing_pc()
+ * doesn't have to grab any locks (modctls are persistent; modules are not.)
+ */
+typedef struct modctl {
+ struct modctl *mod_next; /* &modules based list */
+ struct modctl *mod_prev;
+ int mod_id;
+ void *mod_mp;
+ kthread_t *mod_inprogress_thread;
+ struct mod_modinfo *mod_modinfo;
+ struct modlinkage *mod_linkage;
+ char *mod_filename;
+ char *mod_modname;
+
+ char mod_busy; /* inprogress_thread has locked */
+ char mod_want; /* someone waiting for unlock */
+ char mod_prim; /* primary module */
+
+ int mod_ref; /* ref count - from dependent or stub */
+
+ char mod_loaded; /* module in memory */
+ char mod_installed; /* post _init pre _fini */
+ char mod_loadflags;
+ char mod_delay_unload; /* deferred unload */
+
+ struct modctl_list *mod_requisites; /* mods this one depends on. */
+ void *__unused; /* NOTE: reuse (same size) is OK, */
+ /* deletion causes mdb.vs.core issues */
+ int mod_loadcnt; /* number of times mod was loaded */
+ int mod_nenabled; /* # of enabled DTrace probes in mod */
+ char *mod_text;
+ size_t mod_text_size;
+
+ int mod_gencount; /* # times loaded/unloaded */
+ struct modctl *mod_requisite_loading; /* mod circular dependency */
+} modctl_t;
+
+/*
+ * mod_loadflags
+ */
+
+#define MOD_NOAUTOUNLOAD 0x1 /* Auto mod-unloader skips this mod */
+#define MOD_NONOTIFY 0x2 /* No krtld notifications on (un)load */
+#define MOD_NOUNLOAD 0x4 /* Assume EBUSY for all _fini's */
+
+#define MOD_BIND_HASHSIZE 64
+#define MOD_BIND_HASHMASK (MOD_BIND_HASHSIZE-1)
+
+typedef int modid_t;
+
+/*
+ * global function and data declarations
+ */
+extern kmutex_t mod_lock;
+
+extern char *systemfile;
+extern char **syscallnames;
+extern int moddebug;
+
+/*
+ * this is the head of a doubly linked list. Only the next and prev
+ * pointers are used
+ */
+extern modctl_t modules;
+
+/*
+ * Only the following are part of the DDI/DKI
+ */
+extern int mod_install(struct modlinkage *);
+extern int mod_remove(struct modlinkage *);
+extern int mod_info(struct modlinkage *, struct modinfo *);
+
+/*
+ * bit definitions for moddebug.
+ */
+#define MODDEBUG_LOADMSG 0x80000000 /* print "[un]loading..." msg */
+#define MODDEBUG_ERRMSG 0x40000000 /* print detailed error msgs */
+#define MODDEBUG_LOADMSG2 0x20000000 /* print 2nd level msgs */
+#define MODDEBUG_RETIRE 0x10000000 /* print retire msgs */
+#define MODDEBUG_BINDING 0x00040000 /* driver/alias binding */
+#define MODDEBUG_FINI_EBUSY 0x00020000 /* pretend fini returns EBUSY */
+#define MODDEBUG_NOAUL_IPP 0x00010000 /* no Autounloading ipp mods */
+#define MODDEBUG_NOAUL_DACF 0x00008000 /* no Autounloading dacf mods */
+#define MODDEBUG_KEEPTEXT 0x00004000 /* keep text after unloading */
+#define MODDEBUG_NOAUL_DRV 0x00001000 /* no Autounloading Drivers */
+#define MODDEBUG_NOAUL_EXEC 0x00000800 /* no Autounloading Execs */
+#define MODDEBUG_NOAUL_FS 0x00000400 /* no Autounloading File sys */
+#define MODDEBUG_NOAUL_MISC 0x00000200 /* no Autounloading misc */
+#define MODDEBUG_NOAUL_SCHED 0x00000100 /* no Autounloading scheds */
+#define MODDEBUG_NOAUL_STR 0x00000080 /* no Autounloading streams */
+#define MODDEBUG_NOAUL_SYS 0x00000040 /* no Autounloading syscalls */
+#define MODDEBUG_NOCTF 0x00000020 /* do not load CTF debug data */
+#define MODDEBUG_NOAUTOUNLOAD 0x00000010 /* no autounloading at all */
+#define MODDEBUG_DDI_MOD 0x00000008 /* ddi_mod{open,sym,close} */
+#define MODDEBUG_MP_MATCH 0x00000004 /* dev_minorperm */
+#define MODDEBUG_MINORPERM 0x00000002 /* minor perm modctls */
+#define MODDEBUG_USERDEBUG 0x00000001 /* bpt after init_module() */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_MODCTL_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_MODHASH_H
+#define _SYS_MODHASH_H
+
+/*
+ * Generic hash implementation for the kernel.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+
+/*
+ * Opaque data types for storing keys and values
+ */
+typedef void *mod_hash_val_t;
+typedef void *mod_hash_key_t;
+
+/*
+ * Opaque data type for reservation
+ */
+typedef void *mod_hash_hndl_t;
+
+/*
+ * Opaque type for hash itself.
+ */
+struct mod_hash;
+typedef struct mod_hash mod_hash_t;
+
+/*
+ * String hash table
+ */
+mod_hash_t *mod_hash_create_strhash_nodtr(char *, size_t,
+ void (*)(mod_hash_val_t));
+mod_hash_t *mod_hash_create_strhash(char *, size_t, void (*)(mod_hash_val_t));
+void mod_hash_destroy_strhash(mod_hash_t *);
+int mod_hash_strkey_cmp(mod_hash_key_t, mod_hash_key_t);
+void mod_hash_strkey_dtor(mod_hash_key_t);
+void mod_hash_strval_dtor(mod_hash_val_t);
+uint_t mod_hash_bystr(void *, mod_hash_key_t);
+
+/*
+ * Pointer hash table
+ */
+mod_hash_t *mod_hash_create_ptrhash(char *, size_t, void (*)(mod_hash_val_t),
+ size_t);
+void mod_hash_destroy_ptrhash(mod_hash_t *);
+int mod_hash_ptrkey_cmp(mod_hash_key_t, mod_hash_key_t);
+uint_t mod_hash_byptr(void *, mod_hash_key_t);
+
+/*
+ * ID hash table
+ */
+mod_hash_t *mod_hash_create_idhash(char *, size_t, void (*)(mod_hash_val_t));
+void mod_hash_destroy_idhash(mod_hash_t *);
+int mod_hash_idkey_cmp(mod_hash_key_t, mod_hash_key_t);
+uint_t mod_hash_byid(void *, mod_hash_key_t);
+uint_t mod_hash_iddata_gen(size_t);
+
+/*
+ * Hash management functions
+ */
+mod_hash_t *mod_hash_create_extended(char *, size_t, void (*)(mod_hash_key_t),
+ void (*)(mod_hash_val_t), uint_t (*)(void *, mod_hash_key_t), void *,
+ int (*)(mod_hash_key_t, mod_hash_key_t), int);
+
+void mod_hash_destroy_hash(mod_hash_t *);
+void mod_hash_clear(mod_hash_t *);
+
+/*
+ * Null key and value destructors
+ */
+void mod_hash_null_keydtor(mod_hash_key_t);
+void mod_hash_null_valdtor(mod_hash_val_t);
+
+/*
+ * Basic hash operations
+ */
+
+/*
+ * Error codes for insert, remove, find, destroy.
+ */
+#define MH_ERR_NOMEM -1
+#define MH_ERR_DUPLICATE -2
+#define MH_ERR_NOTFOUND -3
+
+/*
+ * Return codes for hash walkers
+ */
+#define MH_WALK_CONTINUE 0
+#define MH_WALK_TERMINATE 1
+
+/*
+ * Basic hash operations
+ */
+int mod_hash_insert(mod_hash_t *, mod_hash_key_t, mod_hash_val_t);
+int mod_hash_replace(mod_hash_t *, mod_hash_key_t, mod_hash_val_t);
+int mod_hash_remove(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *);
+int mod_hash_destroy(mod_hash_t *, mod_hash_key_t);
+int mod_hash_find(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *);
+int mod_hash_find_cb(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *,
+ void (*)(mod_hash_key_t, mod_hash_val_t));
+int mod_hash_find_cb_rval(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *,
+ int (*)(mod_hash_key_t, mod_hash_val_t), int *);
+void mod_hash_walk(mod_hash_t *,
+ uint_t (*)(mod_hash_key_t, mod_hash_val_t *, void *), void *);
+
+/*
+ * Reserving hash operations
+ */
+int mod_hash_reserve(mod_hash_t *, mod_hash_hndl_t *);
+int mod_hash_reserve_nosleep(mod_hash_t *, mod_hash_hndl_t *);
+void mod_hash_cancel(mod_hash_t *, mod_hash_hndl_t *);
+int mod_hash_insert_reserve(mod_hash_t *, mod_hash_key_t, mod_hash_val_t,
+ mod_hash_hndl_t);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_MODHASH_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_MODHASH_IMPL_H
+#define _SYS_MODHASH_IMPL_H
+
+/*
+ * Internal details for the kernel's generic hash implementation.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+#include <sys/modhash.h>
+
+struct mod_hash_entry {
+ mod_hash_key_t mhe_key; /* stored hash key */
+ mod_hash_val_t mhe_val; /* stored hash value */
+ struct mod_hash_entry *mhe_next; /* next item in chain */
+};
+
+struct mod_hash_stat {
+ ulong_t mhs_hit; /* tried a 'find' and it succeeded */
+ ulong_t mhs_miss; /* tried a 'find' but it failed */
+ ulong_t mhs_coll; /* occur when insert fails because of dup's */
+ ulong_t mhs_nelems; /* total number of stored key/value pairs */
+ ulong_t mhs_nomem; /* number of times kmem_alloc failed */
+};
+
+struct mod_hash {
+ krwlock_t mh_contents; /* lock protecting contents */
+ char *mh_name; /* hash name */
+ int mh_sleep; /* kmem_alloc flag */
+ size_t mh_nchains; /* # of elements in mh_entries */
+
+ /* key and val destructor */
+ void (*mh_kdtor)(mod_hash_key_t);
+ void (*mh_vdtor)(mod_hash_val_t);
+
+ /* key comparator */
+ int (*mh_keycmp)(mod_hash_key_t, mod_hash_key_t);
+
+ /* hash algorithm, and algorithm-private data */
+ uint_t (*mh_hashalg)(void *, mod_hash_key_t);
+ void *mh_hashalg_data;
+
+ struct mod_hash *mh_next; /* next hash in list */
+
+ struct mod_hash_stat mh_stat;
+
+ struct mod_hash_entry *mh_entries[1];
+};
+
+/*
+ * MH_SIZE()
+ * Compute the size of a mod_hash_t, in bytes, given the number of
+ * elements it contains.
+ */
+#define MH_SIZE(n) \
+ (sizeof (mod_hash_t) + ((n) - 1) * (sizeof (struct mod_hash_entry *)))
+
+/*
+ * Module initialization; called once.
+ */
+void mod_hash_fini(void);
+void mod_hash_init(void);
+
+/*
+ * Internal routines. Use directly with care.
+ */
+uint_t i_mod_hash(mod_hash_t *, mod_hash_key_t);
+int i_mod_hash_insert_nosync(mod_hash_t *, mod_hash_key_t, mod_hash_val_t,
+ mod_hash_hndl_t);
+int i_mod_hash_remove_nosync(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *);
+int i_mod_hash_find_nosync(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *);
+void i_mod_hash_walk_nosync(mod_hash_t *, uint_t (*)(mod_hash_key_t,
+ mod_hash_val_t *, void *), void *);
+void i_mod_hash_clear_nosync(mod_hash_t *hash);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_MODHASH_IMPL_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_STACK_H
+#define _SYS_STACK_H
+
+#if defined(__i386) || defined(__amd64)
+
+#include <sys/ia32/stack.h> /* XX64 x86/sys/stack.h */
+
+#endif
+
+#endif /* _SYS_STACK_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_TRAP_H
+#define _SYS_TRAP_H
+
+#if defined(__i386) || defined(__amd64)
+
+#include <sys/ia32/trap.h> /* XX64 x86/sys/trap.h */
+
+#endif
+
+#endif /* _SYS_TRAP_H */
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+
+/*
+ * AES provider for the Kernel Cryptographic Framework (KCF)
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/spi.h>
+#include <sys/crypto/icp.h>
+#include <modes/modes.h>
+#include <sys/modctl.h>
+#define _AES_IMPL
+#include <aes/aes_impl.h>
+
+#define CRYPTO_PROVIDER_NAME "aes"
+
+extern struct mod_ops mod_cryptoops;
+
+/*
+ * Module linkage information for the kernel.
+ */
+static struct modlcrypto modlcrypto = {
+ &mod_cryptoops,
+ "AES Kernel SW Provider"
+};
+
+static struct modlinkage modlinkage = {
+ MODREV_1, { (void *)&modlcrypto, NULL }
+};
+
+/*
+ * Mechanism info structure passed to KCF during registration.
+ */
+static crypto_mech_info_t aes_mech_info_tab[] = {
+ /* AES_ECB */
+ {SUN_CKM_AES_ECB, AES_ECB_MECH_INFO_TYPE,
+ CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
+ CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC,
+ AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
+ /* AES_CBC */
+ {SUN_CKM_AES_CBC, AES_CBC_MECH_INFO_TYPE,
+ CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
+ CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC,
+ AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
+ /* AES_CTR */
+ {SUN_CKM_AES_CTR, AES_CTR_MECH_INFO_TYPE,
+ CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
+ CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC,
+ AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
+ /* AES_CCM */
+ {SUN_CKM_AES_CCM, AES_CCM_MECH_INFO_TYPE,
+ CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
+ CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC,
+ AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
+ /* AES_GCM */
+ {SUN_CKM_AES_GCM, AES_GCM_MECH_INFO_TYPE,
+ CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
+ CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC,
+ AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
+ /* AES_GMAC */
+ {SUN_CKM_AES_GMAC, AES_GMAC_MECH_INFO_TYPE,
+ CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
+ CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
+ CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC |
+ CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
+ CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
+ AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}
+};
+
+/* operations are in-place if the output buffer is NULL */
+#define AES_ARG_INPLACE(input, output) \
+ if ((output) == NULL) \
+ (output) = (input);
+
+static void aes_provider_status(crypto_provider_handle_t, uint_t *);
+
+static crypto_control_ops_t aes_control_ops = {
+ aes_provider_status
+};
+
+static int aes_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
+ crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
+static int aes_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
+ crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
+static int aes_common_init(crypto_ctx_t *, crypto_mechanism_t *,
+ crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t, boolean_t);
+static int aes_common_init_ctx(aes_ctx_t *, crypto_spi_ctx_template_t *,
+ crypto_mechanism_t *, crypto_key_t *, int, boolean_t);
+static int aes_encrypt_final(crypto_ctx_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int aes_decrypt_final(crypto_ctx_t *, crypto_data_t *,
+ crypto_req_handle_t);
+
+static int aes_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int aes_encrypt_update(crypto_ctx_t *, crypto_data_t *,
+ crypto_data_t *, crypto_req_handle_t);
+static int aes_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
+
+static int aes_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int aes_decrypt_update(crypto_ctx_t *, crypto_data_t *,
+ crypto_data_t *, crypto_req_handle_t);
+static int aes_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
+ crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
+
+static crypto_cipher_ops_t aes_cipher_ops = {
+ aes_encrypt_init,
+ aes_encrypt,
+ aes_encrypt_update,
+ aes_encrypt_final,
+ aes_encrypt_atomic,
+ aes_decrypt_init,
+ aes_decrypt,
+ aes_decrypt_update,
+ aes_decrypt_final,
+ aes_decrypt_atomic
+};
+
+static int aes_mac_atomic(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+static int aes_mac_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+
+static crypto_mac_ops_t aes_mac_ops = {
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ aes_mac_atomic,
+ aes_mac_verify_atomic
+};
+
+static int aes_create_ctx_template(crypto_provider_handle_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t *,
+ size_t *, crypto_req_handle_t);
+static int aes_free_context(crypto_ctx_t *);
+
+static crypto_ctx_ops_t aes_ctx_ops = {
+ aes_create_ctx_template,
+ aes_free_context
+};
+
+static crypto_ops_t aes_crypto_ops = {{{{{
+ &aes_control_ops,
+ NULL,
+ &aes_cipher_ops,
+ &aes_mac_ops,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ &aes_ctx_ops
+}}}}};
+
+static crypto_provider_info_t aes_prov_info = {{{{
+ CRYPTO_SPI_VERSION_1,
+ "AES Software Provider",
+ CRYPTO_SW_PROVIDER,
+ NULL,
+ &aes_crypto_ops,
+ sizeof (aes_mech_info_tab)/sizeof (crypto_mech_info_t),
+ aes_mech_info_tab
+}}}};
+
+static crypto_kcf_provider_handle_t aes_prov_handle = 0;
+static crypto_data_t null_crypto_data = { CRYPTO_DATA_RAW };
+
+int
+aes_mod_init(void)
+{
+ int ret;
+
+ if ((ret = mod_install(&modlinkage)) != 0)
+ return (ret);
+
+ /* Register with KCF. If the registration fails, remove the module. */
+ if (crypto_register_provider(&aes_prov_info, &aes_prov_handle)) {
+ (void) mod_remove(&modlinkage);
+ return (EACCES);
+ }
+
+ return (0);
+}
+
+int
+aes_mod_fini(void)
+{
+ /* Unregister from KCF if module is registered */
+ if (aes_prov_handle != 0) {
+ if (crypto_unregister_provider(aes_prov_handle))
+ return (EBUSY);
+
+ aes_prov_handle = 0;
+ }
+
+ return (mod_remove(&modlinkage));
+}
+
+static int
+aes_check_mech_param(crypto_mechanism_t *mechanism, aes_ctx_t **ctx, int kmflag)
+{
+ void *p = NULL;
+ boolean_t param_required = B_TRUE;
+ size_t param_len;
+ void *(*alloc_fun)(int);
+ int rv = CRYPTO_SUCCESS;
+
+ switch (mechanism->cm_type) {
+ case AES_ECB_MECH_INFO_TYPE:
+ param_required = B_FALSE;
+ alloc_fun = ecb_alloc_ctx;
+ break;
+ case AES_CBC_MECH_INFO_TYPE:
+ param_len = AES_BLOCK_LEN;
+ alloc_fun = cbc_alloc_ctx;
+ break;
+ case AES_CTR_MECH_INFO_TYPE:
+ param_len = sizeof (CK_AES_CTR_PARAMS);
+ alloc_fun = ctr_alloc_ctx;
+ break;
+ case AES_CCM_MECH_INFO_TYPE:
+ param_len = sizeof (CK_AES_CCM_PARAMS);
+ alloc_fun = ccm_alloc_ctx;
+ break;
+ case AES_GCM_MECH_INFO_TYPE:
+ param_len = sizeof (CK_AES_GCM_PARAMS);
+ alloc_fun = gcm_alloc_ctx;
+ break;
+ case AES_GMAC_MECH_INFO_TYPE:
+ param_len = sizeof (CK_AES_GMAC_PARAMS);
+ alloc_fun = gmac_alloc_ctx;
+ break;
+ default:
+ rv = CRYPTO_MECHANISM_INVALID;
+ return (rv);
+ }
+ if (param_required && mechanism->cm_param != NULL &&
+ mechanism->cm_param_len != param_len) {
+ rv = CRYPTO_MECHANISM_PARAM_INVALID;
+ }
+ if (ctx != NULL) {
+ p = (alloc_fun)(kmflag);
+ *ctx = p;
+ }
+ return (rv);
+}
+
+/*
+ * Initialize key schedules for AES
+ */
+static int
+init_keysched(crypto_key_t *key, void *newbie)
+{
+ /*
+ * Only keys by value are supported by this module.
+ */
+ switch (key->ck_format) {
+ case CRYPTO_KEY_RAW:
+ if (key->ck_length < AES_MINBITS ||
+ key->ck_length > AES_MAXBITS) {
+ return (CRYPTO_KEY_SIZE_RANGE);
+ }
+
+ /* key length must be either 128, 192, or 256 */
+ if ((key->ck_length & 63) != 0)
+ return (CRYPTO_KEY_SIZE_RANGE);
+ break;
+ default:
+ return (CRYPTO_KEY_TYPE_INCONSISTENT);
+ }
+
+ aes_init_keysched(key->ck_data, key->ck_length, newbie);
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * KCF software provider control entry points.
+ */
+/* ARGSUSED */
+static void
+aes_provider_status(crypto_provider_handle_t provider, uint_t *status)
+{
+ *status = CRYPTO_PROVIDER_READY;
+}
+
+static int
+aes_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_spi_ctx_template_t template,
+ crypto_req_handle_t req) {
+ return (aes_common_init(ctx, mechanism, key, template, req, B_TRUE));
+}
+
+static int
+aes_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_spi_ctx_template_t template,
+ crypto_req_handle_t req) {
+ return (aes_common_init(ctx, mechanism, key, template, req, B_FALSE));
+}
+
+
+
+/*
+ * KCF software provider encrypt entry points.
+ */
+static int
+aes_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_spi_ctx_template_t template,
+ crypto_req_handle_t req, boolean_t is_encrypt_init)
+{
+ aes_ctx_t *aes_ctx;
+ int rv;
+ int kmflag;
+
+ /*
+ * Only keys by value are supported by this module.
+ */
+ if (key->ck_format != CRYPTO_KEY_RAW) {
+ return (CRYPTO_KEY_TYPE_INCONSISTENT);
+ }
+
+ kmflag = crypto_kmflag(req);
+ if ((rv = aes_check_mech_param(mechanism, &aes_ctx, kmflag))
+ != CRYPTO_SUCCESS)
+ return (rv);
+
+ rv = aes_common_init_ctx(aes_ctx, template, mechanism, key, kmflag,
+ is_encrypt_init);
+ if (rv != CRYPTO_SUCCESS) {
+ crypto_free_mode_ctx(aes_ctx);
+ return (rv);
+ }
+
+ ctx->cc_provider_private = aes_ctx;
+
+ return (CRYPTO_SUCCESS);
+}
+
+static void
+aes_copy_block64(uint8_t *in, uint64_t *out)
+{
+ if (IS_P2ALIGNED(in, sizeof (uint64_t))) {
+ /* LINTED: pointer alignment */
+ out[0] = *(uint64_t *)&in[0];
+ /* LINTED: pointer alignment */
+ out[1] = *(uint64_t *)&in[8];
+ } else {
+ uint8_t *iv8 = (uint8_t *)&out[0];
+
+ AES_COPY_BLOCK(in, iv8);
+ }
+}
+
+
+static int
+aes_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
+ crypto_data_t *ciphertext, crypto_req_handle_t req)
+{
+ int ret = CRYPTO_FAILED;
+
+ aes_ctx_t *aes_ctx;
+ size_t saved_length, saved_offset, length_needed;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+ aes_ctx = ctx->cc_provider_private;
+
+ /*
+ * For block ciphers, plaintext must be a multiple of AES block size.
+ * This test is only valid for ciphers whose blocksize is a power of 2.
+ */
+ if (((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE))
+ == 0) && (plaintext->cd_length & (AES_BLOCK_LEN - 1)) != 0)
+ return (CRYPTO_DATA_LEN_RANGE);
+
+ AES_ARG_INPLACE(plaintext, ciphertext);
+
+ /*
+ * We need to just return the length needed to store the output.
+ * We should not destroy the context for the following case.
+ */
+ switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) {
+ case CCM_MODE:
+ length_needed = plaintext->cd_length + aes_ctx->ac_mac_len;
+ break;
+ case GCM_MODE:
+ length_needed = plaintext->cd_length + aes_ctx->ac_tag_len;
+ break;
+ case GMAC_MODE:
+ if (plaintext->cd_length != 0)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ length_needed = aes_ctx->ac_tag_len;
+ break;
+ default:
+ length_needed = plaintext->cd_length;
+ }
+
+ if (ciphertext->cd_length < length_needed) {
+ ciphertext->cd_length = length_needed;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ saved_length = ciphertext->cd_length;
+ saved_offset = ciphertext->cd_offset;
+
+ /*
+ * Do an update on the specified input data.
+ */
+ ret = aes_encrypt_update(ctx, plaintext, ciphertext, req);
+ if (ret != CRYPTO_SUCCESS) {
+ return (ret);
+ }
+
+ /*
+ * For CCM mode, aes_ccm_encrypt_final() will take care of any
+ * left-over unprocessed data, and compute the MAC
+ */
+ if (aes_ctx->ac_flags & CCM_MODE) {
+ /*
+ * ccm_encrypt_final() will compute the MAC and append
+ * it to existing ciphertext. So, need to adjust the left over
+ * length value accordingly
+ */
+
+ /* order of following 2 lines MUST not be reversed */
+ ciphertext->cd_offset = ciphertext->cd_length;
+ ciphertext->cd_length = saved_length - ciphertext->cd_length;
+ ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+ if (ret != CRYPTO_SUCCESS) {
+ return (ret);
+ }
+
+ if (plaintext != ciphertext) {
+ ciphertext->cd_length =
+ ciphertext->cd_offset - saved_offset;
+ }
+ ciphertext->cd_offset = saved_offset;
+ } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
+ /*
+ * gcm_encrypt_final() will compute the MAC and append
+ * it to existing ciphertext. So, need to adjust the left over
+ * length value accordingly
+ */
+
+ /* order of following 2 lines MUST not be reversed */
+ ciphertext->cd_offset = ciphertext->cd_length;
+ ciphertext->cd_length = saved_length - ciphertext->cd_length;
+ ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+ aes_xor_block);
+ if (ret != CRYPTO_SUCCESS) {
+ return (ret);
+ }
+
+ if (plaintext != ciphertext) {
+ ciphertext->cd_length =
+ ciphertext->cd_offset - saved_offset;
+ }
+ ciphertext->cd_offset = saved_offset;
+ }
+
+ ASSERT(aes_ctx->ac_remainder_len == 0);
+ (void) aes_free_context(ctx);
+
+ return (ret);
+}
+
+
+static int
+aes_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
+ crypto_data_t *plaintext, crypto_req_handle_t req)
+{
+ int ret = CRYPTO_FAILED;
+
+ aes_ctx_t *aes_ctx;
+ off_t saved_offset;
+ size_t saved_length, length_needed;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+ aes_ctx = ctx->cc_provider_private;
+
+ /*
+ * For block ciphers, plaintext must be a multiple of AES block size.
+ * This test is only valid for ciphers whose blocksize is a power of 2.
+ */
+ if (((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE))
+ == 0) && (ciphertext->cd_length & (AES_BLOCK_LEN - 1)) != 0) {
+ return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+ }
+
+ AES_ARG_INPLACE(ciphertext, plaintext);
+
+ /*
+ * Return length needed to store the output.
+ * Do not destroy context when plaintext buffer is too small.
+ *
+ * CCM: plaintext is MAC len smaller than cipher text
+ * GCM: plaintext is TAG len smaller than cipher text
+ * GMAC: plaintext length must be zero
+ */
+ switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) {
+ case CCM_MODE:
+ length_needed = aes_ctx->ac_processed_data_len;
+ break;
+ case GCM_MODE:
+ length_needed = ciphertext->cd_length - aes_ctx->ac_tag_len;
+ break;
+ case GMAC_MODE:
+ if (plaintext->cd_length != 0)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ length_needed = 0;
+ break;
+ default:
+ length_needed = ciphertext->cd_length;
+ }
+
+ if (plaintext->cd_length < length_needed) {
+ plaintext->cd_length = length_needed;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ saved_offset = plaintext->cd_offset;
+ saved_length = plaintext->cd_length;
+
+ /*
+ * Do an update on the specified input data.
+ */
+ ret = aes_decrypt_update(ctx, ciphertext, plaintext, req);
+ if (ret != CRYPTO_SUCCESS) {
+ goto cleanup;
+ }
+
+ if (aes_ctx->ac_flags & CCM_MODE) {
+ ASSERT(aes_ctx->ac_processed_data_len == aes_ctx->ac_data_len);
+ ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len);
+
+ /* order of following 2 lines MUST not be reversed */
+ plaintext->cd_offset = plaintext->cd_length;
+ plaintext->cd_length = saved_length - plaintext->cd_length;
+
+ ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+ aes_xor_block);
+ if (ret == CRYPTO_SUCCESS) {
+ if (plaintext != ciphertext) {
+ plaintext->cd_length =
+ plaintext->cd_offset - saved_offset;
+ }
+ } else {
+ plaintext->cd_length = saved_length;
+ }
+
+ plaintext->cd_offset = saved_offset;
+ } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
+ /* order of following 2 lines MUST not be reversed */
+ plaintext->cd_offset = plaintext->cd_length;
+ plaintext->cd_length = saved_length - plaintext->cd_length;
+
+ ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+ if (ret == CRYPTO_SUCCESS) {
+ if (plaintext != ciphertext) {
+ plaintext->cd_length =
+ plaintext->cd_offset - saved_offset;
+ }
+ } else {
+ plaintext->cd_length = saved_length;
+ }
+
+ plaintext->cd_offset = saved_offset;
+ }
+
+ ASSERT(aes_ctx->ac_remainder_len == 0);
+
+cleanup:
+ (void) aes_free_context(ctx);
+
+ return (ret);
+}
+
+
+/* ARGSUSED */
+static int
+aes_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext,
+ crypto_data_t *ciphertext, crypto_req_handle_t req)
+{
+ off_t saved_offset;
+ size_t saved_length, out_len;
+ int ret = CRYPTO_SUCCESS;
+ aes_ctx_t *aes_ctx;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+ aes_ctx = ctx->cc_provider_private;
+
+ AES_ARG_INPLACE(plaintext, ciphertext);
+
+ /* compute number of bytes that will hold the ciphertext */
+ out_len = aes_ctx->ac_remainder_len;
+ out_len += plaintext->cd_length;
+ out_len &= ~(AES_BLOCK_LEN - 1);
+
+ /* return length needed to store the output */
+ if (ciphertext->cd_length < out_len) {
+ ciphertext->cd_length = out_len;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ saved_offset = ciphertext->cd_offset;
+ saved_length = ciphertext->cd_length;
+
+ /*
+ * Do the AES update on the specified input data.
+ */
+ switch (plaintext->cd_format) {
+ case CRYPTO_DATA_RAW:
+ ret = crypto_update_iov(ctx->cc_provider_private,
+ plaintext, ciphertext, aes_encrypt_contiguous_blocks,
+ aes_copy_block64);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = crypto_update_uio(ctx->cc_provider_private,
+ plaintext, ciphertext, aes_encrypt_contiguous_blocks,
+ aes_copy_block64);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ /*
+ * Since AES counter mode is a stream cipher, we call
+ * ctr_mode_final() to pick up any remaining bytes.
+ * It is an internal function that does not destroy
+ * the context like *normal* final routines.
+ */
+ if ((aes_ctx->ac_flags & CTR_MODE) && (aes_ctx->ac_remainder_len > 0)) {
+ ret = ctr_mode_final((ctr_ctx_t *)aes_ctx,
+ ciphertext, aes_encrypt_block);
+ }
+
+ if (ret == CRYPTO_SUCCESS) {
+ if (plaintext != ciphertext)
+ ciphertext->cd_length =
+ ciphertext->cd_offset - saved_offset;
+ } else {
+ ciphertext->cd_length = saved_length;
+ }
+ ciphertext->cd_offset = saved_offset;
+
+ return (ret);
+}
+
+
+static int
+aes_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
+ crypto_data_t *plaintext, crypto_req_handle_t req)
+{
+ off_t saved_offset;
+ size_t saved_length, out_len;
+ int ret = CRYPTO_SUCCESS;
+ aes_ctx_t *aes_ctx;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+ aes_ctx = ctx->cc_provider_private;
+
+ AES_ARG_INPLACE(ciphertext, plaintext);
+
+ /*
+ * Compute number of bytes that will hold the plaintext.
+ * This is not necessary for CCM, GCM, and GMAC since these
+ * mechanisms never return plaintext for update operations.
+ */
+ if ((aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
+ out_len = aes_ctx->ac_remainder_len;
+ out_len += ciphertext->cd_length;
+ out_len &= ~(AES_BLOCK_LEN - 1);
+
+ /* return length needed to store the output */
+ if (plaintext->cd_length < out_len) {
+ plaintext->cd_length = out_len;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+ }
+
+ saved_offset = plaintext->cd_offset;
+ saved_length = plaintext->cd_length;
+
+ if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE))
+ gcm_set_kmflag((gcm_ctx_t *)aes_ctx, crypto_kmflag(req));
+
+ /*
+ * Do the AES update on the specified input data.
+ */
+ switch (ciphertext->cd_format) {
+ case CRYPTO_DATA_RAW:
+ ret = crypto_update_iov(ctx->cc_provider_private,
+ ciphertext, plaintext, aes_decrypt_contiguous_blocks,
+ aes_copy_block64);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = crypto_update_uio(ctx->cc_provider_private,
+ ciphertext, plaintext, aes_decrypt_contiguous_blocks,
+ aes_copy_block64);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ /*
+ * Since AES counter mode is a stream cipher, we call
+ * ctr_mode_final() to pick up any remaining bytes.
+ * It is an internal function that does not destroy
+ * the context like *normal* final routines.
+ */
+ if ((aes_ctx->ac_flags & CTR_MODE) && (aes_ctx->ac_remainder_len > 0)) {
+ ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, plaintext,
+ aes_encrypt_block);
+ if (ret == CRYPTO_DATA_LEN_RANGE)
+ ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
+ }
+
+ if (ret == CRYPTO_SUCCESS) {
+ if (ciphertext != plaintext)
+ plaintext->cd_length =
+ plaintext->cd_offset - saved_offset;
+ } else {
+ plaintext->cd_length = saved_length;
+ }
+ plaintext->cd_offset = saved_offset;
+
+
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+aes_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *data,
+ crypto_req_handle_t req)
+{
+ aes_ctx_t *aes_ctx;
+ int ret;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+ aes_ctx = ctx->cc_provider_private;
+
+ if (data->cd_format != CRYPTO_DATA_RAW &&
+ data->cd_format != CRYPTO_DATA_UIO) {
+ return (CRYPTO_ARGUMENTS_BAD);
+ }
+
+ if (aes_ctx->ac_flags & CTR_MODE) {
+ if (aes_ctx->ac_remainder_len > 0) {
+ ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, data,
+ aes_encrypt_block);
+ if (ret != CRYPTO_SUCCESS)
+ return (ret);
+ }
+ } else if (aes_ctx->ac_flags & CCM_MODE) {
+ ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+ if (ret != CRYPTO_SUCCESS) {
+ return (ret);
+ }
+ } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
+ size_t saved_offset = data->cd_offset;
+
+ ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+ aes_xor_block);
+ if (ret != CRYPTO_SUCCESS) {
+ return (ret);
+ }
+ data->cd_length = data->cd_offset - saved_offset;
+ data->cd_offset = saved_offset;
+ } else {
+ /*
+ * There must be no unprocessed plaintext.
+ * This happens if the length of the last data is
+ * not a multiple of the AES block length.
+ */
+ if (aes_ctx->ac_remainder_len > 0) {
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+ data->cd_length = 0;
+ }
+
+ (void) aes_free_context(ctx);
+
+ return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+static int
+aes_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *data,
+ crypto_req_handle_t req)
+{
+ aes_ctx_t *aes_ctx;
+ int ret;
+ off_t saved_offset;
+ size_t saved_length;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+ aes_ctx = ctx->cc_provider_private;
+
+ if (data->cd_format != CRYPTO_DATA_RAW &&
+ data->cd_format != CRYPTO_DATA_UIO) {
+ return (CRYPTO_ARGUMENTS_BAD);
+ }
+
+ /*
+ * There must be no unprocessed ciphertext.
+ * This happens if the length of the last ciphertext is
+ * not a multiple of the AES block length.
+ */
+ if (aes_ctx->ac_remainder_len > 0) {
+ if ((aes_ctx->ac_flags & CTR_MODE) == 0)
+ return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+ else {
+ ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, data,
+ aes_encrypt_block);
+ if (ret == CRYPTO_DATA_LEN_RANGE)
+ ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
+ if (ret != CRYPTO_SUCCESS)
+ return (ret);
+ }
+ }
+
+ if (aes_ctx->ac_flags & CCM_MODE) {
+ /*
+ * This is where all the plaintext is returned, make sure
+ * the plaintext buffer is big enough
+ */
+ size_t pt_len = aes_ctx->ac_data_len;
+ if (data->cd_length < pt_len) {
+ data->cd_length = pt_len;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ ASSERT(aes_ctx->ac_processed_data_len == pt_len);
+ ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len);
+ saved_offset = data->cd_offset;
+ saved_length = data->cd_length;
+ ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+ aes_xor_block);
+ if (ret == CRYPTO_SUCCESS) {
+ data->cd_length = data->cd_offset - saved_offset;
+ } else {
+ data->cd_length = saved_length;
+ }
+
+ data->cd_offset = saved_offset;
+ if (ret != CRYPTO_SUCCESS) {
+ return (ret);
+ }
+ } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
+ /*
+ * This is where all the plaintext is returned, make sure
+ * the plaintext buffer is big enough
+ */
+ gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx;
+ size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
+
+ if (data->cd_length < pt_len) {
+ data->cd_length = pt_len;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ saved_offset = data->cd_offset;
+ saved_length = data->cd_length;
+ ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+ if (ret == CRYPTO_SUCCESS) {
+ data->cd_length = data->cd_offset - saved_offset;
+ } else {
+ data->cd_length = saved_length;
+ }
+
+ data->cd_offset = saved_offset;
+ if (ret != CRYPTO_SUCCESS) {
+ return (ret);
+ }
+ }
+
+
+ if ((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
+ data->cd_length = 0;
+ }
+
+ (void) aes_free_context(ctx);
+
+ return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+static int
+aes_encrypt_atomic(crypto_provider_handle_t provider,
+ crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
+ crypto_spi_ctx_template_t template, crypto_req_handle_t req)
+{
+ aes_ctx_t aes_ctx; /* on the stack */
+ off_t saved_offset;
+ size_t saved_length;
+ size_t length_needed;
+ int ret;
+
+ AES_ARG_INPLACE(plaintext, ciphertext);
+
+ /*
+ * CTR, CCM, GCM, and GMAC modes do not require that plaintext
+ * be a multiple of AES block size.
+ */
+ switch (mechanism->cm_type) {
+ case AES_CTR_MECH_INFO_TYPE:
+ case AES_CCM_MECH_INFO_TYPE:
+ case AES_GCM_MECH_INFO_TYPE:
+ case AES_GMAC_MECH_INFO_TYPE:
+ break;
+ default:
+ if ((plaintext->cd_length & (AES_BLOCK_LEN - 1)) != 0)
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ if ((ret = aes_check_mech_param(mechanism, NULL, 0)) != CRYPTO_SUCCESS)
+ return (ret);
+
+ bzero(&aes_ctx, sizeof (aes_ctx_t));
+
+ ret = aes_common_init_ctx(&aes_ctx, template, mechanism, key,
+ crypto_kmflag(req), B_TRUE);
+ if (ret != CRYPTO_SUCCESS)
+ return (ret);
+
+ switch (mechanism->cm_type) {
+ case AES_CCM_MECH_INFO_TYPE:
+ length_needed = plaintext->cd_length + aes_ctx.ac_mac_len;
+ break;
+ case AES_GMAC_MECH_INFO_TYPE:
+ if (plaintext->cd_length != 0)
+ return (CRYPTO_ARGUMENTS_BAD);
+ /* FALLTHRU */
+ case AES_GCM_MECH_INFO_TYPE:
+ length_needed = plaintext->cd_length + aes_ctx.ac_tag_len;
+ break;
+ default:
+ length_needed = plaintext->cd_length;
+ }
+
+ /* return size of buffer needed to store output */
+ if (ciphertext->cd_length < length_needed) {
+ ciphertext->cd_length = length_needed;
+ ret = CRYPTO_BUFFER_TOO_SMALL;
+ goto out;
+ }
+
+ saved_offset = ciphertext->cd_offset;
+ saved_length = ciphertext->cd_length;
+
+ /*
+ * Do an update on the specified input data.
+ */
+ switch (plaintext->cd_format) {
+ case CRYPTO_DATA_RAW:
+ ret = crypto_update_iov(&aes_ctx, plaintext, ciphertext,
+ aes_encrypt_contiguous_blocks, aes_copy_block64);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = crypto_update_uio(&aes_ctx, plaintext, ciphertext,
+ aes_encrypt_contiguous_blocks, aes_copy_block64);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret == CRYPTO_SUCCESS) {
+ if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) {
+ ret = ccm_encrypt_final((ccm_ctx_t *)&aes_ctx,
+ ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
+ aes_xor_block);
+ if (ret != CRYPTO_SUCCESS)
+ goto out;
+ ASSERT(aes_ctx.ac_remainder_len == 0);
+ } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
+ mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
+ ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx,
+ ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
+ aes_copy_block, aes_xor_block);
+ if (ret != CRYPTO_SUCCESS)
+ goto out;
+ ASSERT(aes_ctx.ac_remainder_len == 0);
+ } else if (mechanism->cm_type == AES_CTR_MECH_INFO_TYPE) {
+ if (aes_ctx.ac_remainder_len > 0) {
+ ret = ctr_mode_final((ctr_ctx_t *)&aes_ctx,
+ ciphertext, aes_encrypt_block);
+ if (ret != CRYPTO_SUCCESS)
+ goto out;
+ }
+ } else {
+ ASSERT(aes_ctx.ac_remainder_len == 0);
+ }
+
+ if (plaintext != ciphertext) {
+ ciphertext->cd_length =
+ ciphertext->cd_offset - saved_offset;
+ }
+ } else {
+ ciphertext->cd_length = saved_length;
+ }
+ ciphertext->cd_offset = saved_offset;
+
+out:
+ if (aes_ctx.ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) {
+ bzero(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
+ kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
+ }
+
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+aes_decrypt_atomic(crypto_provider_handle_t provider,
+ crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
+ crypto_spi_ctx_template_t template, crypto_req_handle_t req)
+{
+ aes_ctx_t aes_ctx; /* on the stack */
+ off_t saved_offset;
+ size_t saved_length;
+ size_t length_needed;
+ int ret;
+
+ AES_ARG_INPLACE(ciphertext, plaintext);
+
+ /*
+ * CCM, GCM, CTR, and GMAC modes do not require that ciphertext
+ * be a multiple of AES block size.
+ */
+ switch (mechanism->cm_type) {
+ case AES_CTR_MECH_INFO_TYPE:
+ case AES_CCM_MECH_INFO_TYPE:
+ case AES_GCM_MECH_INFO_TYPE:
+ case AES_GMAC_MECH_INFO_TYPE:
+ break;
+ default:
+ if ((ciphertext->cd_length & (AES_BLOCK_LEN - 1)) != 0)
+ return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+ }
+
+ if ((ret = aes_check_mech_param(mechanism, NULL, 0)) != CRYPTO_SUCCESS)
+ return (ret);
+
+ bzero(&aes_ctx, sizeof (aes_ctx_t));
+
+ ret = aes_common_init_ctx(&aes_ctx, template, mechanism, key,
+ crypto_kmflag(req), B_FALSE);
+ if (ret != CRYPTO_SUCCESS)
+ return (ret);
+
+ switch (mechanism->cm_type) {
+ case AES_CCM_MECH_INFO_TYPE:
+ length_needed = aes_ctx.ac_data_len;
+ break;
+ case AES_GCM_MECH_INFO_TYPE:
+ length_needed = ciphertext->cd_length - aes_ctx.ac_tag_len;
+ break;
+ case AES_GMAC_MECH_INFO_TYPE:
+ if (plaintext->cd_length != 0)
+ return (CRYPTO_ARGUMENTS_BAD);
+ length_needed = 0;
+ break;
+ default:
+ length_needed = ciphertext->cd_length;
+ }
+
+ /* return size of buffer needed to store output */
+ if (plaintext->cd_length < length_needed) {
+ plaintext->cd_length = length_needed;
+ ret = CRYPTO_BUFFER_TOO_SMALL;
+ goto out;
+ }
+
+ saved_offset = plaintext->cd_offset;
+ saved_length = plaintext->cd_length;
+
+ if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
+ mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE)
+ gcm_set_kmflag((gcm_ctx_t *)&aes_ctx, crypto_kmflag(req));
+
+ /*
+ * Do an update on the specified input data.
+ */
+ switch (ciphertext->cd_format) {
+ case CRYPTO_DATA_RAW:
+ ret = crypto_update_iov(&aes_ctx, ciphertext, plaintext,
+ aes_decrypt_contiguous_blocks, aes_copy_block64);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = crypto_update_uio(&aes_ctx, ciphertext, plaintext,
+ aes_decrypt_contiguous_blocks, aes_copy_block64);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret == CRYPTO_SUCCESS) {
+ if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) {
+ ASSERT(aes_ctx.ac_processed_data_len
+ == aes_ctx.ac_data_len);
+ ASSERT(aes_ctx.ac_processed_mac_len
+ == aes_ctx.ac_mac_len);
+ ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx,
+ plaintext, AES_BLOCK_LEN, aes_encrypt_block,
+ aes_copy_block, aes_xor_block);
+ ASSERT(aes_ctx.ac_remainder_len == 0);
+ if ((ret == CRYPTO_SUCCESS) &&
+ (ciphertext != plaintext)) {
+ plaintext->cd_length =
+ plaintext->cd_offset - saved_offset;
+ } else {
+ plaintext->cd_length = saved_length;
+ }
+ } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
+ mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
+ ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx,
+ plaintext, AES_BLOCK_LEN, aes_encrypt_block,
+ aes_xor_block);
+ ASSERT(aes_ctx.ac_remainder_len == 0);
+ if ((ret == CRYPTO_SUCCESS) &&
+ (ciphertext != plaintext)) {
+ plaintext->cd_length =
+ plaintext->cd_offset - saved_offset;
+ } else {
+ plaintext->cd_length = saved_length;
+ }
+ } else if (mechanism->cm_type != AES_CTR_MECH_INFO_TYPE) {
+ ASSERT(aes_ctx.ac_remainder_len == 0);
+ if (ciphertext != plaintext)
+ plaintext->cd_length =
+ plaintext->cd_offset - saved_offset;
+ } else {
+ if (aes_ctx.ac_remainder_len > 0) {
+ ret = ctr_mode_final((ctr_ctx_t *)&aes_ctx,
+ plaintext, aes_encrypt_block);
+ if (ret == CRYPTO_DATA_LEN_RANGE)
+ ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
+ if (ret != CRYPTO_SUCCESS)
+ goto out;
+ }
+ if (ciphertext != plaintext)
+ plaintext->cd_length =
+ plaintext->cd_offset - saved_offset;
+ }
+ } else {
+ plaintext->cd_length = saved_length;
+ }
+ plaintext->cd_offset = saved_offset;
+
+out:
+ if (aes_ctx.ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) {
+ bzero(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
+ kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
+ }
+
+ if (aes_ctx.ac_flags & CCM_MODE) {
+ if (aes_ctx.ac_pt_buf != NULL) {
+ vmem_free(aes_ctx.ac_pt_buf, aes_ctx.ac_data_len);
+ }
+ } else if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) {
+ if (((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf != NULL) {
+ vmem_free(((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf,
+ ((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf_len);
+ }
+ }
+
+ return (ret);
+}
+
+/*
+ * KCF software provider context template entry points.
+ */
+/* ARGSUSED */
+static int
+aes_create_ctx_template(crypto_provider_handle_t provider,
+ crypto_mechanism_t *mechanism, crypto_key_t *key,
+ crypto_spi_ctx_template_t *tmpl, size_t *tmpl_size, crypto_req_handle_t req)
+{
+ void *keysched;
+ size_t size;
+ int rv;
+
+ if (mechanism->cm_type != AES_ECB_MECH_INFO_TYPE &&
+ mechanism->cm_type != AES_CBC_MECH_INFO_TYPE &&
+ mechanism->cm_type != AES_CTR_MECH_INFO_TYPE &&
+ mechanism->cm_type != AES_CCM_MECH_INFO_TYPE &&
+ mechanism->cm_type != AES_GCM_MECH_INFO_TYPE &&
+ mechanism->cm_type != AES_GMAC_MECH_INFO_TYPE)
+ return (CRYPTO_MECHANISM_INVALID);
+
+ if ((keysched = aes_alloc_keysched(&size,
+ crypto_kmflag(req))) == NULL) {
+ return (CRYPTO_HOST_MEMORY);
+ }
+
+ /*
+ * Initialize key schedule. Key length information is stored
+ * in the key.
+ */
+ if ((rv = init_keysched(key, keysched)) != CRYPTO_SUCCESS) {
+ bzero(keysched, size);
+ kmem_free(keysched, size);
+ return (rv);
+ }
+
+ *tmpl = keysched;
+ *tmpl_size = size;
+
+ return (CRYPTO_SUCCESS);
+}
+
+
+static int
+aes_free_context(crypto_ctx_t *ctx)
+{
+ aes_ctx_t *aes_ctx = ctx->cc_provider_private;
+
+ if (aes_ctx != NULL) {
+ if (aes_ctx->ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) {
+ ASSERT(aes_ctx->ac_keysched_len != 0);
+ bzero(aes_ctx->ac_keysched, aes_ctx->ac_keysched_len);
+ kmem_free(aes_ctx->ac_keysched,
+ aes_ctx->ac_keysched_len);
+ }
+ crypto_free_mode_ctx(aes_ctx);
+ ctx->cc_provider_private = NULL;
+ }
+
+ return (CRYPTO_SUCCESS);
+}
+
+
+static int
+aes_common_init_ctx(aes_ctx_t *aes_ctx, crypto_spi_ctx_template_t *template,
+ crypto_mechanism_t *mechanism, crypto_key_t *key, int kmflag,
+ boolean_t is_encrypt_init)
+{
+ int rv = CRYPTO_SUCCESS;
+ void *keysched;
+ size_t size;
+
+ if (template == NULL) {
+ if ((keysched = aes_alloc_keysched(&size, kmflag)) == NULL)
+ return (CRYPTO_HOST_MEMORY);
+ /*
+ * Initialize key schedule.
+ * Key length is stored in the key.
+ */
+ if ((rv = init_keysched(key, keysched)) != CRYPTO_SUCCESS) {
+ kmem_free(keysched, size);
+ return (rv);
+ }
+
+ aes_ctx->ac_flags |= PROVIDER_OWNS_KEY_SCHEDULE;
+ aes_ctx->ac_keysched_len = size;
+ } else {
+ keysched = template;
+ }
+ aes_ctx->ac_keysched = keysched;
+
+ switch (mechanism->cm_type) {
+ case AES_CBC_MECH_INFO_TYPE:
+ rv = cbc_init_ctx((cbc_ctx_t *)aes_ctx, mechanism->cm_param,
+ mechanism->cm_param_len, AES_BLOCK_LEN, aes_copy_block64);
+ break;
+ case AES_CTR_MECH_INFO_TYPE: {
+ CK_AES_CTR_PARAMS *pp;
+
+ if (mechanism->cm_param == NULL ||
+ mechanism->cm_param_len != sizeof (CK_AES_CTR_PARAMS)) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+ pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param;
+ rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits,
+ pp->cb, aes_copy_block);
+ break;
+ }
+ case AES_CCM_MECH_INFO_TYPE:
+ if (mechanism->cm_param == NULL ||
+ mechanism->cm_param_len != sizeof (CK_AES_CCM_PARAMS)) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+ rv = ccm_init_ctx((ccm_ctx_t *)aes_ctx, mechanism->cm_param,
+ kmflag, is_encrypt_init, AES_BLOCK_LEN, aes_encrypt_block,
+ aes_xor_block);
+ break;
+ case AES_GCM_MECH_INFO_TYPE:
+ if (mechanism->cm_param == NULL ||
+ mechanism->cm_param_len != sizeof (CK_AES_GCM_PARAMS)) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+ rv = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+ aes_xor_block);
+ break;
+ case AES_GMAC_MECH_INFO_TYPE:
+ if (mechanism->cm_param == NULL ||
+ mechanism->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+ rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
+ AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+ aes_xor_block);
+ break;
+ case AES_ECB_MECH_INFO_TYPE:
+ aes_ctx->ac_flags |= ECB_MODE;
+ }
+
+ if (rv != CRYPTO_SUCCESS) {
+ if (aes_ctx->ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) {
+ bzero(keysched, size);
+ kmem_free(keysched, size);
+ }
+ }
+
+ return (rv);
+}
+
+static int
+process_gmac_mech(crypto_mechanism_t *mech, crypto_data_t *data,
+ CK_AES_GCM_PARAMS *gcm_params)
+{
+ /* LINTED: pointer alignment */
+ CK_AES_GMAC_PARAMS *params = (CK_AES_GMAC_PARAMS *)mech->cm_param;
+
+ if (mech->cm_type != AES_GMAC_MECH_INFO_TYPE)
+ return (CRYPTO_MECHANISM_INVALID);
+
+ if (mech->cm_param_len != sizeof (CK_AES_GMAC_PARAMS))
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+
+ if (params->pIv == NULL)
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+
+ gcm_params->pIv = params->pIv;
+ gcm_params->ulIvLen = AES_GMAC_IV_LEN;
+ gcm_params->ulTagBits = AES_GMAC_TAG_BITS;
+
+ if (data == NULL)
+ return (CRYPTO_SUCCESS);
+
+ if (data->cd_format != CRYPTO_DATA_RAW)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ gcm_params->pAAD = (uchar_t *)data->cd_raw.iov_base;
+ gcm_params->ulAADLen = data->cd_length;
+ return (CRYPTO_SUCCESS);
+}
+
+static int
+aes_mac_atomic(crypto_provider_handle_t provider,
+ crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac,
+ crypto_spi_ctx_template_t template, crypto_req_handle_t req)
+{
+ CK_AES_GCM_PARAMS gcm_params;
+ crypto_mechanism_t gcm_mech;
+ int rv;
+
+ if ((rv = process_gmac_mech(mechanism, data, &gcm_params))
+ != CRYPTO_SUCCESS)
+ return (rv);
+
+ gcm_mech.cm_type = AES_GCM_MECH_INFO_TYPE;
+ gcm_mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS);
+ gcm_mech.cm_param = (char *)&gcm_params;
+
+ return (aes_encrypt_atomic(provider, session_id, &gcm_mech,
+ key, &null_crypto_data, mac, template, req));
+}
+
+static int
+aes_mac_verify_atomic(crypto_provider_handle_t provider,
+ crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac,
+ crypto_spi_ctx_template_t template, crypto_req_handle_t req)
+{
+ CK_AES_GCM_PARAMS gcm_params;
+ crypto_mechanism_t gcm_mech;
+ int rv;
+
+ if ((rv = process_gmac_mech(mechanism, data, &gcm_params))
+ != CRYPTO_SUCCESS)
+ return (rv);
+
+ gcm_mech.cm_type = AES_GCM_MECH_INFO_TYPE;
+ gcm_mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS);
+ gcm_mech.cm_param = (char *)&gcm_params;
+
+ return (aes_decrypt_atomic(provider, session_id, &gcm_mech,
+ key, mac, &null_crypto_data, template, req));
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/modctl.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/spi.h>
+
+#include <sha1/sha1.h>
+#include <sha1/sha1_impl.h>
+
+/*
+ * The sha1 module is created with two modlinkages:
+ * - a modlmisc that allows consumers to directly call the entry points
+ * SHA1Init, SHA1Update, and SHA1Final.
+ * - a modlcrypto that allows the module to register with the Kernel
+ * Cryptographic Framework (KCF) as a software provider for the SHA1
+ * mechanisms.
+ */
+
+static struct modlcrypto modlcrypto = {
+ &mod_cryptoops,
+ "SHA1 Kernel SW Provider 1.1"
+};
+
+static struct modlinkage modlinkage = {
+ MODREV_1, { &modlcrypto, NULL }
+};
+
+
+/*
+ * Macros to access the SHA1 or SHA1-HMAC contexts from a context passed
+ * by KCF to one of the entry points.
+ */
+
+#define PROV_SHA1_CTX(ctx) ((sha1_ctx_t *)(ctx)->cc_provider_private)
+#define PROV_SHA1_HMAC_CTX(ctx) ((sha1_hmac_ctx_t *)(ctx)->cc_provider_private)
+
+/* to extract the digest length passed as mechanism parameter */
+#define PROV_SHA1_GET_DIGEST_LEN(m, len) { \
+ if (IS_P2ALIGNED((m)->cm_param, sizeof (ulong_t))) \
+ (len) = (uint32_t)*((ulong_t *)(void *)mechanism->cm_param); \
+ else { \
+ ulong_t tmp_ulong; \
+ bcopy((m)->cm_param, &tmp_ulong, sizeof (ulong_t)); \
+ (len) = (uint32_t)tmp_ulong; \
+ } \
+}
+
+#define PROV_SHA1_DIGEST_KEY(ctx, key, len, digest) { \
+ SHA1Init(ctx); \
+ SHA1Update(ctx, key, len); \
+ SHA1Final(digest, ctx); \
+}
+
+/*
+ * Mechanism info structure passed to KCF during registration.
+ */
+static crypto_mech_info_t sha1_mech_info_tab[] = {
+ /* SHA1 */
+ {SUN_CKM_SHA1, SHA1_MECH_INFO_TYPE,
+ CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC,
+ 0, 0, CRYPTO_KEYSIZE_UNIT_IN_BITS},
+ /* SHA1-HMAC */
+ {SUN_CKM_SHA1_HMAC, SHA1_HMAC_MECH_INFO_TYPE,
+ CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC,
+ SHA1_HMAC_MIN_KEY_LEN, SHA1_HMAC_MAX_KEY_LEN,
+ CRYPTO_KEYSIZE_UNIT_IN_BYTES},
+ /* SHA1-HMAC GENERAL */
+ {SUN_CKM_SHA1_HMAC_GENERAL, SHA1_HMAC_GEN_MECH_INFO_TYPE,
+ CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC,
+ SHA1_HMAC_MIN_KEY_LEN, SHA1_HMAC_MAX_KEY_LEN,
+ CRYPTO_KEYSIZE_UNIT_IN_BYTES}
+};
+
+static void sha1_provider_status(crypto_provider_handle_t, uint_t *);
+
+static crypto_control_ops_t sha1_control_ops = {
+ sha1_provider_status
+};
+
+static int sha1_digest_init(crypto_ctx_t *, crypto_mechanism_t *,
+ crypto_req_handle_t);
+static int sha1_digest(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int sha1_digest_update(crypto_ctx_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int sha1_digest_final(crypto_ctx_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int sha1_digest_atomic(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_data_t *, crypto_data_t *,
+ crypto_req_handle_t);
+
+static crypto_digest_ops_t sha1_digest_ops = {
+ sha1_digest_init,
+ sha1_digest,
+ sha1_digest_update,
+ NULL,
+ sha1_digest_final,
+ sha1_digest_atomic
+};
+
+static int sha1_mac_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+static int sha1_mac_update(crypto_ctx_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int sha1_mac_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t);
+static int sha1_mac_atomic(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+static int sha1_mac_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+
+static crypto_mac_ops_t sha1_mac_ops = {
+ sha1_mac_init,
+ NULL,
+ sha1_mac_update,
+ sha1_mac_final,
+ sha1_mac_atomic,
+ sha1_mac_verify_atomic
+};
+
+static int sha1_create_ctx_template(crypto_provider_handle_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t *,
+ size_t *, crypto_req_handle_t);
+static int sha1_free_context(crypto_ctx_t *);
+
+static crypto_ctx_ops_t sha1_ctx_ops = {
+ sha1_create_ctx_template,
+ sha1_free_context
+};
+
+static crypto_ops_t sha1_crypto_ops = {{{{{
+ &sha1_control_ops,
+ &sha1_digest_ops,
+ NULL,
+ &sha1_mac_ops,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ &sha1_ctx_ops,
+}}}}};
+
+static crypto_provider_info_t sha1_prov_info = {{{{
+ CRYPTO_SPI_VERSION_1,
+ "SHA1 Software Provider",
+ CRYPTO_SW_PROVIDER,
+ NULL,
+ &sha1_crypto_ops,
+ sizeof (sha1_mech_info_tab)/sizeof (crypto_mech_info_t),
+ sha1_mech_info_tab
+}}}};
+
+static crypto_kcf_provider_handle_t sha1_prov_handle = 0;
+
+int
+sha1_mod_init(void)
+{
+ int ret;
+
+ if ((ret = mod_install(&modlinkage)) != 0)
+ return (ret);
+
+ /*
+ * Register with KCF. If the registration fails, log an
+ * error but do not uninstall the module, since the functionality
+ * provided by misc/sha1 should still be available.
+ */
+ if ((ret = crypto_register_provider(&sha1_prov_info,
+ &sha1_prov_handle)) != CRYPTO_SUCCESS)
+ cmn_err(CE_WARN, "sha1 _init: "
+ "crypto_register_provider() failed (0x%x)", ret);
+
+ return (0);
+}
+
+int
+sha1_mod_fini(void)
+{
+ int ret;
+
+ if (sha1_prov_handle != 0) {
+ if ((ret = crypto_unregister_provider(sha1_prov_handle)) !=
+ CRYPTO_SUCCESS) {
+ cmn_err(CE_WARN,
+ "sha1 _fini: crypto_unregister_provider() "
+ "failed (0x%x)", ret);
+ return (EBUSY);
+ }
+ sha1_prov_handle = 0;
+ }
+
+ return (mod_remove(&modlinkage));
+}
+
+/*
+ * KCF software provider control entry points.
+ */
+/* ARGSUSED */
+static void
+sha1_provider_status(crypto_provider_handle_t provider, uint_t *status)
+{
+ *status = CRYPTO_PROVIDER_READY;
+}
+
+/*
+ * KCF software provider digest entry points.
+ */
+
+static int
+sha1_digest_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
+ crypto_req_handle_t req)
+{
+ if (mechanism->cm_type != SHA1_MECH_INFO_TYPE)
+ return (CRYPTO_MECHANISM_INVALID);
+
+ /*
+ * Allocate and initialize SHA1 context.
+ */
+ ctx->cc_provider_private = kmem_alloc(sizeof (sha1_ctx_t),
+ crypto_kmflag(req));
+ if (ctx->cc_provider_private == NULL)
+ return (CRYPTO_HOST_MEMORY);
+
+ PROV_SHA1_CTX(ctx)->sc_mech_type = SHA1_MECH_INFO_TYPE;
+ SHA1Init(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx);
+
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Helper SHA1 digest update function for uio data.
+ */
+static int
+sha1_digest_update_uio(SHA1_CTX *sha1_ctx, crypto_data_t *data)
+{
+ off_t offset = data->cd_offset;
+ size_t length = data->cd_length;
+ uint_t vec_idx;
+ size_t cur_len;
+
+ /* we support only kernel buffer */
+ if (data->cd_uio->uio_segflg != UIO_SYSSPACE)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ /*
+ * Jump to the first iovec containing data to be
+ * digested.
+ */
+ for (vec_idx = 0; vec_idx < data->cd_uio->uio_iovcnt &&
+ offset >= data->cd_uio->uio_iov[vec_idx].iov_len;
+ offset -= data->cd_uio->uio_iov[vec_idx++].iov_len)
+ ;
+ if (vec_idx == data->cd_uio->uio_iovcnt) {
+ /*
+ * The caller specified an offset that is larger than the
+ * total size of the buffers it provided.
+ */
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ /*
+ * Now do the digesting on the iovecs.
+ */
+ while (vec_idx < data->cd_uio->uio_iovcnt && length > 0) {
+ cur_len = MIN(data->cd_uio->uio_iov[vec_idx].iov_len -
+ offset, length);
+
+ SHA1Update(sha1_ctx,
+ (uint8_t *)data->cd_uio->uio_iov[vec_idx].iov_base + offset,
+ cur_len);
+
+ length -= cur_len;
+ vec_idx++;
+ offset = 0;
+ }
+
+ if (vec_idx == data->cd_uio->uio_iovcnt && length > 0) {
+ /*
+ * The end of the specified iovec's was reached but
+ * the length requested could not be processed, i.e.
+ * The caller requested to digest more data than it provided.
+ */
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Helper SHA1 digest final function for uio data.
+ * digest_len is the length of the desired digest. If digest_len
+ * is smaller than the default SHA1 digest length, the caller
+ * must pass a scratch buffer, digest_scratch, which must
+ * be at least SHA1_DIGEST_LENGTH bytes.
+ */
+static int
+sha1_digest_final_uio(SHA1_CTX *sha1_ctx, crypto_data_t *digest,
+ ulong_t digest_len, uchar_t *digest_scratch)
+{
+ off_t offset = digest->cd_offset;
+ uint_t vec_idx;
+
+ /* we support only kernel buffer */
+ if (digest->cd_uio->uio_segflg != UIO_SYSSPACE)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ /*
+ * Jump to the first iovec containing ptr to the digest to
+ * be returned.
+ */
+ for (vec_idx = 0; offset >= digest->cd_uio->uio_iov[vec_idx].iov_len &&
+ vec_idx < digest->cd_uio->uio_iovcnt;
+ offset -= digest->cd_uio->uio_iov[vec_idx++].iov_len)
+ ;
+ if (vec_idx == digest->cd_uio->uio_iovcnt) {
+ /*
+ * The caller specified an offset that is
+ * larger than the total size of the buffers
+ * it provided.
+ */
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ if (offset + digest_len <=
+ digest->cd_uio->uio_iov[vec_idx].iov_len) {
+ /*
+ * The computed SHA1 digest will fit in the current
+ * iovec.
+ */
+ if (digest_len != SHA1_DIGEST_LENGTH) {
+ /*
+ * The caller requested a short digest. Digest
+ * into a scratch buffer and return to
+ * the user only what was requested.
+ */
+ SHA1Final(digest_scratch, sha1_ctx);
+ bcopy(digest_scratch, (uchar_t *)digest->
+ cd_uio->uio_iov[vec_idx].iov_base + offset,
+ digest_len);
+ } else {
+ SHA1Final((uchar_t *)digest->
+ cd_uio->uio_iov[vec_idx].iov_base + offset,
+ sha1_ctx);
+ }
+ } else {
+ /*
+ * The computed digest will be crossing one or more iovec's.
+ * This is bad performance-wise but we need to support it.
+ * Allocate a small scratch buffer on the stack and
+ * copy it piece meal to the specified digest iovec's.
+ */
+ uchar_t digest_tmp[SHA1_DIGEST_LENGTH];
+ off_t scratch_offset = 0;
+ size_t length = digest_len;
+ size_t cur_len;
+
+ SHA1Final(digest_tmp, sha1_ctx);
+
+ while (vec_idx < digest->cd_uio->uio_iovcnt && length > 0) {
+ cur_len = MIN(digest->cd_uio->uio_iov[vec_idx].iov_len -
+ offset, length);
+ bcopy(digest_tmp + scratch_offset,
+ digest->cd_uio->uio_iov[vec_idx].iov_base + offset,
+ cur_len);
+
+ length -= cur_len;
+ vec_idx++;
+ scratch_offset += cur_len;
+ offset = 0;
+ }
+
+ if (vec_idx == digest->cd_uio->uio_iovcnt && length > 0) {
+ /*
+ * The end of the specified iovec's was reached but
+ * the length requested could not be processed, i.e.
+ * The caller requested to digest more data than it
+ * provided.
+ */
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+ }
+
+ return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+static int
+sha1_digest(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *digest,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+
+ /*
+ * We need to just return the length needed to store the output.
+ * We should not destroy the context for the following cases.
+ */
+ if ((digest->cd_length == 0) ||
+ (digest->cd_length < SHA1_DIGEST_LENGTH)) {
+ digest->cd_length = SHA1_DIGEST_LENGTH;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ /*
+ * Do the SHA1 update on the specified input data.
+ */
+ switch (data->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA1Update(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx,
+ (uint8_t *)data->cd_raw.iov_base + data->cd_offset,
+ data->cd_length);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha1_digest_update_uio(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx,
+ data);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret != CRYPTO_SUCCESS) {
+ /* the update failed, free context and bail */
+ kmem_free(ctx->cc_provider_private, sizeof (sha1_ctx_t));
+ ctx->cc_provider_private = NULL;
+ digest->cd_length = 0;
+ return (ret);
+ }
+
+ /*
+ * Do a SHA1 final, must be done separately since the digest
+ * type can be different than the input data type.
+ */
+ switch (digest->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA1Final((unsigned char *)digest->cd_raw.iov_base +
+ digest->cd_offset, &PROV_SHA1_CTX(ctx)->sc_sha1_ctx);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha1_digest_final_uio(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx,
+ digest, SHA1_DIGEST_LENGTH, NULL);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ /* all done, free context and return */
+
+ if (ret == CRYPTO_SUCCESS) {
+ digest->cd_length = SHA1_DIGEST_LENGTH;
+ } else {
+ digest->cd_length = 0;
+ }
+
+ kmem_free(ctx->cc_provider_private, sizeof (sha1_ctx_t));
+ ctx->cc_provider_private = NULL;
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha1_digest_update(crypto_ctx_t *ctx, crypto_data_t *data,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+
+ /*
+ * Do the SHA1 update on the specified input data.
+ */
+ switch (data->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA1Update(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx,
+ (uint8_t *)data->cd_raw.iov_base + data->cd_offset,
+ data->cd_length);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha1_digest_update_uio(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx,
+ data);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha1_digest_final(crypto_ctx_t *ctx, crypto_data_t *digest,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+
+ /*
+ * We need to just return the length needed to store the output.
+ * We should not destroy the context for the following cases.
+ */
+ if ((digest->cd_length == 0) ||
+ (digest->cd_length < SHA1_DIGEST_LENGTH)) {
+ digest->cd_length = SHA1_DIGEST_LENGTH;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ /*
+ * Do a SHA1 final.
+ */
+ switch (digest->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA1Final((unsigned char *)digest->cd_raw.iov_base +
+ digest->cd_offset, &PROV_SHA1_CTX(ctx)->sc_sha1_ctx);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha1_digest_final_uio(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx,
+ digest, SHA1_DIGEST_LENGTH, NULL);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ /* all done, free context and return */
+
+ if (ret == CRYPTO_SUCCESS) {
+ digest->cd_length = SHA1_DIGEST_LENGTH;
+ } else {
+ digest->cd_length = 0;
+ }
+
+ kmem_free(ctx->cc_provider_private, sizeof (sha1_ctx_t));
+ ctx->cc_provider_private = NULL;
+
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha1_digest_atomic(crypto_provider_handle_t provider,
+ crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+ crypto_data_t *data, crypto_data_t *digest,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ SHA1_CTX sha1_ctx;
+
+ if (mechanism->cm_type != SHA1_MECH_INFO_TYPE)
+ return (CRYPTO_MECHANISM_INVALID);
+
+ /*
+ * Do the SHA1 init.
+ */
+ SHA1Init(&sha1_ctx);
+
+ /*
+ * Do the SHA1 update on the specified input data.
+ */
+ switch (data->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA1Update(&sha1_ctx,
+ (uint8_t *)data->cd_raw.iov_base + data->cd_offset,
+ data->cd_length);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha1_digest_update_uio(&sha1_ctx, data);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret != CRYPTO_SUCCESS) {
+ /* the update failed, bail */
+ digest->cd_length = 0;
+ return (ret);
+ }
+
+ /*
+ * Do a SHA1 final, must be done separately since the digest
+ * type can be different than the input data type.
+ */
+ switch (digest->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA1Final((unsigned char *)digest->cd_raw.iov_base +
+ digest->cd_offset, &sha1_ctx);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha1_digest_final_uio(&sha1_ctx, digest,
+ SHA1_DIGEST_LENGTH, NULL);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret == CRYPTO_SUCCESS) {
+ digest->cd_length = SHA1_DIGEST_LENGTH;
+ } else {
+ digest->cd_length = 0;
+ }
+
+ return (ret);
+}
+
+/*
+ * KCF software provider mac entry points.
+ *
+ * SHA1 HMAC is: SHA1(key XOR opad, SHA1(key XOR ipad, text))
+ *
+ * Init:
+ * The initialization routine initializes what we denote
+ * as the inner and outer contexts by doing
+ * - for inner context: SHA1(key XOR ipad)
+ * - for outer context: SHA1(key XOR opad)
+ *
+ * Update:
+ * Each subsequent SHA1 HMAC update will result in an
+ * update of the inner context with the specified data.
+ *
+ * Final:
+ * The SHA1 HMAC final will do a SHA1 final operation on the
+ * inner context, and the resulting digest will be used
+ * as the data for an update on the outer context. Last
+ * but not least, a SHA1 final on the outer context will
+ * be performed to obtain the SHA1 HMAC digest to return
+ * to the user.
+ */
+
+/*
+ * Initialize a SHA1-HMAC context.
+ */
+static void
+sha1_mac_init_ctx(sha1_hmac_ctx_t *ctx, void *keyval, uint_t length_in_bytes)
+{
+ uint32_t ipad[SHA1_HMAC_INTS_PER_BLOCK];
+ uint32_t opad[SHA1_HMAC_INTS_PER_BLOCK];
+ uint_t i;
+
+ bzero(ipad, SHA1_HMAC_BLOCK_SIZE);
+ bzero(opad, SHA1_HMAC_BLOCK_SIZE);
+
+ bcopy(keyval, ipad, length_in_bytes);
+ bcopy(keyval, opad, length_in_bytes);
+
+ /* XOR key with ipad (0x36) and opad (0x5c) */
+ for (i = 0; i < SHA1_HMAC_INTS_PER_BLOCK; i++) {
+ ipad[i] ^= 0x36363636;
+ opad[i] ^= 0x5c5c5c5c;
+ }
+
+ /* perform SHA1 on ipad */
+ SHA1Init(&ctx->hc_icontext);
+ SHA1Update(&ctx->hc_icontext, (uint8_t *)ipad, SHA1_HMAC_BLOCK_SIZE);
+
+ /* perform SHA1 on opad */
+ SHA1Init(&ctx->hc_ocontext);
+ SHA1Update(&ctx->hc_ocontext, (uint8_t *)opad, SHA1_HMAC_BLOCK_SIZE);
+}
+
+/*
+ */
+static int
+sha1_mac_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length);
+
+ if (mechanism->cm_type != SHA1_HMAC_MECH_INFO_TYPE &&
+ mechanism->cm_type != SHA1_HMAC_GEN_MECH_INFO_TYPE)
+ return (CRYPTO_MECHANISM_INVALID);
+
+ /* Add support for key by attributes (RFE 4706552) */
+ if (key->ck_format != CRYPTO_KEY_RAW)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ ctx->cc_provider_private = kmem_alloc(sizeof (sha1_hmac_ctx_t),
+ crypto_kmflag(req));
+ if (ctx->cc_provider_private == NULL)
+ return (CRYPTO_HOST_MEMORY);
+
+ if (ctx_template != NULL) {
+ /* reuse context template */
+ bcopy(ctx_template, PROV_SHA1_HMAC_CTX(ctx),
+ sizeof (sha1_hmac_ctx_t));
+ } else {
+ /* no context template, compute context */
+ if (keylen_in_bytes > SHA1_HMAC_BLOCK_SIZE) {
+ uchar_t digested_key[SHA1_DIGEST_LENGTH];
+ sha1_hmac_ctx_t *hmac_ctx = ctx->cc_provider_private;
+
+ /*
+ * Hash the passed-in key to get a smaller key.
+ * The inner context is used since it hasn't been
+ * initialized yet.
+ */
+ PROV_SHA1_DIGEST_KEY(&hmac_ctx->hc_icontext,
+ key->ck_data, keylen_in_bytes, digested_key);
+ sha1_mac_init_ctx(PROV_SHA1_HMAC_CTX(ctx),
+ digested_key, SHA1_DIGEST_LENGTH);
+ } else {
+ sha1_mac_init_ctx(PROV_SHA1_HMAC_CTX(ctx),
+ key->ck_data, keylen_in_bytes);
+ }
+ }
+
+ /*
+ * Get the mechanism parameters, if applicable.
+ */
+ PROV_SHA1_HMAC_CTX(ctx)->hc_mech_type = mechanism->cm_type;
+ if (mechanism->cm_type == SHA1_HMAC_GEN_MECH_INFO_TYPE) {
+ if (mechanism->cm_param == NULL ||
+ mechanism->cm_param_len != sizeof (ulong_t))
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ PROV_SHA1_GET_DIGEST_LEN(mechanism,
+ PROV_SHA1_HMAC_CTX(ctx)->hc_digest_len);
+ if (PROV_SHA1_HMAC_CTX(ctx)->hc_digest_len >
+ SHA1_DIGEST_LENGTH)
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ }
+
+ if (ret != CRYPTO_SUCCESS) {
+ bzero(ctx->cc_provider_private, sizeof (sha1_hmac_ctx_t));
+ kmem_free(ctx->cc_provider_private, sizeof (sha1_hmac_ctx_t));
+ ctx->cc_provider_private = NULL;
+ }
+
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha1_mac_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+
+ /*
+ * Do a SHA1 update of the inner context using the specified
+ * data.
+ */
+ switch (data->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA1Update(&PROV_SHA1_HMAC_CTX(ctx)->hc_icontext,
+ (uint8_t *)data->cd_raw.iov_base + data->cd_offset,
+ data->cd_length);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha1_digest_update_uio(
+ &PROV_SHA1_HMAC_CTX(ctx)->hc_icontext, data);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha1_mac_final(crypto_ctx_t *ctx, crypto_data_t *mac, crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ uchar_t digest[SHA1_DIGEST_LENGTH];
+ uint32_t digest_len = SHA1_DIGEST_LENGTH;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+
+ if (PROV_SHA1_HMAC_CTX(ctx)->hc_mech_type ==
+ SHA1_HMAC_GEN_MECH_INFO_TYPE)
+ digest_len = PROV_SHA1_HMAC_CTX(ctx)->hc_digest_len;
+
+ /*
+ * We need to just return the length needed to store the output.
+ * We should not destroy the context for the following cases.
+ */
+ if ((mac->cd_length == 0) || (mac->cd_length < digest_len)) {
+ mac->cd_length = digest_len;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ /*
+ * Do a SHA1 final on the inner context.
+ */
+ SHA1Final(digest, &PROV_SHA1_HMAC_CTX(ctx)->hc_icontext);
+
+ /*
+ * Do a SHA1 update on the outer context, feeding the inner
+ * digest as data.
+ */
+ SHA1Update(&PROV_SHA1_HMAC_CTX(ctx)->hc_ocontext, digest,
+ SHA1_DIGEST_LENGTH);
+
+ /*
+ * Do a SHA1 final on the outer context, storing the computing
+ * digest in the users buffer.
+ */
+ switch (mac->cd_format) {
+ case CRYPTO_DATA_RAW:
+ if (digest_len != SHA1_DIGEST_LENGTH) {
+ /*
+ * The caller requested a short digest. Digest
+ * into a scratch buffer and return to
+ * the user only what was requested.
+ */
+ SHA1Final(digest,
+ &PROV_SHA1_HMAC_CTX(ctx)->hc_ocontext);
+ bcopy(digest, (unsigned char *)mac->cd_raw.iov_base +
+ mac->cd_offset, digest_len);
+ } else {
+ SHA1Final((unsigned char *)mac->cd_raw.iov_base +
+ mac->cd_offset,
+ &PROV_SHA1_HMAC_CTX(ctx)->hc_ocontext);
+ }
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha1_digest_final_uio(
+ &PROV_SHA1_HMAC_CTX(ctx)->hc_ocontext, mac,
+ digest_len, digest);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret == CRYPTO_SUCCESS) {
+ mac->cd_length = digest_len;
+ } else {
+ mac->cd_length = 0;
+ }
+
+ bzero(ctx->cc_provider_private, sizeof (sha1_hmac_ctx_t));
+ kmem_free(ctx->cc_provider_private, sizeof (sha1_hmac_ctx_t));
+ ctx->cc_provider_private = NULL;
+
+ return (ret);
+}
+
+#define SHA1_MAC_UPDATE(data, ctx, ret) { \
+ switch (data->cd_format) { \
+ case CRYPTO_DATA_RAW: \
+ SHA1Update(&(ctx).hc_icontext, \
+ (uint8_t *)data->cd_raw.iov_base + \
+ data->cd_offset, data->cd_length); \
+ break; \
+ case CRYPTO_DATA_UIO: \
+ ret = sha1_digest_update_uio(&(ctx).hc_icontext, data); \
+ break; \
+ default: \
+ ret = CRYPTO_ARGUMENTS_BAD; \
+ } \
+}
+
+/* ARGSUSED */
+static int
+sha1_mac_atomic(crypto_provider_handle_t provider,
+ crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac,
+ crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ uchar_t digest[SHA1_DIGEST_LENGTH];
+ sha1_hmac_ctx_t sha1_hmac_ctx;
+ uint32_t digest_len = SHA1_DIGEST_LENGTH;
+ uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length);
+
+ if (mechanism->cm_type != SHA1_HMAC_MECH_INFO_TYPE &&
+ mechanism->cm_type != SHA1_HMAC_GEN_MECH_INFO_TYPE)
+ return (CRYPTO_MECHANISM_INVALID);
+
+ /* Add support for key by attributes (RFE 4706552) */
+ if (key->ck_format != CRYPTO_KEY_RAW)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ if (ctx_template != NULL) {
+ /* reuse context template */
+ bcopy(ctx_template, &sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t));
+ } else {
+ /* no context template, initialize context */
+ if (keylen_in_bytes > SHA1_HMAC_BLOCK_SIZE) {
+ /*
+ * Hash the passed-in key to get a smaller key.
+ * The inner context is used since it hasn't been
+ * initialized yet.
+ */
+ PROV_SHA1_DIGEST_KEY(&sha1_hmac_ctx.hc_icontext,
+ key->ck_data, keylen_in_bytes, digest);
+ sha1_mac_init_ctx(&sha1_hmac_ctx, digest,
+ SHA1_DIGEST_LENGTH);
+ } else {
+ sha1_mac_init_ctx(&sha1_hmac_ctx, key->ck_data,
+ keylen_in_bytes);
+ }
+ }
+
+ /* get the mechanism parameters, if applicable */
+ if (mechanism->cm_type == SHA1_HMAC_GEN_MECH_INFO_TYPE) {
+ if (mechanism->cm_param == NULL ||
+ mechanism->cm_param_len != sizeof (ulong_t)) {
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto bail;
+ }
+ PROV_SHA1_GET_DIGEST_LEN(mechanism, digest_len);
+ if (digest_len > SHA1_DIGEST_LENGTH) {
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto bail;
+ }
+ }
+
+ /* do a SHA1 update of the inner context using the specified data */
+ SHA1_MAC_UPDATE(data, sha1_hmac_ctx, ret);
+ if (ret != CRYPTO_SUCCESS)
+ /* the update failed, free context and bail */
+ goto bail;
+
+ /*
+ * Do a SHA1 final on the inner context.
+ */
+ SHA1Final(digest, &sha1_hmac_ctx.hc_icontext);
+
+ /*
+ * Do an SHA1 update on the outer context, feeding the inner
+ * digest as data.
+ */
+ SHA1Update(&sha1_hmac_ctx.hc_ocontext, digest, SHA1_DIGEST_LENGTH);
+
+ /*
+ * Do a SHA1 final on the outer context, storing the computed
+ * digest in the users buffer.
+ */
+ switch (mac->cd_format) {
+ case CRYPTO_DATA_RAW:
+ if (digest_len != SHA1_DIGEST_LENGTH) {
+ /*
+ * The caller requested a short digest. Digest
+ * into a scratch buffer and return to
+ * the user only what was requested.
+ */
+ SHA1Final(digest, &sha1_hmac_ctx.hc_ocontext);
+ bcopy(digest, (unsigned char *)mac->cd_raw.iov_base +
+ mac->cd_offset, digest_len);
+ } else {
+ SHA1Final((unsigned char *)mac->cd_raw.iov_base +
+ mac->cd_offset, &sha1_hmac_ctx.hc_ocontext);
+ }
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha1_digest_final_uio(&sha1_hmac_ctx.hc_ocontext, mac,
+ digest_len, digest);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret == CRYPTO_SUCCESS) {
+ mac->cd_length = digest_len;
+ } else {
+ mac->cd_length = 0;
+ }
+ /* Extra paranoia: zeroize the context on the stack */
+ bzero(&sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t));
+
+ return (ret);
+bail:
+ bzero(&sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t));
+ mac->cd_length = 0;
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha1_mac_verify_atomic(crypto_provider_handle_t provider,
+ crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac,
+ crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ uchar_t digest[SHA1_DIGEST_LENGTH];
+ sha1_hmac_ctx_t sha1_hmac_ctx;
+ uint32_t digest_len = SHA1_DIGEST_LENGTH;
+ uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length);
+
+ if (mechanism->cm_type != SHA1_HMAC_MECH_INFO_TYPE &&
+ mechanism->cm_type != SHA1_HMAC_GEN_MECH_INFO_TYPE)
+ return (CRYPTO_MECHANISM_INVALID);
+
+ /* Add support for key by attributes (RFE 4706552) */
+ if (key->ck_format != CRYPTO_KEY_RAW)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ if (ctx_template != NULL) {
+ /* reuse context template */
+ bcopy(ctx_template, &sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t));
+ } else {
+ /* no context template, initialize context */
+ if (keylen_in_bytes > SHA1_HMAC_BLOCK_SIZE) {
+ /*
+ * Hash the passed-in key to get a smaller key.
+ * The inner context is used since it hasn't been
+ * initialized yet.
+ */
+ PROV_SHA1_DIGEST_KEY(&sha1_hmac_ctx.hc_icontext,
+ key->ck_data, keylen_in_bytes, digest);
+ sha1_mac_init_ctx(&sha1_hmac_ctx, digest,
+ SHA1_DIGEST_LENGTH);
+ } else {
+ sha1_mac_init_ctx(&sha1_hmac_ctx, key->ck_data,
+ keylen_in_bytes);
+ }
+ }
+
+ /* get the mechanism parameters, if applicable */
+ if (mechanism->cm_type == SHA1_HMAC_GEN_MECH_INFO_TYPE) {
+ if (mechanism->cm_param == NULL ||
+ mechanism->cm_param_len != sizeof (ulong_t)) {
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto bail;
+ }
+ PROV_SHA1_GET_DIGEST_LEN(mechanism, digest_len);
+ if (digest_len > SHA1_DIGEST_LENGTH) {
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto bail;
+ }
+ }
+
+ if (mac->cd_length != digest_len) {
+ ret = CRYPTO_INVALID_MAC;
+ goto bail;
+ }
+
+ /* do a SHA1 update of the inner context using the specified data */
+ SHA1_MAC_UPDATE(data, sha1_hmac_ctx, ret);
+ if (ret != CRYPTO_SUCCESS)
+ /* the update failed, free context and bail */
+ goto bail;
+
+ /* do a SHA1 final on the inner context */
+ SHA1Final(digest, &sha1_hmac_ctx.hc_icontext);
+
+ /*
+ * Do an SHA1 update on the outer context, feeding the inner
+ * digest as data.
+ */
+ SHA1Update(&sha1_hmac_ctx.hc_ocontext, digest, SHA1_DIGEST_LENGTH);
+
+ /*
+ * Do a SHA1 final on the outer context, storing the computed
+ * digest in the users buffer.
+ */
+ SHA1Final(digest, &sha1_hmac_ctx.hc_ocontext);
+
+ /*
+ * Compare the computed digest against the expected digest passed
+ * as argument.
+ */
+
+ switch (mac->cd_format) {
+
+ case CRYPTO_DATA_RAW:
+ if (bcmp(digest, (unsigned char *)mac->cd_raw.iov_base +
+ mac->cd_offset, digest_len) != 0)
+ ret = CRYPTO_INVALID_MAC;
+ break;
+
+ case CRYPTO_DATA_UIO: {
+ off_t offset = mac->cd_offset;
+ uint_t vec_idx;
+ off_t scratch_offset = 0;
+ size_t length = digest_len;
+ size_t cur_len;
+
+ /* we support only kernel buffer */
+ if (mac->cd_uio->uio_segflg != UIO_SYSSPACE)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ /* jump to the first iovec containing the expected digest */
+ for (vec_idx = 0;
+ offset >= mac->cd_uio->uio_iov[vec_idx].iov_len &&
+ vec_idx < mac->cd_uio->uio_iovcnt;
+ offset -= mac->cd_uio->uio_iov[vec_idx++].iov_len)
+ ;
+ if (vec_idx == mac->cd_uio->uio_iovcnt) {
+ /*
+ * The caller specified an offset that is
+ * larger than the total size of the buffers
+ * it provided.
+ */
+ ret = CRYPTO_DATA_LEN_RANGE;
+ break;
+ }
+
+ /* do the comparison of computed digest vs specified one */
+ while (vec_idx < mac->cd_uio->uio_iovcnt && length > 0) {
+ cur_len = MIN(mac->cd_uio->uio_iov[vec_idx].iov_len -
+ offset, length);
+
+ if (bcmp(digest + scratch_offset,
+ mac->cd_uio->uio_iov[vec_idx].iov_base + offset,
+ cur_len) != 0) {
+ ret = CRYPTO_INVALID_MAC;
+ break;
+ }
+
+ length -= cur_len;
+ vec_idx++;
+ scratch_offset += cur_len;
+ offset = 0;
+ }
+ break;
+ }
+
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ bzero(&sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t));
+ return (ret);
+bail:
+ bzero(&sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t));
+ mac->cd_length = 0;
+ return (ret);
+}
+
+/*
+ * KCF software provider context management entry points.
+ */
+
+/* ARGSUSED */
+static int
+sha1_create_ctx_template(crypto_provider_handle_t provider,
+ crypto_mechanism_t *mechanism, crypto_key_t *key,
+ crypto_spi_ctx_template_t *ctx_template, size_t *ctx_template_size,
+ crypto_req_handle_t req)
+{
+ sha1_hmac_ctx_t *sha1_hmac_ctx_tmpl;
+ uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length);
+
+ if ((mechanism->cm_type != SHA1_HMAC_MECH_INFO_TYPE) &&
+ (mechanism->cm_type != SHA1_HMAC_GEN_MECH_INFO_TYPE)) {
+ return (CRYPTO_MECHANISM_INVALID);
+ }
+
+ /* Add support for key by attributes (RFE 4706552) */
+ if (key->ck_format != CRYPTO_KEY_RAW)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ /*
+ * Allocate and initialize SHA1 context.
+ */
+ sha1_hmac_ctx_tmpl = kmem_alloc(sizeof (sha1_hmac_ctx_t),
+ crypto_kmflag(req));
+ if (sha1_hmac_ctx_tmpl == NULL)
+ return (CRYPTO_HOST_MEMORY);
+
+ if (keylen_in_bytes > SHA1_HMAC_BLOCK_SIZE) {
+ uchar_t digested_key[SHA1_DIGEST_LENGTH];
+
+ /*
+ * Hash the passed-in key to get a smaller key.
+ * The inner context is used since it hasn't been
+ * initialized yet.
+ */
+ PROV_SHA1_DIGEST_KEY(&sha1_hmac_ctx_tmpl->hc_icontext,
+ key->ck_data, keylen_in_bytes, digested_key);
+ sha1_mac_init_ctx(sha1_hmac_ctx_tmpl, digested_key,
+ SHA1_DIGEST_LENGTH);
+ } else {
+ sha1_mac_init_ctx(sha1_hmac_ctx_tmpl, key->ck_data,
+ keylen_in_bytes);
+ }
+
+ sha1_hmac_ctx_tmpl->hc_mech_type = mechanism->cm_type;
+ *ctx_template = (crypto_spi_ctx_template_t)sha1_hmac_ctx_tmpl;
+ *ctx_template_size = sizeof (sha1_hmac_ctx_t);
+
+
+ return (CRYPTO_SUCCESS);
+}
+
+static int
+sha1_free_context(crypto_ctx_t *ctx)
+{
+ uint_t ctx_len;
+ sha1_mech_type_t mech_type;
+
+ if (ctx->cc_provider_private == NULL)
+ return (CRYPTO_SUCCESS);
+
+ /*
+ * We have to free either SHA1 or SHA1-HMAC contexts, which
+ * have different lengths.
+ */
+
+ mech_type = PROV_SHA1_CTX(ctx)->sc_mech_type;
+ if (mech_type == SHA1_MECH_INFO_TYPE)
+ ctx_len = sizeof (sha1_ctx_t);
+ else {
+ ASSERT(mech_type == SHA1_HMAC_MECH_INFO_TYPE ||
+ mech_type == SHA1_HMAC_GEN_MECH_INFO_TYPE);
+ ctx_len = sizeof (sha1_hmac_ctx_t);
+ }
+
+ bzero(ctx->cc_provider_private, ctx_len);
+ kmem_free(ctx->cc_provider_private, ctx_len);
+ ctx->cc_provider_private = NULL;
+
+ return (CRYPTO_SUCCESS);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/modctl.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/spi.h>
+#include <sys/crypto/icp.h>
+#define _SHA2_IMPL
+#include <sha2/sha2.h>
+#include <sha2/sha2_impl.h>
+
+/*
+ * The sha2 module is created with two modlinkages:
+ * - a modlmisc that allows consumers to directly call the entry points
+ * SHA2Init, SHA2Update, and SHA2Final.
+ * - a modlcrypto that allows the module to register with the Kernel
+ * Cryptographic Framework (KCF) as a software provider for the SHA2
+ * mechanisms.
+ */
+
+static struct modlcrypto modlcrypto = {
+ &mod_cryptoops,
+ "SHA2 Kernel SW Provider"
+};
+
+static struct modlinkage modlinkage = {
+ MODREV_1, {&modlcrypto, NULL}
+};
+
+/*
+ * Macros to access the SHA2 or SHA2-HMAC contexts from a context passed
+ * by KCF to one of the entry points.
+ */
+
+#define PROV_SHA2_CTX(ctx) ((sha2_ctx_t *)(ctx)->cc_provider_private)
+#define PROV_SHA2_HMAC_CTX(ctx) ((sha2_hmac_ctx_t *)(ctx)->cc_provider_private)
+
+/* to extract the digest length passed as mechanism parameter */
+#define PROV_SHA2_GET_DIGEST_LEN(m, len) { \
+ if (IS_P2ALIGNED((m)->cm_param, sizeof (ulong_t))) \
+ (len) = (uint32_t)*((ulong_t *)(m)->cm_param); \
+ else { \
+ ulong_t tmp_ulong; \
+ bcopy((m)->cm_param, &tmp_ulong, sizeof (ulong_t)); \
+ (len) = (uint32_t)tmp_ulong; \
+ } \
+}
+
+#define PROV_SHA2_DIGEST_KEY(mech, ctx, key, len, digest) { \
+ SHA2Init(mech, ctx); \
+ SHA2Update(ctx, key, len); \
+ SHA2Final(digest, ctx); \
+}
+
+/*
+ * Mechanism info structure passed to KCF during registration.
+ */
+static crypto_mech_info_t sha2_mech_info_tab[] = {
+ /* SHA256 */
+ {SUN_CKM_SHA256, SHA256_MECH_INFO_TYPE,
+ CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC,
+ 0, 0, CRYPTO_KEYSIZE_UNIT_IN_BITS},
+ /* SHA256-HMAC */
+ {SUN_CKM_SHA256_HMAC, SHA256_HMAC_MECH_INFO_TYPE,
+ CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC,
+ SHA2_HMAC_MIN_KEY_LEN, SHA2_HMAC_MAX_KEY_LEN,
+ CRYPTO_KEYSIZE_UNIT_IN_BYTES},
+ /* SHA256-HMAC GENERAL */
+ {SUN_CKM_SHA256_HMAC_GENERAL, SHA256_HMAC_GEN_MECH_INFO_TYPE,
+ CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC,
+ SHA2_HMAC_MIN_KEY_LEN, SHA2_HMAC_MAX_KEY_LEN,
+ CRYPTO_KEYSIZE_UNIT_IN_BYTES}
+};
+
+static void sha2_provider_status(crypto_provider_handle_t, uint_t *);
+
+static crypto_control_ops_t sha2_control_ops = {
+ sha2_provider_status
+};
+
+static int sha2_digest_init(crypto_ctx_t *, crypto_mechanism_t *,
+ crypto_req_handle_t);
+static int sha2_digest(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int sha2_digest_update(crypto_ctx_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int sha2_digest_final(crypto_ctx_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int sha2_digest_atomic(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_data_t *, crypto_data_t *,
+ crypto_req_handle_t);
+
+static crypto_digest_ops_t sha2_digest_ops = {
+ sha2_digest_init,
+ sha2_digest,
+ sha2_digest_update,
+ NULL,
+ sha2_digest_final,
+ sha2_digest_atomic
+};
+
+static int sha2_mac_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+static int sha2_mac_update(crypto_ctx_t *, crypto_data_t *,
+ crypto_req_handle_t);
+static int sha2_mac_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t);
+static int sha2_mac_atomic(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+static int sha2_mac_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
+ crypto_spi_ctx_template_t, crypto_req_handle_t);
+
+static crypto_mac_ops_t sha2_mac_ops = {
+ sha2_mac_init,
+ NULL,
+ sha2_mac_update,
+ sha2_mac_final,
+ sha2_mac_atomic,
+ sha2_mac_verify_atomic
+};
+
+static int sha2_create_ctx_template(crypto_provider_handle_t,
+ crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t *,
+ size_t *, crypto_req_handle_t);
+static int sha2_free_context(crypto_ctx_t *);
+
+static crypto_ctx_ops_t sha2_ctx_ops = {
+ sha2_create_ctx_template,
+ sha2_free_context
+};
+
+static crypto_ops_t sha2_crypto_ops = {{{{{
+ &sha2_control_ops,
+ &sha2_digest_ops,
+ NULL,
+ &sha2_mac_ops,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ NULL,
+ &sha2_ctx_ops
+}}}}};
+
+static crypto_provider_info_t sha2_prov_info = {{{{
+ CRYPTO_SPI_VERSION_1,
+ "SHA2 Software Provider",
+ CRYPTO_SW_PROVIDER,
+ NULL,
+ &sha2_crypto_ops,
+ sizeof (sha2_mech_info_tab)/sizeof (crypto_mech_info_t),
+ sha2_mech_info_tab
+}}}};
+
+static crypto_kcf_provider_handle_t sha2_prov_handle = 0;
+
+int
+sha2_mod_init(void)
+{
+ int ret;
+
+ if ((ret = mod_install(&modlinkage)) != 0)
+ return (ret);
+
+ /*
+ * Register with KCF. If the registration fails, log an
+ * error but do not uninstall the module, since the functionality
+ * provided by misc/sha2 should still be available.
+ */
+ if ((ret = crypto_register_provider(&sha2_prov_info,
+ &sha2_prov_handle)) != CRYPTO_SUCCESS)
+ cmn_err(CE_WARN, "sha2 _init: "
+ "crypto_register_provider() failed (0x%x)", ret);
+
+ return (0);
+}
+
+int
+sha2_mod_fini(void)
+{
+ int ret;
+
+ if (sha2_prov_handle != 0) {
+ if ((ret = crypto_unregister_provider(sha2_prov_handle)) !=
+ CRYPTO_SUCCESS) {
+ cmn_err(CE_WARN,
+ "sha2 _fini: crypto_unregister_provider() "
+ "failed (0x%x)", ret);
+ return (EBUSY);
+ }
+ sha2_prov_handle = 0;
+ }
+
+ return (mod_remove(&modlinkage));
+}
+
+/*
+ * KCF software provider control entry points.
+ */
+/* ARGSUSED */
+static void
+sha2_provider_status(crypto_provider_handle_t provider, uint_t *status)
+{
+ *status = CRYPTO_PROVIDER_READY;
+}
+
+/*
+ * KCF software provider digest entry points.
+ */
+
+static int
+sha2_digest_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
+ crypto_req_handle_t req)
+{
+
+ /*
+ * Allocate and initialize SHA2 context.
+ */
+ ctx->cc_provider_private = kmem_alloc(sizeof (sha2_ctx_t),
+ crypto_kmflag(req));
+ if (ctx->cc_provider_private == NULL)
+ return (CRYPTO_HOST_MEMORY);
+
+ PROV_SHA2_CTX(ctx)->sc_mech_type = mechanism->cm_type;
+ SHA2Init(mechanism->cm_type, &PROV_SHA2_CTX(ctx)->sc_sha2_ctx);
+
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Helper SHA2 digest update function for uio data.
+ */
+static int
+sha2_digest_update_uio(SHA2_CTX *sha2_ctx, crypto_data_t *data)
+{
+ off_t offset = data->cd_offset;
+ size_t length = data->cd_length;
+ uint_t vec_idx;
+ size_t cur_len;
+
+ /* we support only kernel buffer */
+ if (data->cd_uio->uio_segflg != UIO_SYSSPACE)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ /*
+ * Jump to the first iovec containing data to be
+ * digested.
+ */
+ for (vec_idx = 0; vec_idx < data->cd_uio->uio_iovcnt &&
+ offset >= data->cd_uio->uio_iov[vec_idx].iov_len;
+ offset -= data->cd_uio->uio_iov[vec_idx++].iov_len)
+ ;
+ if (vec_idx == data->cd_uio->uio_iovcnt) {
+ /*
+ * The caller specified an offset that is larger than the
+ * total size of the buffers it provided.
+ */
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ /*
+ * Now do the digesting on the iovecs.
+ */
+ while (vec_idx < data->cd_uio->uio_iovcnt && length > 0) {
+ cur_len = MIN(data->cd_uio->uio_iov[vec_idx].iov_len -
+ offset, length);
+
+ SHA2Update(sha2_ctx, (uint8_t *)data->cd_uio->
+ uio_iov[vec_idx].iov_base + offset, cur_len);
+ length -= cur_len;
+ vec_idx++;
+ offset = 0;
+ }
+
+ if (vec_idx == data->cd_uio->uio_iovcnt && length > 0) {
+ /*
+ * The end of the specified iovec's was reached but
+ * the length requested could not be processed, i.e.
+ * The caller requested to digest more data than it provided.
+ */
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Helper SHA2 digest final function for uio data.
+ * digest_len is the length of the desired digest. If digest_len
+ * is smaller than the default SHA2 digest length, the caller
+ * must pass a scratch buffer, digest_scratch, which must
+ * be at least the algorithm's digest length bytes.
+ */
+static int
+sha2_digest_final_uio(SHA2_CTX *sha2_ctx, crypto_data_t *digest,
+ ulong_t digest_len, uchar_t *digest_scratch)
+{
+ off_t offset = digest->cd_offset;
+ uint_t vec_idx;
+
+ /* we support only kernel buffer */
+ if (digest->cd_uio->uio_segflg != UIO_SYSSPACE)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ /*
+ * Jump to the first iovec containing ptr to the digest to
+ * be returned.
+ */
+ for (vec_idx = 0; offset >= digest->cd_uio->uio_iov[vec_idx].iov_len &&
+ vec_idx < digest->cd_uio->uio_iovcnt;
+ offset -= digest->cd_uio->uio_iov[vec_idx++].iov_len)
+ ;
+ if (vec_idx == digest->cd_uio->uio_iovcnt) {
+ /*
+ * The caller specified an offset that is
+ * larger than the total size of the buffers
+ * it provided.
+ */
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+
+ if (offset + digest_len <=
+ digest->cd_uio->uio_iov[vec_idx].iov_len) {
+ /*
+ * The computed SHA2 digest will fit in the current
+ * iovec.
+ */
+ if (((sha2_ctx->algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) &&
+ (digest_len != SHA256_DIGEST_LENGTH))) {
+ /*
+ * The caller requested a short digest. Digest
+ * into a scratch buffer and return to
+ * the user only what was requested.
+ */
+ SHA2Final(digest_scratch, sha2_ctx);
+
+ bcopy(digest_scratch, (uchar_t *)digest->
+ cd_uio->uio_iov[vec_idx].iov_base + offset,
+ digest_len);
+ } else {
+ SHA2Final((uchar_t *)digest->
+ cd_uio->uio_iov[vec_idx].iov_base + offset,
+ sha2_ctx);
+
+ }
+ } else {
+ /*
+ * The computed digest will be crossing one or more iovec's.
+ * This is bad performance-wise but we need to support it.
+ * Allocate a small scratch buffer on the stack and
+ * copy it piece meal to the specified digest iovec's.
+ */
+ uchar_t digest_tmp[SHA256_DIGEST_LENGTH];
+ off_t scratch_offset = 0;
+ size_t length = digest_len;
+ size_t cur_len;
+
+ SHA2Final(digest_tmp, sha2_ctx);
+
+ while (vec_idx < digest->cd_uio->uio_iovcnt && length > 0) {
+ cur_len =
+ MIN(digest->cd_uio->uio_iov[vec_idx].iov_len -
+ offset, length);
+ bcopy(digest_tmp + scratch_offset,
+ digest->cd_uio->uio_iov[vec_idx].iov_base + offset,
+ cur_len);
+
+ length -= cur_len;
+ vec_idx++;
+ scratch_offset += cur_len;
+ offset = 0;
+ }
+
+ if (vec_idx == digest->cd_uio->uio_iovcnt && length > 0) {
+ /*
+ * The end of the specified iovec's was reached but
+ * the length requested could not be processed, i.e.
+ * The caller requested to digest more data than it
+ * provided.
+ */
+ return (CRYPTO_DATA_LEN_RANGE);
+ }
+ }
+
+ return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+static int
+sha2_digest(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *digest,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ uint_t sha_digest_len;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+
+ switch (PROV_SHA2_CTX(ctx)->sc_mech_type) {
+ case SHA256_MECH_INFO_TYPE:
+ sha_digest_len = SHA256_DIGEST_LENGTH;
+ break;
+ default:
+ return (CRYPTO_MECHANISM_INVALID);
+ }
+
+ /*
+ * We need to just return the length needed to store the output.
+ * We should not destroy the context for the following cases.
+ */
+ if ((digest->cd_length == 0) ||
+ (digest->cd_length < sha_digest_len)) {
+ digest->cd_length = sha_digest_len;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ /*
+ * Do the SHA2 update on the specified input data.
+ */
+ switch (data->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA2Update(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx,
+ (uint8_t *)data->cd_raw.iov_base + data->cd_offset,
+ data->cd_length);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha2_digest_update_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx,
+ data);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret != CRYPTO_SUCCESS) {
+ /* the update failed, free context and bail */
+ kmem_free(ctx->cc_provider_private, sizeof (sha2_ctx_t));
+ ctx->cc_provider_private = NULL;
+ digest->cd_length = 0;
+ return (ret);
+ }
+
+ /*
+ * Do a SHA2 final, must be done separately since the digest
+ * type can be different than the input data type.
+ */
+ switch (digest->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA2Final((unsigned char *)digest->cd_raw.iov_base +
+ digest->cd_offset, &PROV_SHA2_CTX(ctx)->sc_sha2_ctx);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha2_digest_final_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx,
+ digest, sha_digest_len, NULL);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ /* all done, free context and return */
+
+ if (ret == CRYPTO_SUCCESS)
+ digest->cd_length = sha_digest_len;
+ else
+ digest->cd_length = 0;
+
+ kmem_free(ctx->cc_provider_private, sizeof (sha2_ctx_t));
+ ctx->cc_provider_private = NULL;
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha2_digest_update(crypto_ctx_t *ctx, crypto_data_t *data,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+
+ /*
+ * Do the SHA2 update on the specified input data.
+ */
+ switch (data->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA2Update(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx,
+ (uint8_t *)data->cd_raw.iov_base + data->cd_offset,
+ data->cd_length);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha2_digest_update_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx,
+ data);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha2_digest_final(crypto_ctx_t *ctx, crypto_data_t *digest,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ uint_t sha_digest_len;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+
+ switch (PROV_SHA2_CTX(ctx)->sc_mech_type) {
+ case SHA256_MECH_INFO_TYPE:
+ sha_digest_len = SHA256_DIGEST_LENGTH;
+ break;
+ default:
+ return (CRYPTO_MECHANISM_INVALID);
+ }
+
+ /*
+ * We need to just return the length needed to store the output.
+ * We should not destroy the context for the following cases.
+ */
+ if ((digest->cd_length == 0) ||
+ (digest->cd_length < sha_digest_len)) {
+ digest->cd_length = sha_digest_len;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ /*
+ * Do a SHA2 final.
+ */
+ switch (digest->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA2Final((unsigned char *)digest->cd_raw.iov_base +
+ digest->cd_offset, &PROV_SHA2_CTX(ctx)->sc_sha2_ctx);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha2_digest_final_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx,
+ digest, sha_digest_len, NULL);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ /* all done, free context and return */
+
+ if (ret == CRYPTO_SUCCESS)
+ digest->cd_length = sha_digest_len;
+ else
+ digest->cd_length = 0;
+
+ kmem_free(ctx->cc_provider_private, sizeof (sha2_ctx_t));
+ ctx->cc_provider_private = NULL;
+
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha2_digest_atomic(crypto_provider_handle_t provider,
+ crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+ crypto_data_t *data, crypto_data_t *digest,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ SHA2_CTX sha2_ctx;
+ uint32_t sha_digest_len;
+
+ /*
+ * Do the SHA inits.
+ */
+
+ SHA2Init(mechanism->cm_type, &sha2_ctx);
+
+ switch (data->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA2Update(&sha2_ctx, (uint8_t *)data->
+ cd_raw.iov_base + data->cd_offset, data->cd_length);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha2_digest_update_uio(&sha2_ctx, data);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ /*
+ * Do the SHA updates on the specified input data.
+ */
+
+ if (ret != CRYPTO_SUCCESS) {
+ /* the update failed, bail */
+ digest->cd_length = 0;
+ return (ret);
+ }
+
+ if (mechanism->cm_type <= SHA256_HMAC_GEN_MECH_INFO_TYPE)
+ sha_digest_len = SHA256_DIGEST_LENGTH;
+
+ /*
+ * Do a SHA2 final, must be done separately since the digest
+ * type can be different than the input data type.
+ */
+ switch (digest->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA2Final((unsigned char *)digest->cd_raw.iov_base +
+ digest->cd_offset, &sha2_ctx);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha2_digest_final_uio(&sha2_ctx, digest,
+ sha_digest_len, NULL);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret == CRYPTO_SUCCESS)
+ digest->cd_length = sha_digest_len;
+ else
+ digest->cd_length = 0;
+
+ return (ret);
+}
+
+/*
+ * KCF software provider mac entry points.
+ *
+ * SHA2 HMAC is: SHA2(key XOR opad, SHA2(key XOR ipad, text))
+ *
+ * Init:
+ * The initialization routine initializes what we denote
+ * as the inner and outer contexts by doing
+ * - for inner context: SHA2(key XOR ipad)
+ * - for outer context: SHA2(key XOR opad)
+ *
+ * Update:
+ * Each subsequent SHA2 HMAC update will result in an
+ * update of the inner context with the specified data.
+ *
+ * Final:
+ * The SHA2 HMAC final will do a SHA2 final operation on the
+ * inner context, and the resulting digest will be used
+ * as the data for an update on the outer context. Last
+ * but not least, a SHA2 final on the outer context will
+ * be performed to obtain the SHA2 HMAC digest to return
+ * to the user.
+ */
+
+/*
+ * Initialize a SHA2-HMAC context.
+ */
+static void
+sha2_mac_init_ctx(sha2_hmac_ctx_t *ctx, void *keyval, uint_t length_in_bytes)
+{
+ uint64_t ipad[SHA256_HMAC_BLOCK_SIZE / sizeof (uint64_t)];
+ uint64_t opad[SHA256_HMAC_BLOCK_SIZE / sizeof (uint64_t)];
+ int i, block_size, blocks_per_int64;
+
+ /* Determine the block size */
+ if (ctx->hc_mech_type <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
+ block_size = SHA256_HMAC_BLOCK_SIZE;
+ blocks_per_int64 = SHA256_HMAC_BLOCK_SIZE / sizeof (uint64_t);
+ }
+
+ (void) bzero(ipad, block_size);
+ (void) bzero(opad, block_size);
+ (void) bcopy(keyval, ipad, length_in_bytes);
+ (void) bcopy(keyval, opad, length_in_bytes);
+
+ /* XOR key with ipad (0x36) and opad (0x5c) */
+ for (i = 0; i < blocks_per_int64; i ++) {
+ ipad[i] ^= 0x3636363636363636;
+ opad[i] ^= 0x5c5c5c5c5c5c5c5c;
+ }
+
+ /* perform SHA2 on ipad */
+ SHA2Init(ctx->hc_mech_type, &ctx->hc_icontext);
+ SHA2Update(&ctx->hc_icontext, (uint8_t *)ipad, block_size);
+
+ /* perform SHA2 on opad */
+ SHA2Init(ctx->hc_mech_type, &ctx->hc_ocontext);
+ SHA2Update(&ctx->hc_ocontext, (uint8_t *)opad, block_size);
+
+}
+
+/*
+ */
+static int
+sha2_mac_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length);
+ uint_t sha_digest_len, sha_hmac_block_size;
+
+ /*
+ * Set the digest length and block size to values approriate to the
+ * mechanism
+ */
+ switch (mechanism->cm_type) {
+ case SHA256_HMAC_MECH_INFO_TYPE:
+ case SHA256_HMAC_GEN_MECH_INFO_TYPE:
+ sha_digest_len = SHA256_DIGEST_LENGTH;
+ sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE;
+ break;
+ default:
+ return (CRYPTO_MECHANISM_INVALID);
+ }
+
+ if (key->ck_format != CRYPTO_KEY_RAW)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ ctx->cc_provider_private = kmem_alloc(sizeof (sha2_hmac_ctx_t),
+ crypto_kmflag(req));
+ if (ctx->cc_provider_private == NULL)
+ return (CRYPTO_HOST_MEMORY);
+
+ PROV_SHA2_HMAC_CTX(ctx)->hc_mech_type = mechanism->cm_type;
+ if (ctx_template != NULL) {
+ /* reuse context template */
+ bcopy(ctx_template, PROV_SHA2_HMAC_CTX(ctx),
+ sizeof (sha2_hmac_ctx_t));
+ } else {
+ /* no context template, compute context */
+ if (keylen_in_bytes > sha_hmac_block_size) {
+ uchar_t digested_key[SHA256_DIGEST_LENGTH];
+ sha2_hmac_ctx_t *hmac_ctx = ctx->cc_provider_private;
+
+ /*
+ * Hash the passed-in key to get a smaller key.
+ * The inner context is used since it hasn't been
+ * initialized yet.
+ */
+ PROV_SHA2_DIGEST_KEY(mechanism->cm_type / 3,
+ &hmac_ctx->hc_icontext,
+ key->ck_data, keylen_in_bytes, digested_key);
+ sha2_mac_init_ctx(PROV_SHA2_HMAC_CTX(ctx),
+ digested_key, sha_digest_len);
+ } else {
+ sha2_mac_init_ctx(PROV_SHA2_HMAC_CTX(ctx),
+ key->ck_data, keylen_in_bytes);
+ }
+ }
+
+ /*
+ * Get the mechanism parameters, if applicable.
+ */
+ if (mechanism->cm_type % 3 == 2) {
+ if (mechanism->cm_param == NULL ||
+ mechanism->cm_param_len != sizeof (ulong_t))
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ PROV_SHA2_GET_DIGEST_LEN(mechanism,
+ PROV_SHA2_HMAC_CTX(ctx)->hc_digest_len);
+ if (PROV_SHA2_HMAC_CTX(ctx)->hc_digest_len > sha_digest_len)
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ }
+
+ if (ret != CRYPTO_SUCCESS) {
+ bzero(ctx->cc_provider_private, sizeof (sha2_hmac_ctx_t));
+ kmem_free(ctx->cc_provider_private, sizeof (sha2_hmac_ctx_t));
+ ctx->cc_provider_private = NULL;
+ }
+
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha2_mac_update(crypto_ctx_t *ctx, crypto_data_t *data,
+ crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+
+ /*
+ * Do a SHA2 update of the inner context using the specified
+ * data.
+ */
+ switch (data->cd_format) {
+ case CRYPTO_DATA_RAW:
+ SHA2Update(&PROV_SHA2_HMAC_CTX(ctx)->hc_icontext,
+ (uint8_t *)data->cd_raw.iov_base + data->cd_offset,
+ data->cd_length);
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha2_digest_update_uio(
+ &PROV_SHA2_HMAC_CTX(ctx)->hc_icontext, data);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha2_mac_final(crypto_ctx_t *ctx, crypto_data_t *mac, crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ uchar_t digest[SHA256_DIGEST_LENGTH];
+ uint32_t digest_len = 0, sha_digest_len = 0;
+
+ ASSERT(ctx->cc_provider_private != NULL);
+
+ /* Set the digest lengths to values approriate to the mechanism */
+ switch (PROV_SHA2_HMAC_CTX(ctx)->hc_mech_type) {
+ case SHA256_HMAC_MECH_INFO_TYPE:
+ sha_digest_len = digest_len = SHA256_DIGEST_LENGTH;
+ break;
+ case SHA256_HMAC_GEN_MECH_INFO_TYPE:
+ sha_digest_len = SHA256_DIGEST_LENGTH;
+ digest_len = PROV_SHA2_HMAC_CTX(ctx)->hc_digest_len;
+ break;
+ default:
+ break;
+ }
+
+ /*
+ * We need to just return the length needed to store the output.
+ * We should not destroy the context for the following cases.
+ */
+ if ((mac->cd_length == 0) || (mac->cd_length < digest_len)) {
+ mac->cd_length = digest_len;
+ return (CRYPTO_BUFFER_TOO_SMALL);
+ }
+
+ /*
+ * Do a SHA2 final on the inner context.
+ */
+ SHA2Final(digest, &PROV_SHA2_HMAC_CTX(ctx)->hc_icontext);
+
+ /*
+ * Do a SHA2 update on the outer context, feeding the inner
+ * digest as data.
+ */
+ SHA2Update(&PROV_SHA2_HMAC_CTX(ctx)->hc_ocontext, digest,
+ sha_digest_len);
+
+ /*
+ * Do a SHA2 final on the outer context, storing the computing
+ * digest in the users buffer.
+ */
+ switch (mac->cd_format) {
+ case CRYPTO_DATA_RAW:
+ if (digest_len != sha_digest_len) {
+ /*
+ * The caller requested a short digest. Digest
+ * into a scratch buffer and return to
+ * the user only what was requested.
+ */
+ SHA2Final(digest,
+ &PROV_SHA2_HMAC_CTX(ctx)->hc_ocontext);
+ bcopy(digest, (unsigned char *)mac->cd_raw.iov_base +
+ mac->cd_offset, digest_len);
+ } else {
+ SHA2Final((unsigned char *)mac->cd_raw.iov_base +
+ mac->cd_offset,
+ &PROV_SHA2_HMAC_CTX(ctx)->hc_ocontext);
+ }
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha2_digest_final_uio(
+ &PROV_SHA2_HMAC_CTX(ctx)->hc_ocontext, mac,
+ digest_len, digest);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret == CRYPTO_SUCCESS)
+ mac->cd_length = digest_len;
+ else
+ mac->cd_length = 0;
+
+ bzero(ctx->cc_provider_private, sizeof (sha2_hmac_ctx_t));
+ kmem_free(ctx->cc_provider_private, sizeof (sha2_hmac_ctx_t));
+ ctx->cc_provider_private = NULL;
+
+ return (ret);
+}
+
+#define SHA2_MAC_UPDATE(data, ctx, ret) { \
+ switch (data->cd_format) { \
+ case CRYPTO_DATA_RAW: \
+ SHA2Update(&(ctx).hc_icontext, \
+ (uint8_t *)data->cd_raw.iov_base + \
+ data->cd_offset, data->cd_length); \
+ break; \
+ case CRYPTO_DATA_UIO: \
+ ret = sha2_digest_update_uio(&(ctx).hc_icontext, data); \
+ break; \
+ default: \
+ ret = CRYPTO_ARGUMENTS_BAD; \
+ } \
+}
+
+/* ARGSUSED */
+static int
+sha2_mac_atomic(crypto_provider_handle_t provider,
+ crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac,
+ crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ uchar_t digest[SHA256_DIGEST_LENGTH];
+ sha2_hmac_ctx_t sha2_hmac_ctx;
+ uint32_t sha_digest_len, digest_len, sha_hmac_block_size;
+ uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length);
+
+ /*
+ * Set the digest length and block size to values appropriate to the
+ * mechanism
+ */
+ switch (mechanism->cm_type) {
+ case SHA256_HMAC_MECH_INFO_TYPE:
+ case SHA256_HMAC_GEN_MECH_INFO_TYPE:
+ sha_digest_len = digest_len = SHA256_DIGEST_LENGTH;
+ sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE;
+ break;
+ default:
+ return (CRYPTO_MECHANISM_INVALID);
+ }
+
+ /* Add support for key by attributes (RFE 4706552) */
+ if (key->ck_format != CRYPTO_KEY_RAW)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ if (ctx_template != NULL) {
+ /* reuse context template */
+ bcopy(ctx_template, &sha2_hmac_ctx, sizeof (sha2_hmac_ctx_t));
+ } else {
+ sha2_hmac_ctx.hc_mech_type = mechanism->cm_type;
+ /* no context template, initialize context */
+ if (keylen_in_bytes > sha_hmac_block_size) {
+ /*
+ * Hash the passed-in key to get a smaller key.
+ * The inner context is used since it hasn't been
+ * initialized yet.
+ */
+ PROV_SHA2_DIGEST_KEY(mechanism->cm_type / 3,
+ &sha2_hmac_ctx.hc_icontext,
+ key->ck_data, keylen_in_bytes, digest);
+ sha2_mac_init_ctx(&sha2_hmac_ctx, digest,
+ sha_digest_len);
+ } else {
+ sha2_mac_init_ctx(&sha2_hmac_ctx, key->ck_data,
+ keylen_in_bytes);
+ }
+ }
+
+ /* get the mechanism parameters, if applicable */
+ if ((mechanism->cm_type % 3) == 2) {
+ if (mechanism->cm_param == NULL ||
+ mechanism->cm_param_len != sizeof (ulong_t)) {
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto bail;
+ }
+ PROV_SHA2_GET_DIGEST_LEN(mechanism, digest_len);
+ if (digest_len > sha_digest_len) {
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto bail;
+ }
+ }
+
+ /* do a SHA2 update of the inner context using the specified data */
+ SHA2_MAC_UPDATE(data, sha2_hmac_ctx, ret);
+ if (ret != CRYPTO_SUCCESS)
+ /* the update failed, free context and bail */
+ goto bail;
+
+ /*
+ * Do a SHA2 final on the inner context.
+ */
+ SHA2Final(digest, &sha2_hmac_ctx.hc_icontext);
+
+ /*
+ * Do an SHA2 update on the outer context, feeding the inner
+ * digest as data.
+ */
+ SHA2Update(&sha2_hmac_ctx.hc_ocontext, digest, sha_digest_len);
+
+ /*
+ * Do a SHA2 final on the outer context, storing the computed
+ * digest in the users buffer.
+ */
+ switch (mac->cd_format) {
+ case CRYPTO_DATA_RAW:
+ if (digest_len != sha_digest_len) {
+ /*
+ * The caller requested a short digest. Digest
+ * into a scratch buffer and return to
+ * the user only what was requested.
+ */
+ SHA2Final(digest, &sha2_hmac_ctx.hc_ocontext);
+ bcopy(digest, (unsigned char *)mac->cd_raw.iov_base +
+ mac->cd_offset, digest_len);
+ } else {
+ SHA2Final((unsigned char *)mac->cd_raw.iov_base +
+ mac->cd_offset, &sha2_hmac_ctx.hc_ocontext);
+ }
+ break;
+ case CRYPTO_DATA_UIO:
+ ret = sha2_digest_final_uio(&sha2_hmac_ctx.hc_ocontext, mac,
+ digest_len, digest);
+ break;
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ if (ret == CRYPTO_SUCCESS) {
+ mac->cd_length = digest_len;
+ return (CRYPTO_SUCCESS);
+ }
+bail:
+ bzero(&sha2_hmac_ctx, sizeof (sha2_hmac_ctx_t));
+ mac->cd_length = 0;
+ return (ret);
+}
+
+/* ARGSUSED */
+static int
+sha2_mac_verify_atomic(crypto_provider_handle_t provider,
+ crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
+ crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac,
+ crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
+{
+ int ret = CRYPTO_SUCCESS;
+ uchar_t digest[SHA256_DIGEST_LENGTH];
+ sha2_hmac_ctx_t sha2_hmac_ctx;
+ uint32_t sha_digest_len, digest_len, sha_hmac_block_size;
+ uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length);
+
+ /*
+ * Set the digest length and block size to values appropriate to the
+ * mechanism
+ */
+ switch (mechanism->cm_type) {
+ case SHA256_HMAC_MECH_INFO_TYPE:
+ case SHA256_HMAC_GEN_MECH_INFO_TYPE:
+ sha_digest_len = digest_len = SHA256_DIGEST_LENGTH;
+ sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE;
+ break;
+ default:
+ return (CRYPTO_MECHANISM_INVALID);
+ }
+
+ /* Add support for key by attributes (RFE 4706552) */
+ if (key->ck_format != CRYPTO_KEY_RAW)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ if (ctx_template != NULL) {
+ /* reuse context template */
+ bcopy(ctx_template, &sha2_hmac_ctx, sizeof (sha2_hmac_ctx_t));
+ } else {
+ sha2_hmac_ctx.hc_mech_type = mechanism->cm_type;
+ /* no context template, initialize context */
+ if (keylen_in_bytes > sha_hmac_block_size) {
+ /*
+ * Hash the passed-in key to get a smaller key.
+ * The inner context is used since it hasn't been
+ * initialized yet.
+ */
+ PROV_SHA2_DIGEST_KEY(mechanism->cm_type / 3,
+ &sha2_hmac_ctx.hc_icontext,
+ key->ck_data, keylen_in_bytes, digest);
+ sha2_mac_init_ctx(&sha2_hmac_ctx, digest,
+ sha_digest_len);
+ } else {
+ sha2_mac_init_ctx(&sha2_hmac_ctx, key->ck_data,
+ keylen_in_bytes);
+ }
+ }
+
+ /* get the mechanism parameters, if applicable */
+ if (mechanism->cm_type % 3 == 2) {
+ if (mechanism->cm_param == NULL ||
+ mechanism->cm_param_len != sizeof (ulong_t)) {
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto bail;
+ }
+ PROV_SHA2_GET_DIGEST_LEN(mechanism, digest_len);
+ if (digest_len > sha_digest_len) {
+ ret = CRYPTO_MECHANISM_PARAM_INVALID;
+ goto bail;
+ }
+ }
+
+ if (mac->cd_length != digest_len) {
+ ret = CRYPTO_INVALID_MAC;
+ goto bail;
+ }
+
+ /* do a SHA2 update of the inner context using the specified data */
+ SHA2_MAC_UPDATE(data, sha2_hmac_ctx, ret);
+ if (ret != CRYPTO_SUCCESS)
+ /* the update failed, free context and bail */
+ goto bail;
+
+ /* do a SHA2 final on the inner context */
+ SHA2Final(digest, &sha2_hmac_ctx.hc_icontext);
+
+ /*
+ * Do an SHA2 update on the outer context, feeding the inner
+ * digest as data.
+ */
+ SHA2Update(&sha2_hmac_ctx.hc_ocontext, digest, sha_digest_len);
+
+ /*
+ * Do a SHA2 final on the outer context, storing the computed
+ * digest in the users buffer.
+ */
+ SHA2Final(digest, &sha2_hmac_ctx.hc_ocontext);
+
+ /*
+ * Compare the computed digest against the expected digest passed
+ * as argument.
+ */
+
+ switch (mac->cd_format) {
+
+ case CRYPTO_DATA_RAW:
+ if (bcmp(digest, (unsigned char *)mac->cd_raw.iov_base +
+ mac->cd_offset, digest_len) != 0)
+ ret = CRYPTO_INVALID_MAC;
+ break;
+
+ case CRYPTO_DATA_UIO: {
+ off_t offset = mac->cd_offset;
+ uint_t vec_idx;
+ off_t scratch_offset = 0;
+ size_t length = digest_len;
+ size_t cur_len;
+
+ /* we support only kernel buffer */
+ if (mac->cd_uio->uio_segflg != UIO_SYSSPACE)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ /* jump to the first iovec containing the expected digest */
+ for (vec_idx = 0;
+ offset >= mac->cd_uio->uio_iov[vec_idx].iov_len &&
+ vec_idx < mac->cd_uio->uio_iovcnt;
+ offset -= mac->cd_uio->uio_iov[vec_idx++].iov_len)
+ ;
+ if (vec_idx == mac->cd_uio->uio_iovcnt) {
+ /*
+ * The caller specified an offset that is
+ * larger than the total size of the buffers
+ * it provided.
+ */
+ ret = CRYPTO_DATA_LEN_RANGE;
+ break;
+ }
+
+ /* do the comparison of computed digest vs specified one */
+ while (vec_idx < mac->cd_uio->uio_iovcnt && length > 0) {
+ cur_len = MIN(mac->cd_uio->uio_iov[vec_idx].iov_len -
+ offset, length);
+
+ if (bcmp(digest + scratch_offset,
+ mac->cd_uio->uio_iov[vec_idx].iov_base + offset,
+ cur_len) != 0) {
+ ret = CRYPTO_INVALID_MAC;
+ break;
+ }
+
+ length -= cur_len;
+ vec_idx++;
+ scratch_offset += cur_len;
+ offset = 0;
+ }
+ break;
+ }
+
+ default:
+ ret = CRYPTO_ARGUMENTS_BAD;
+ }
+
+ return (ret);
+bail:
+ bzero(&sha2_hmac_ctx, sizeof (sha2_hmac_ctx_t));
+ mac->cd_length = 0;
+ return (ret);
+}
+
+/*
+ * KCF software provider context management entry points.
+ */
+
+/* ARGSUSED */
+static int
+sha2_create_ctx_template(crypto_provider_handle_t provider,
+ crypto_mechanism_t *mechanism, crypto_key_t *key,
+ crypto_spi_ctx_template_t *ctx_template, size_t *ctx_template_size,
+ crypto_req_handle_t req)
+{
+ sha2_hmac_ctx_t *sha2_hmac_ctx_tmpl;
+ uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length);
+ uint32_t sha_digest_len, sha_hmac_block_size;
+
+ /*
+ * Set the digest length and block size to values appropriate to the
+ * mechanism
+ */
+ switch (mechanism->cm_type) {
+ case SHA256_HMAC_MECH_INFO_TYPE:
+ case SHA256_HMAC_GEN_MECH_INFO_TYPE:
+ sha_digest_len = SHA256_DIGEST_LENGTH;
+ sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE;
+ break;
+ default:
+ return (CRYPTO_MECHANISM_INVALID);
+ }
+
+ /* Add support for key by attributes (RFE 4706552) */
+ if (key->ck_format != CRYPTO_KEY_RAW)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ /*
+ * Allocate and initialize SHA2 context.
+ */
+ sha2_hmac_ctx_tmpl = kmem_alloc(sizeof (sha2_hmac_ctx_t),
+ crypto_kmflag(req));
+ if (sha2_hmac_ctx_tmpl == NULL)
+ return (CRYPTO_HOST_MEMORY);
+
+ sha2_hmac_ctx_tmpl->hc_mech_type = mechanism->cm_type;
+
+ if (keylen_in_bytes > sha_hmac_block_size) {
+ uchar_t digested_key[SHA256_DIGEST_LENGTH];
+
+ /*
+ * Hash the passed-in key to get a smaller key.
+ * The inner context is used since it hasn't been
+ * initialized yet.
+ */
+ PROV_SHA2_DIGEST_KEY(mechanism->cm_type / 3,
+ &sha2_hmac_ctx_tmpl->hc_icontext,
+ key->ck_data, keylen_in_bytes, digested_key);
+ sha2_mac_init_ctx(sha2_hmac_ctx_tmpl, digested_key,
+ sha_digest_len);
+ } else {
+ sha2_mac_init_ctx(sha2_hmac_ctx_tmpl, key->ck_data,
+ keylen_in_bytes);
+ }
+
+ *ctx_template = (crypto_spi_ctx_template_t)sha2_hmac_ctx_tmpl;
+ *ctx_template_size = sizeof (sha2_hmac_ctx_t);
+
+ return (CRYPTO_SUCCESS);
+}
+
+static int
+sha2_free_context(crypto_ctx_t *ctx)
+{
+ uint_t ctx_len;
+
+ if (ctx->cc_provider_private == NULL)
+ return (CRYPTO_SUCCESS);
+
+ /*
+ * We have to free either SHA2 or SHA2-HMAC contexts, which
+ * have different lengths.
+ *
+ * Note: Below is dependent on the mechanism ordering.
+ */
+
+ if (PROV_SHA2_CTX(ctx)->sc_mech_type % 3 == 0)
+ ctx_len = sizeof (sha2_ctx_t);
+ else
+ ctx_len = sizeof (sha2_hmac_ctx_t);
+
+ bzero(ctx->cc_provider_private, ctx_len);
+ kmem_free(ctx->cc_provider_private, ctx_len);
+ ctx->cc_provider_private = NULL;
+
+ return (CRYPTO_SUCCESS);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/modctl.h>
+
+/*
+ * Null operations; used for uninitialized and "misc" modules.
+ */
+static int mod_null(struct modlmisc *, struct modlinkage *);
+static int mod_infonull(void *, struct modlinkage *, int *);
+
+/*
+ * Cryptographic Modules
+ */
+struct mod_ops mod_cryptoops = {
+ mod_null, mod_null, mod_infonull
+};
+
+/*
+ * Null operation; return 0.
+ */
+static int
+mod_null(struct modlmisc *modl, struct modlinkage *modlp)
+{
+ return (0);
+}
+
+/*
+ * Status for User modules.
+ */
+static int
+mod_infonull(void *modl, struct modlinkage *modlp, int *p0)
+{
+ *p0 = -1; /* for modinfo display */
+ return (0);
+}
+
+/*
+ * Install a module.
+ * (This routine is in the Solaris SPARC DDI/DKI)
+ */
+int
+mod_install(struct modlinkage *modlp)
+{
+ int retval = -1; /* No linkage structures */
+ struct modlmisc **linkpp;
+ struct modlmisc **linkpp1;
+
+ if (modlp->ml_rev != MODREV_1) {
+ cmn_err(CE_WARN, "mod_install: "
+ "modlinkage structure is not MODREV_1\n");
+ return (EINVAL);
+ }
+ linkpp = (struct modlmisc **)&modlp->ml_linkage[0];
+
+ while (*linkpp != NULL) {
+ if ((retval = MODL_INSTALL(*linkpp, modlp)) != 0) {
+ linkpp1 = (struct modlmisc **)&modlp->ml_linkage[0];
+
+ while (linkpp1 != linkpp) {
+ MODL_REMOVE(*linkpp1, modlp); /* clean up */
+ linkpp1++;
+ }
+ break;
+ }
+ linkpp++;
+ }
+ return (retval);
+}
+
+static char *reins_err =
+ "Could not reinstall %s\nReboot to correct the problem";
+
+/*
+ * Remove a module. This is called by the module wrapper routine.
+ * (This routine is in the Solaris SPARC DDI/DKI)
+ */
+int
+mod_remove(struct modlinkage *modlp)
+{
+ int retval = 0;
+ struct modlmisc **linkpp, *last_linkp;
+
+ linkpp = (struct modlmisc **)&modlp->ml_linkage[0];
+
+ while (*linkpp != NULL) {
+ if ((retval = MODL_REMOVE(*linkpp, modlp)) != 0) {
+ last_linkp = *linkpp;
+ linkpp = (struct modlmisc **)&modlp->ml_linkage[0];
+ while (*linkpp != last_linkp) {
+ if (MODL_INSTALL(*linkpp, modlp) != 0) {
+ cmn_err(CE_WARN, reins_err,
+ (*linkpp)->misc_linkinfo);
+ break;
+ }
+ linkpp++;
+ }
+ break;
+ }
+ linkpp++;
+ }
+ return (retval);
+}
+
+/*
+ * Get module status.
+ * (This routine is in the Solaris SPARC DDI/DKI)
+ */
+int
+mod_info(struct modlinkage *modlp, struct modinfo *modinfop)
+{
+ int i;
+ int retval = 0;
+ struct modspecific_info *msip;
+ struct modlmisc **linkpp;
+
+ modinfop->mi_rev = modlp->ml_rev;
+
+ linkpp = (struct modlmisc **)modlp->ml_linkage;
+ msip = &modinfop->mi_msinfo[0];
+
+ for (i = 0; i < MODMAXLINK; i++) {
+ if (*linkpp == NULL) {
+ msip->msi_linkinfo[0] = '\0';
+ } else {
+ (void) strncpy(msip->msi_linkinfo,
+ (*linkpp)->misc_linkinfo, MODMAXLINKINFOLEN);
+ retval = MODL_INFO(*linkpp, modlp, &msip->msi_p0);
+ if (retval != 0)
+ break;
+ linkpp++;
+ }
+ msip++;
+ }
+
+ if (modinfop->mi_info == MI_INFO_LINKAGE) {
+ /*
+ * Slight kludge used to extract the address of the
+ * modlinkage structure from the module (just after
+ * loading a module for the very first time)
+ */
+ modinfop->mi_base = (void *)modlp;
+ }
+
+ if (retval == 0)
+ return (1);
+ return (0);
+}
\ No newline at end of file
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * mod_hash: flexible hash table implementation.
+ *
+ * This is a reasonably fast, reasonably flexible hash table implementation
+ * which features pluggable hash algorithms to support storing arbitrary keys
+ * and values. It is designed to handle small (< 100,000 items) amounts of
+ * data. The hash uses chaining to resolve collisions, and does not feature a
+ * mechanism to grow the hash. Care must be taken to pick nchains to be large
+ * enough for the application at hand, or lots of time will be wasted searching
+ * hash chains.
+ *
+ * The client of the hash is required to supply a number of items to support
+ * the various hash functions:
+ *
+ * - Destructor functions for the key and value being hashed.
+ * A destructor is responsible for freeing an object when the hash
+ * table is no longer storing it. Since keys and values can be of
+ * arbitrary type, separate destructors for keys & values are used.
+ * These may be mod_hash_null_keydtor and mod_hash_null_valdtor if no
+ * destructor is needed for either a key or value.
+ *
+ * - A hashing algorithm which returns a uint_t representing a hash index
+ * The number returned need _not_ be between 0 and nchains. The mod_hash
+ * code will take care of doing that. The second argument (after the
+ * key) to the hashing function is a void * that represents
+ * hash_alg_data-- this is provided so that the hashing algrorithm can
+ * maintain some state across calls, or keep algorithm-specific
+ * constants associated with the hash table.
+ *
+ * A pointer-hashing and a string-hashing algorithm are supplied in
+ * this file.
+ *
+ * - A key comparator (a la qsort).
+ * This is used when searching the hash chain. The key comparator
+ * determines if two keys match. It should follow the return value
+ * semantics of strcmp.
+ *
+ * string and pointer comparators are supplied in this file.
+ *
+ * mod_hash_create_strhash() and mod_hash_create_ptrhash() provide good
+ * examples of how to create a customized hash table.
+ *
+ * Basic hash operations:
+ *
+ * mod_hash_create_strhash(name, nchains, dtor),
+ * create a hash using strings as keys.
+ * NOTE: This create a hash which automatically cleans up the string
+ * values it is given for keys.
+ *
+ * mod_hash_create_ptrhash(name, nchains, dtor, key_elem_size):
+ * create a hash using pointers as keys.
+ *
+ * mod_hash_create_extended(name, nchains, kdtor, vdtor,
+ * hash_alg, hash_alg_data,
+ * keycmp, sleep)
+ * create a customized hash table.
+ *
+ * mod_hash_destroy_hash(hash):
+ * destroy the given hash table, calling the key and value destructors
+ * on each key-value pair stored in the hash.
+ *
+ * mod_hash_insert(hash, key, val):
+ * place a key, value pair into the given hash.
+ * duplicate keys are rejected.
+ *
+ * mod_hash_insert_reserve(hash, key, val, handle):
+ * place a key, value pair into the given hash, using handle to indicate
+ * the reserved storage for the pair. (no memory allocation is needed
+ * during a mod_hash_insert_reserve.) duplicate keys are rejected.
+ *
+ * mod_hash_reserve(hash, *handle):
+ * reserve storage for a key-value pair using the memory allocation
+ * policy of 'hash', returning the storage handle in 'handle'.
+ *
+ * mod_hash_reserve_nosleep(hash, *handle): reserve storage for a key-value
+ * pair ignoring the memory allocation policy of 'hash' and always without
+ * sleep, returning the storage handle in 'handle'.
+ *
+ * mod_hash_remove(hash, key, *val):
+ * remove a key-value pair with key 'key' from 'hash', destroying the
+ * stored key, and returning the value in val.
+ *
+ * mod_hash_replace(hash, key, val)
+ * atomically remove an existing key-value pair from a hash, and replace
+ * the key and value with the ones supplied. The removed key and value
+ * (if any) are destroyed.
+ *
+ * mod_hash_destroy(hash, key):
+ * remove a key-value pair with key 'key' from 'hash', destroying both
+ * stored key and stored value.
+ *
+ * mod_hash_find(hash, key, val):
+ * find a value in the hash table corresponding to the given key.
+ *
+ * mod_hash_find_cb(hash, key, val, found_callback)
+ * find a value in the hash table corresponding to the given key.
+ * If a value is found, call specified callback passing key and val to it.
+ * The callback is called with the hash lock held.
+ * It is intended to be used in situations where the act of locating the
+ * data must also modify it - such as in reference counting schemes.
+ *
+ * mod_hash_walk(hash, callback(key, elem, arg), arg)
+ * walks all the elements in the hashtable and invokes the callback
+ * function with the key/value pair for each element. the hashtable
+ * is locked for readers so the callback function should not attempt
+ * to do any updates to the hashable. the callback function should
+ * return MH_WALK_CONTINUE to continue walking the hashtable or
+ * MH_WALK_TERMINATE to abort the walk of the hashtable.
+ *
+ * mod_hash_clear(hash):
+ * clears the given hash table of entries, calling the key and value
+ * destructors for every element in the hash.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/bitmap.h>
+#include <sys/modhash_impl.h>
+#include <sys/sysmacros.h>
+
+/*
+ * MH_KEY_DESTROY()
+ * Invoke the key destructor.
+ */
+#define MH_KEY_DESTROY(hash, key) ((hash->mh_kdtor)(key))
+
+/*
+ * MH_VAL_DESTROY()
+ * Invoke the value destructor.
+ */
+#define MH_VAL_DESTROY(hash, val) ((hash->mh_vdtor)(val))
+
+/*
+ * MH_KEYCMP()
+ * Call the key comparator for the given hash keys.
+ */
+#define MH_KEYCMP(hash, key1, key2) ((hash->mh_keycmp)(key1, key2))
+
+/*
+ * Cache for struct mod_hash_entry
+ */
+kmem_cache_t *mh_e_cache = NULL;
+mod_hash_t *mh_head = NULL;
+kmutex_t mh_head_lock;
+
+/*
+ * mod_hash_null_keydtor()
+ * mod_hash_null_valdtor()
+ * no-op key and value destructors.
+ */
+/*ARGSUSED*/
+void
+mod_hash_null_keydtor(mod_hash_key_t key)
+{
+}
+
+/*ARGSUSED*/
+void
+mod_hash_null_valdtor(mod_hash_val_t val)
+{
+}
+
+/*
+ * mod_hash_bystr()
+ * mod_hash_strkey_cmp()
+ * mod_hash_strkey_dtor()
+ * mod_hash_strval_dtor()
+ * Hash and key comparison routines for hashes with string keys.
+ *
+ * mod_hash_create_strhash()
+ * Create a hash using strings as keys
+ *
+ * The string hashing algorithm is from the "Dragon Book" --
+ * "Compilers: Principles, Tools & Techniques", by Aho, Sethi, Ullman
+ */
+
+/*ARGSUSED*/
+uint_t
+mod_hash_bystr(void *hash_data, mod_hash_key_t key)
+{
+ uint_t hash = 0;
+ uint_t g;
+ char *p, *k = (char *)key;
+
+ ASSERT(k);
+ for (p = k; *p != '\0'; p++) {
+ hash = (hash << 4) + *p;
+ if ((g = (hash & 0xf0000000)) != 0) {
+ hash ^= (g >> 24);
+ hash ^= g;
+ }
+ }
+ return (hash);
+}
+
+int
+mod_hash_strkey_cmp(mod_hash_key_t key1, mod_hash_key_t key2)
+{
+ return (strcmp((char *)key1, (char *)key2));
+}
+
+void
+mod_hash_strkey_dtor(mod_hash_key_t key)
+{
+ char *c = (char *)key;
+ kmem_free(c, strlen(c) + 1);
+}
+
+void
+mod_hash_strval_dtor(mod_hash_val_t val)
+{
+ char *c = (char *)val;
+ kmem_free(c, strlen(c) + 1);
+}
+
+mod_hash_t *
+mod_hash_create_strhash_nodtr(char *name, size_t nchains,
+ void (*val_dtor)(mod_hash_val_t))
+{
+ return mod_hash_create_extended(name, nchains, mod_hash_null_keydtor,
+ val_dtor, mod_hash_bystr, NULL, mod_hash_strkey_cmp, KM_SLEEP);
+}
+
+mod_hash_t *
+mod_hash_create_strhash(char *name, size_t nchains,
+ void (*val_dtor)(mod_hash_val_t))
+{
+ return mod_hash_create_extended(name, nchains, mod_hash_strkey_dtor,
+ val_dtor, mod_hash_bystr, NULL, mod_hash_strkey_cmp, KM_SLEEP);
+}
+
+void
+mod_hash_destroy_strhash(mod_hash_t *strhash)
+{
+ ASSERT(strhash);
+ mod_hash_destroy_hash(strhash);
+}
+
+
+/*
+ * mod_hash_byptr()
+ * mod_hash_ptrkey_cmp()
+ * Hash and key comparison routines for hashes with pointer keys.
+ *
+ * mod_hash_create_ptrhash()
+ * mod_hash_destroy_ptrhash()
+ * Create a hash that uses pointers as keys. This hash algorithm
+ * picks an appropriate set of middle bits in the address to hash on
+ * based on the size of the hash table and a hint about the size of
+ * the items pointed at.
+ */
+uint_t
+mod_hash_byptr(void *hash_data, mod_hash_key_t key)
+{
+ uintptr_t k = (uintptr_t)key;
+ k >>= (int)(uintptr_t)hash_data;
+
+ return ((uint_t)k);
+}
+
+int
+mod_hash_ptrkey_cmp(mod_hash_key_t key1, mod_hash_key_t key2)
+{
+ uintptr_t k1 = (uintptr_t)key1;
+ uintptr_t k2 = (uintptr_t)key2;
+ if (k1 > k2)
+ return (-1);
+ else if (k1 < k2)
+ return (1);
+ else
+ return (0);
+}
+
+mod_hash_t *
+mod_hash_create_ptrhash(char *name, size_t nchains,
+ void (*val_dtor)(mod_hash_val_t), size_t key_elem_size)
+{
+ size_t rshift;
+
+ /*
+ * We want to hash on the bits in the middle of the address word
+ * Bits far to the right in the word have little significance, and
+ * are likely to all look the same (for example, an array of
+ * 256-byte structures will have the bottom 8 bits of address
+ * words the same). So we want to right-shift each address to
+ * ignore the bottom bits.
+ *
+ * The high bits, which are also unused, will get taken out when
+ * mod_hash takes hashkey % nchains.
+ */
+ rshift = highbit(key_elem_size);
+
+ return mod_hash_create_extended(name, nchains, mod_hash_null_keydtor,
+ val_dtor, mod_hash_byptr, (void *)rshift, mod_hash_ptrkey_cmp,
+ KM_SLEEP);
+}
+
+void
+mod_hash_destroy_ptrhash(mod_hash_t *hash)
+{
+ ASSERT(hash);
+ mod_hash_destroy_hash(hash);
+}
+
+/*
+ * mod_hash_byid()
+ * mod_hash_idkey_cmp()
+ * Hash and key comparison routines for hashes with 32-bit unsigned keys.
+ *
+ * mod_hash_create_idhash()
+ * mod_hash_destroy_idhash()
+ * mod_hash_iddata_gen()
+ * Create a hash that uses numeric keys.
+ *
+ * The hash algorithm is documented in "Introduction to Algorithms"
+ * (Cormen, Leiserson, Rivest); when the hash table is created, it
+ * attempts to find the next largest prime above the number of hash
+ * slots. The hash index is then this number times the key modulo
+ * the hash size, or (key * prime) % nchains.
+ */
+uint_t
+mod_hash_byid(void *hash_data, mod_hash_key_t key)
+{
+ uint_t kval = (uint_t)(uintptr_t)hash_data;
+ return ((uint_t)(uintptr_t)key * (uint_t)kval);
+}
+
+int
+mod_hash_idkey_cmp(mod_hash_key_t key1, mod_hash_key_t key2)
+{
+ return ((uint_t)(uintptr_t)key1 - (uint_t)(uintptr_t)key2);
+}
+
+/*
+ * Generate the next largest prime number greater than nchains; this value
+ * is intended to be later passed in to mod_hash_create_extended() as the
+ * hash_data.
+ */
+uint_t
+mod_hash_iddata_gen(size_t nchains)
+{
+ uint_t kval, i, prime;
+
+ /*
+ * Pick the first (odd) prime greater than nchains. Make sure kval is
+ * odd (so start with nchains +1 or +2 as appropriate).
+ */
+ kval = (nchains % 2 == 0) ? nchains + 1 : nchains + 2;
+
+ for (;;) {
+ prime = 1;
+ for (i = 3; i * i <= kval; i += 2) {
+ if (kval % i == 0)
+ prime = 0;
+ }
+ if (prime == 1)
+ break;
+ kval += 2;
+ }
+ return (kval);
+}
+
+mod_hash_t *
+mod_hash_create_idhash(char *name, size_t nchains,
+ void (*val_dtor)(mod_hash_val_t))
+{
+ uint_t kval = mod_hash_iddata_gen(nchains);
+
+ return (mod_hash_create_extended(name, nchains, mod_hash_null_keydtor,
+ val_dtor, mod_hash_byid, (void *)(uintptr_t)kval,
+ mod_hash_idkey_cmp, KM_SLEEP));
+}
+
+void
+mod_hash_destroy_idhash(mod_hash_t *hash)
+{
+ ASSERT(hash);
+ mod_hash_destroy_hash(hash);
+}
+
+void
+mod_hash_fini(void)
+{
+ mutex_destroy(&mh_head_lock);
+
+ if (mh_e_cache) {
+ kmem_cache_destroy(mh_e_cache);
+ mh_e_cache = NULL;
+ }
+}
+
+/*
+ * mod_hash_init()
+ * sets up globals, etc for mod_hash_*
+ */
+void
+mod_hash_init(void)
+{
+ ASSERT(mh_e_cache == NULL);
+ mh_e_cache = kmem_cache_create("mod_hash_entries",
+ sizeof (struct mod_hash_entry), 0, NULL, NULL, NULL, NULL,
+ NULL, 0);
+
+ mutex_init(&mh_head_lock, NULL, MUTEX_DEFAULT, NULL);
+}
+
+/*
+ * mod_hash_create_extended()
+ * The full-blown hash creation function.
+ *
+ * notes:
+ * nchains - how many hash slots to create. More hash slots will
+ * result in shorter hash chains, but will consume
+ * slightly more memory up front.
+ * sleep - should be KM_SLEEP or KM_NOSLEEP, to indicate whether
+ * to sleep for memory, or fail in low-memory conditions.
+ *
+ * Fails only if KM_NOSLEEP was specified, and no memory was available.
+ */
+mod_hash_t *
+mod_hash_create_extended(
+ char *hname, /* descriptive name for hash */
+ size_t nchains, /* number of hash slots */
+ void (*kdtor)(mod_hash_key_t), /* key destructor */
+ void (*vdtor)(mod_hash_val_t), /* value destructor */
+ uint_t (*hash_alg)(void *, mod_hash_key_t), /* hash algorithm */
+ void *hash_alg_data, /* pass-thru arg for hash_alg */
+ int (*keycmp)(mod_hash_key_t, mod_hash_key_t), /* key comparator */
+ int sleep) /* whether to sleep for mem */
+{
+ mod_hash_t *mod_hash;
+ ASSERT(hname && keycmp && hash_alg && vdtor && kdtor);
+
+ if ((mod_hash = kmem_zalloc(MH_SIZE(nchains), sleep)) == NULL)
+ return (NULL);
+
+ mod_hash->mh_name = kmem_alloc(strlen(hname) + 1, sleep);
+ if (mod_hash->mh_name == NULL) {
+ kmem_free(mod_hash, MH_SIZE(nchains));
+ return (NULL);
+ }
+ (void) strcpy(mod_hash->mh_name, hname);
+
+ rw_init(&mod_hash->mh_contents, NULL, RW_DEFAULT, NULL);
+ mod_hash->mh_sleep = sleep;
+ mod_hash->mh_nchains = nchains;
+ mod_hash->mh_kdtor = kdtor;
+ mod_hash->mh_vdtor = vdtor;
+ mod_hash->mh_hashalg = hash_alg;
+ mod_hash->mh_hashalg_data = hash_alg_data;
+ mod_hash->mh_keycmp = keycmp;
+
+ /*
+ * Link the hash up on the list of hashes
+ */
+ mutex_enter(&mh_head_lock);
+ mod_hash->mh_next = mh_head;
+ mh_head = mod_hash;
+ mutex_exit(&mh_head_lock);
+
+ return (mod_hash);
+}
+
+/*
+ * mod_hash_destroy_hash()
+ * destroy a hash table, destroying all of its stored keys and values
+ * as well.
+ */
+void
+mod_hash_destroy_hash(mod_hash_t *hash)
+{
+ mod_hash_t *mhp, *mhpp;
+
+ mutex_enter(&mh_head_lock);
+ /*
+ * Remove the hash from the hash list
+ */
+ if (hash == mh_head) { /* removing 1st list elem */
+ mh_head = mh_head->mh_next;
+ } else {
+ /*
+ * mhpp can start out NULL since we know the 1st elem isn't the
+ * droid we're looking for.
+ */
+ mhpp = NULL;
+ for (mhp = mh_head; mhp != NULL; mhp = mhp->mh_next) {
+ if (mhp == hash) {
+ mhpp->mh_next = mhp->mh_next;
+ break;
+ }
+ mhpp = mhp;
+ }
+ }
+ mutex_exit(&mh_head_lock);
+
+ /*
+ * Clean out keys and values.
+ */
+ mod_hash_clear(hash);
+
+ rw_destroy(&hash->mh_contents);
+ kmem_free(hash->mh_name, strlen(hash->mh_name) + 1);
+ kmem_free(hash, MH_SIZE(hash->mh_nchains));
+}
+
+/*
+ * i_mod_hash()
+ * Call the hashing algorithm for this hash table, with the given key.
+ */
+uint_t
+i_mod_hash(mod_hash_t *hash, mod_hash_key_t key)
+{
+ uint_t h;
+ /*
+ * Prevent div by 0 problems;
+ * Also a nice shortcut when using a hash as a list
+ */
+ if (hash->mh_nchains == 1)
+ return (0);
+
+ h = (hash->mh_hashalg)(hash->mh_hashalg_data, key);
+ return (h % (hash->mh_nchains - 1));
+}
+
+/*
+ * i_mod_hash_insert_nosync()
+ * mod_hash_insert()
+ * mod_hash_insert_reserve()
+ * insert 'val' into the hash table, using 'key' as its key. If 'key' is
+ * already a key in the hash, an error will be returned, and the key-val
+ * pair will not be inserted. i_mod_hash_insert_nosync() supports a simple
+ * handle abstraction, allowing hash entry allocation to be separated from
+ * the hash insertion. this abstraction allows simple use of the mod_hash
+ * structure in situations where mod_hash_insert() with a KM_SLEEP
+ * allocation policy would otherwise be unsafe.
+ */
+int
+i_mod_hash_insert_nosync(mod_hash_t *hash, mod_hash_key_t key,
+ mod_hash_val_t val, mod_hash_hndl_t handle)
+{
+ uint_t hashidx;
+ struct mod_hash_entry *entry;
+
+ ASSERT(hash);
+
+ /*
+ * If we've not been given reserved storage, allocate storage directly,
+ * using the hash's allocation policy.
+ */
+ if (handle == (mod_hash_hndl_t)0) {
+ entry = kmem_cache_alloc(mh_e_cache, hash->mh_sleep);
+ if (entry == NULL) {
+ hash->mh_stat.mhs_nomem++;
+ return (MH_ERR_NOMEM);
+ }
+ } else {
+ entry = (struct mod_hash_entry *)handle;
+ }
+
+ hashidx = i_mod_hash(hash, key);
+ entry->mhe_key = key;
+ entry->mhe_val = val;
+ entry->mhe_next = hash->mh_entries[hashidx];
+
+ hash->mh_entries[hashidx] = entry;
+ hash->mh_stat.mhs_nelems++;
+
+ return (0);
+}
+
+int
+mod_hash_insert(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t val)
+{
+ int res;
+ mod_hash_val_t v;
+
+ rw_enter(&hash->mh_contents, RW_WRITER);
+
+ /*
+ * Disallow duplicate keys in the hash
+ */
+ if (i_mod_hash_find_nosync(hash, key, &v) == 0) {
+ rw_exit(&hash->mh_contents);
+ hash->mh_stat.mhs_coll++;
+ return (MH_ERR_DUPLICATE);
+ }
+
+ res = i_mod_hash_insert_nosync(hash, key, val, (mod_hash_hndl_t)0);
+ rw_exit(&hash->mh_contents);
+
+ return (res);
+}
+
+int
+mod_hash_insert_reserve(mod_hash_t *hash, mod_hash_key_t key,
+ mod_hash_val_t val, mod_hash_hndl_t handle)
+{
+ int res;
+ mod_hash_val_t v;
+
+ rw_enter(&hash->mh_contents, RW_WRITER);
+
+ /*
+ * Disallow duplicate keys in the hash
+ */
+ if (i_mod_hash_find_nosync(hash, key, &v) == 0) {
+ rw_exit(&hash->mh_contents);
+ hash->mh_stat.mhs_coll++;
+ return (MH_ERR_DUPLICATE);
+ }
+ res = i_mod_hash_insert_nosync(hash, key, val, handle);
+ rw_exit(&hash->mh_contents);
+
+ return (res);
+}
+
+/*
+ * mod_hash_reserve()
+ * mod_hash_reserve_nosleep()
+ * mod_hash_cancel()
+ * Make or cancel a mod_hash_entry_t reservation. Reservations are used in
+ * mod_hash_insert_reserve() above.
+ */
+int
+mod_hash_reserve(mod_hash_t *hash, mod_hash_hndl_t *handlep)
+{
+ *handlep = kmem_cache_alloc(mh_e_cache, hash->mh_sleep);
+ if (*handlep == NULL) {
+ hash->mh_stat.mhs_nomem++;
+ return (MH_ERR_NOMEM);
+ }
+
+ return (0);
+}
+
+int
+mod_hash_reserve_nosleep(mod_hash_t *hash, mod_hash_hndl_t *handlep)
+{
+ *handlep = kmem_cache_alloc(mh_e_cache, KM_NOSLEEP);
+ if (*handlep == NULL) {
+ hash->mh_stat.mhs_nomem++;
+ return (MH_ERR_NOMEM);
+ }
+
+ return (0);
+
+}
+
+/*ARGSUSED*/
+void
+mod_hash_cancel(mod_hash_t *hash, mod_hash_hndl_t *handlep)
+{
+ kmem_cache_free(mh_e_cache, *handlep);
+ *handlep = (mod_hash_hndl_t)0;
+}
+
+/*
+ * i_mod_hash_remove_nosync()
+ * mod_hash_remove()
+ * Remove an element from the hash table.
+ */
+int
+i_mod_hash_remove_nosync(mod_hash_t *hash, mod_hash_key_t key,
+ mod_hash_val_t *val)
+{
+ int hashidx;
+ struct mod_hash_entry *e, *ep;
+
+ hashidx = i_mod_hash(hash, key);
+ ep = NULL; /* e's parent */
+
+ for (e = hash->mh_entries[hashidx]; e != NULL; e = e->mhe_next) {
+ if (MH_KEYCMP(hash, e->mhe_key, key) == 0)
+ break;
+ ep = e;
+ }
+
+ if (e == NULL) { /* not found */
+ return (MH_ERR_NOTFOUND);
+ }
+
+ if (ep == NULL) /* special case 1st element in bucket */
+ hash->mh_entries[hashidx] = e->mhe_next;
+ else
+ ep->mhe_next = e->mhe_next;
+
+ /*
+ * Clean up resources used by the node's key.
+ */
+ MH_KEY_DESTROY(hash, e->mhe_key);
+
+ *val = e->mhe_val;
+ kmem_cache_free(mh_e_cache, e);
+ hash->mh_stat.mhs_nelems--;
+
+ return (0);
+}
+
+int
+mod_hash_remove(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t *val)
+{
+ int res;
+
+ rw_enter(&hash->mh_contents, RW_WRITER);
+ res = i_mod_hash_remove_nosync(hash, key, val);
+ rw_exit(&hash->mh_contents);
+
+ return (res);
+}
+
+/*
+ * mod_hash_replace()
+ * atomically remove an existing key-value pair from a hash, and replace
+ * the key and value with the ones supplied. The removed key and value
+ * (if any) are destroyed.
+ */
+int
+mod_hash_replace(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t val)
+{
+ int res;
+ mod_hash_val_t v;
+
+ rw_enter(&hash->mh_contents, RW_WRITER);
+
+ if (i_mod_hash_remove_nosync(hash, key, &v) == 0) {
+ /*
+ * mod_hash_remove() takes care of freeing up the key resources.
+ */
+ MH_VAL_DESTROY(hash, v);
+ }
+ res = i_mod_hash_insert_nosync(hash, key, val, (mod_hash_hndl_t)0);
+
+ rw_exit(&hash->mh_contents);
+
+ return (res);
+}
+
+/*
+ * mod_hash_destroy()
+ * Remove an element from the hash table matching 'key', and destroy it.
+ */
+int
+mod_hash_destroy(mod_hash_t *hash, mod_hash_key_t key)
+{
+ mod_hash_val_t val;
+ int rv;
+
+ rw_enter(&hash->mh_contents, RW_WRITER);
+
+ if ((rv = i_mod_hash_remove_nosync(hash, key, &val)) == 0) {
+ /*
+ * mod_hash_remove() takes care of freeing up the key resources.
+ */
+ MH_VAL_DESTROY(hash, val);
+ }
+
+ rw_exit(&hash->mh_contents);
+ return (rv);
+}
+
+/*
+ * i_mod_hash_find_nosync()
+ * mod_hash_find()
+ * Find a value in the hash table corresponding to the given key.
+ */
+int
+i_mod_hash_find_nosync(mod_hash_t *hash, mod_hash_key_t key,
+ mod_hash_val_t *val)
+{
+ uint_t hashidx;
+ struct mod_hash_entry *e;
+
+ hashidx = i_mod_hash(hash, key);
+
+ for (e = hash->mh_entries[hashidx]; e != NULL; e = e->mhe_next) {
+ if (MH_KEYCMP(hash, e->mhe_key, key) == 0) {
+ *val = e->mhe_val;
+ hash->mh_stat.mhs_hit++;
+ return (0);
+ }
+ }
+ hash->mh_stat.mhs_miss++;
+ return (MH_ERR_NOTFOUND);
+}
+
+int
+mod_hash_find(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t *val)
+{
+ int res;
+
+ rw_enter(&hash->mh_contents, RW_READER);
+ res = i_mod_hash_find_nosync(hash, key, val);
+ rw_exit(&hash->mh_contents);
+
+ return (res);
+}
+
+int
+mod_hash_find_cb(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t *val,
+ void (*find_cb)(mod_hash_key_t, mod_hash_val_t))
+{
+ int res;
+
+ rw_enter(&hash->mh_contents, RW_READER);
+ res = i_mod_hash_find_nosync(hash, key, val);
+ if (res == 0) {
+ find_cb(key, *val);
+ }
+ rw_exit(&hash->mh_contents);
+
+ return (res);
+}
+
+int
+mod_hash_find_cb_rval(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t *val,
+ int (*find_cb)(mod_hash_key_t, mod_hash_val_t), int *cb_rval)
+{
+ int res;
+
+ rw_enter(&hash->mh_contents, RW_READER);
+ res = i_mod_hash_find_nosync(hash, key, val);
+ if (res == 0) {
+ *cb_rval = find_cb(key, *val);
+ }
+ rw_exit(&hash->mh_contents);
+
+ return (res);
+}
+
+void
+i_mod_hash_walk_nosync(mod_hash_t *hash,
+ uint_t (*callback)(mod_hash_key_t, mod_hash_val_t *, void *), void *arg)
+{
+ struct mod_hash_entry *e;
+ uint_t hashidx;
+ int res = MH_WALK_CONTINUE;
+
+ for (hashidx = 0;
+ (hashidx < (hash->mh_nchains - 1)) && (res == MH_WALK_CONTINUE);
+ hashidx++) {
+ e = hash->mh_entries[hashidx];
+ while ((e != NULL) && (res == MH_WALK_CONTINUE)) {
+ res = callback(e->mhe_key, e->mhe_val, arg);
+ e = e->mhe_next;
+ }
+ }
+}
+
+/*
+ * mod_hash_walk()
+ * Walks all the elements in the hashtable and invokes the callback
+ * function with the key/value pair for each element. The hashtable
+ * is locked for readers so the callback function should not attempt
+ * to do any updates to the hashable. The callback function should
+ * return MH_WALK_CONTINUE to continue walking the hashtable or
+ * MH_WALK_TERMINATE to abort the walk of the hashtable.
+ */
+void
+mod_hash_walk(mod_hash_t *hash,
+ uint_t (*callback)(mod_hash_key_t, mod_hash_val_t *, void *), void *arg)
+{
+ rw_enter(&hash->mh_contents, RW_READER);
+ i_mod_hash_walk_nosync(hash, callback, arg);
+ rw_exit(&hash->mh_contents);
+}
+
+
+/*
+ * i_mod_hash_clear_nosync()
+ * mod_hash_clear()
+ * Clears the given hash table by calling the destructor of every hash
+ * element and freeing up all mod_hash_entry's.
+ */
+void
+i_mod_hash_clear_nosync(mod_hash_t *hash)
+{
+ int i;
+ struct mod_hash_entry *e, *old_e;
+
+ for (i = 0; i < hash->mh_nchains; i++) {
+ e = hash->mh_entries[i];
+ while (e != NULL) {
+ MH_KEY_DESTROY(hash, e->mhe_key);
+ MH_VAL_DESTROY(hash, e->mhe_val);
+ old_e = e;
+ e = e->mhe_next;
+ kmem_cache_free(mh_e_cache, old_e);
+ }
+ hash->mh_entries[i] = NULL;
+ }
+ hash->mh_stat.mhs_nelems = 0;
+}
+
+void
+mod_hash_clear(mod_hash_t *hash)
+{
+ ASSERT(hash);
+ rw_enter(&hash->mh_contents, RW_WRITER);
+ i_mod_hash_clear_nosync(hash);
+ rw_exit(&hash->mh_contents);
+}
--- /dev/null
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * This file is part of the core Kernel Cryptographic Framework.
+ * It implements the SPI functions exported to cryptographic
+ * providers.
+ */
+
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/crypto/sched_impl.h>
+#include <sys/crypto/spi.h>
+
+/*
+ * minalloc and maxalloc values to be used for taskq_create().
+ */
+int crypto_taskq_threads = CRYPTO_TASKQ_THREADS;
+int crypto_taskq_minalloc = CYRPTO_TASKQ_MIN;
+int crypto_taskq_maxalloc = CRYPTO_TASKQ_MAX;
+
+static void remove_provider(kcf_provider_desc_t *);
+static void process_logical_providers(crypto_provider_info_t *,
+ kcf_provider_desc_t *);
+static int init_prov_mechs(crypto_provider_info_t *, kcf_provider_desc_t *);
+static int kcf_prov_kstat_update(kstat_t *, int);
+static void delete_kstat(kcf_provider_desc_t *);
+
+static kcf_prov_stats_t kcf_stats_ks_data_template = {
+ { "kcf_ops_total", KSTAT_DATA_UINT64 },
+ { "kcf_ops_passed", KSTAT_DATA_UINT64 },
+ { "kcf_ops_failed", KSTAT_DATA_UINT64 },
+ { "kcf_ops_returned_busy", KSTAT_DATA_UINT64 }
+};
+
+#define KCF_SPI_COPY_OPS(src, dst, ops) if ((src)->ops != NULL) \
+ *((dst)->ops) = *((src)->ops);
+
+/*
+ * Copy an ops vector from src to dst. Used during provider registration
+ * to copy the ops vector from the provider info structure to the
+ * provider descriptor maintained by KCF.
+ * Copying the ops vector specified by the provider is needed since the
+ * framework does not require the provider info structure to be
+ * persistent.
+ */
+static void
+copy_ops_vector_v1(crypto_ops_t *src_ops, crypto_ops_t *dst_ops)
+{
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_control_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_digest_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_cipher_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_mac_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_sign_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_verify_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_dual_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_dual_cipher_mac_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_random_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_session_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_object_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_key_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_provider_ops);
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_ctx_ops);
+}
+
+static void
+copy_ops_vector_v2(crypto_ops_t *src_ops, crypto_ops_t *dst_ops)
+{
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_mech_ops);
+}
+
+static void
+copy_ops_vector_v3(crypto_ops_t *src_ops, crypto_ops_t *dst_ops)
+{
+ KCF_SPI_COPY_OPS(src_ops, dst_ops, co_nostore_key_ops);
+}
+
+/*
+ * This routine is used to add cryptographic providers to the KEF framework.
+ * Providers pass a crypto_provider_info structure to crypto_register_provider()
+ * and get back a handle. The crypto_provider_info structure contains a
+ * list of mechanisms supported by the provider and an ops vector containing
+ * provider entry points. Hardware providers call this routine in their attach
+ * routines. Software providers call this routine in their _init() routine.
+ */
+int
+crypto_register_provider(crypto_provider_info_t *info,
+ crypto_kcf_provider_handle_t *handle)
+{
+ char ks_name[KSTAT_STRLEN];
+
+ kcf_provider_desc_t *prov_desc = NULL;
+ int ret = CRYPTO_ARGUMENTS_BAD;
+
+ if (info->pi_interface_version > CRYPTO_SPI_VERSION_3)
+ return (CRYPTO_VERSION_MISMATCH);
+
+ /*
+ * Check provider type, must be software, hardware, or logical.
+ */
+ if (info->pi_provider_type != CRYPTO_HW_PROVIDER &&
+ info->pi_provider_type != CRYPTO_SW_PROVIDER &&
+ info->pi_provider_type != CRYPTO_LOGICAL_PROVIDER)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ /*
+ * Allocate and initialize a new provider descriptor. We also
+ * hold it and release it when done.
+ */
+ prov_desc = kcf_alloc_provider_desc(info);
+ KCF_PROV_REFHOLD(prov_desc);
+
+ prov_desc->pd_prov_type = info->pi_provider_type;
+
+ /* provider-private handle, opaque to KCF */
+ prov_desc->pd_prov_handle = info->pi_provider_handle;
+
+ /* copy provider description string */
+ if (info->pi_provider_description != NULL) {
+ /*
+ * pi_provider_descriptor is a string that can contain
+ * up to CRYPTO_PROVIDER_DESCR_MAX_LEN + 1 characters
+ * INCLUDING the terminating null character. A bcopy()
+ * is necessary here as pd_description should not have
+ * a null character. See comments in kcf_alloc_provider_desc()
+ * for details on pd_description field.
+ */
+ bcopy(info->pi_provider_description, prov_desc->pd_description,
+ MIN(strlen(info->pi_provider_description),
+ (size_t)CRYPTO_PROVIDER_DESCR_MAX_LEN));
+ }
+
+ if (info->pi_provider_type != CRYPTO_LOGICAL_PROVIDER) {
+ if (info->pi_ops_vector == NULL) {
+ goto bail;
+ }
+ copy_ops_vector_v1(info->pi_ops_vector,
+ prov_desc->pd_ops_vector);
+ if (info->pi_interface_version >= CRYPTO_SPI_VERSION_2) {
+ copy_ops_vector_v2(info->pi_ops_vector,
+ prov_desc->pd_ops_vector);
+ prov_desc->pd_flags = info->pi_flags;
+ }
+ if (info->pi_interface_version == CRYPTO_SPI_VERSION_3) {
+ copy_ops_vector_v3(info->pi_ops_vector,
+ prov_desc->pd_ops_vector);
+ }
+ }
+
+ /* object_ops and nostore_key_ops are mutually exclusive */
+ if (prov_desc->pd_ops_vector->co_object_ops &&
+ prov_desc->pd_ops_vector->co_nostore_key_ops) {
+ goto bail;
+ }
+
+ /* process the mechanisms supported by the provider */
+ if ((ret = init_prov_mechs(info, prov_desc)) != CRYPTO_SUCCESS)
+ goto bail;
+
+ /*
+ * Add provider to providers tables, also sets the descriptor
+ * pd_prov_id field.
+ */
+ if ((ret = kcf_prov_tab_add_provider(prov_desc)) != CRYPTO_SUCCESS) {
+ undo_register_provider(prov_desc, B_FALSE);
+ goto bail;
+ }
+
+ /*
+ * We create a taskq only for a hardware provider. The global
+ * software queue is used for software providers. We handle ordering
+ * of multi-part requests in the taskq routine. So, it is safe to
+ * have multiple threads for the taskq. We pass TASKQ_PREPOPULATE flag
+ * to keep some entries cached to improve performance.
+ */
+ if (prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER)
+ prov_desc->pd_sched_info.ks_taskq = taskq_create("kcf_taskq",
+ crypto_taskq_threads, minclsyspri,
+ crypto_taskq_minalloc, crypto_taskq_maxalloc,
+ TASKQ_PREPOPULATE);
+ else
+ prov_desc->pd_sched_info.ks_taskq = NULL;
+
+ /* no kernel session to logical providers */
+ if (prov_desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER) {
+ /*
+ * Open a session for session-oriented providers. This session
+ * is used for all kernel consumers. This is fine as a provider
+ * is required to support multiple thread access to a session.
+ * We can do this only after the taskq has been created as we
+ * do a kcf_submit_request() to open the session.
+ */
+ if (KCF_PROV_SESSION_OPS(prov_desc) != NULL) {
+ kcf_req_params_t params;
+
+ KCF_WRAP_SESSION_OPS_PARAMS(¶ms,
+ KCF_OP_SESSION_OPEN, &prov_desc->pd_sid, 0,
+ CRYPTO_USER, NULL, 0, prov_desc);
+ ret = kcf_submit_request(prov_desc, NULL, NULL, ¶ms,
+ B_FALSE);
+
+ if (ret != CRYPTO_SUCCESS) {
+ undo_register_provider(prov_desc, B_TRUE);
+ ret = CRYPTO_FAILED;
+ goto bail;
+ }
+ }
+ }
+
+ if (prov_desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER) {
+ /*
+ * Create the kstat for this provider. There is a kstat
+ * installed for each successfully registered provider.
+ * This kstat is deleted, when the provider unregisters.
+ */
+ if (prov_desc->pd_prov_type == CRYPTO_SW_PROVIDER) {
+ (void) snprintf(ks_name, KSTAT_STRLEN, "%s_%s",
+ "NONAME", "provider_stats");
+ } else {
+ (void) snprintf(ks_name, KSTAT_STRLEN, "%s_%d_%u_%s",
+ "NONAME", 0,
+ prov_desc->pd_prov_id, "provider_stats");
+ }
+
+ prov_desc->pd_kstat = kstat_create("kcf", 0, ks_name, "crypto",
+ KSTAT_TYPE_NAMED, sizeof (kcf_prov_stats_t) /
+ sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
+
+ if (prov_desc->pd_kstat != NULL) {
+ bcopy(&kcf_stats_ks_data_template,
+ &prov_desc->pd_ks_data,
+ sizeof (kcf_stats_ks_data_template));
+ prov_desc->pd_kstat->ks_data = &prov_desc->pd_ks_data;
+ KCF_PROV_REFHOLD(prov_desc);
+ KCF_PROV_IREFHOLD(prov_desc);
+ prov_desc->pd_kstat->ks_private = prov_desc;
+ prov_desc->pd_kstat->ks_update = kcf_prov_kstat_update;
+ kstat_install(prov_desc->pd_kstat);
+ }
+ }
+
+ if (prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER)
+ process_logical_providers(info, prov_desc);
+
+ mutex_enter(&prov_desc->pd_lock);
+ prov_desc->pd_state = KCF_PROV_READY;
+ mutex_exit(&prov_desc->pd_lock);
+ kcf_do_notify(prov_desc, B_TRUE);
+
+ *handle = prov_desc->pd_kcf_prov_handle;
+ ret = CRYPTO_SUCCESS;
+
+bail:
+ KCF_PROV_REFRELE(prov_desc);
+ return (ret);
+}
+
+/*
+ * This routine is used to notify the framework when a provider is being
+ * removed. Hardware providers call this routine in their detach routines.
+ * Software providers call this routine in their _fini() routine.
+ */
+int
+crypto_unregister_provider(crypto_kcf_provider_handle_t handle)
+{
+ uint_t mech_idx;
+ kcf_provider_desc_t *desc;
+ kcf_prov_state_t saved_state;
+
+ /* lookup provider descriptor */
+ if ((desc = kcf_prov_tab_lookup((crypto_provider_id_t)handle)) == NULL)
+ return (CRYPTO_UNKNOWN_PROVIDER);
+
+ mutex_enter(&desc->pd_lock);
+ /*
+ * Check if any other thread is disabling or removing
+ * this provider. We return if this is the case.
+ */
+ if (desc->pd_state >= KCF_PROV_DISABLED) {
+ mutex_exit(&desc->pd_lock);
+ /* Release reference held by kcf_prov_tab_lookup(). */
+ KCF_PROV_REFRELE(desc);
+ return (CRYPTO_BUSY);
+ }
+
+ saved_state = desc->pd_state;
+ desc->pd_state = KCF_PROV_REMOVED;
+
+ if (saved_state == KCF_PROV_BUSY) {
+ /*
+ * The per-provider taskq threads may be waiting. We
+ * signal them so that they can start failing requests.
+ */
+ cv_broadcast(&desc->pd_resume_cv);
+ }
+
+ if (desc->pd_prov_type == CRYPTO_SW_PROVIDER) {
+ /*
+ * Check if this provider is currently being used.
+ * pd_irefcnt is the number of holds from the internal
+ * structures. We add one to account for the above lookup.
+ */
+ if (desc->pd_refcnt > desc->pd_irefcnt + 1) {
+ desc->pd_state = saved_state;
+ mutex_exit(&desc->pd_lock);
+ /* Release reference held by kcf_prov_tab_lookup(). */
+ KCF_PROV_REFRELE(desc);
+ /*
+ * The administrator presumably will stop the clients
+ * thus removing the holds, when they get the busy
+ * return value. Any retry will succeed then.
+ */
+ return (CRYPTO_BUSY);
+ }
+ }
+ mutex_exit(&desc->pd_lock);
+
+ if (desc->pd_prov_type != CRYPTO_SW_PROVIDER) {
+ remove_provider(desc);
+ }
+
+ if (desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER) {
+ /* remove the provider from the mechanisms tables */
+ for (mech_idx = 0; mech_idx < desc->pd_mech_list_count;
+ mech_idx++) {
+ kcf_remove_mech_provider(
+ desc->pd_mechanisms[mech_idx].cm_mech_name, desc);
+ }
+ }
+
+ /* remove provider from providers table */
+ if (kcf_prov_tab_rem_provider((crypto_provider_id_t)handle) !=
+ CRYPTO_SUCCESS) {
+ /* Release reference held by kcf_prov_tab_lookup(). */
+ KCF_PROV_REFRELE(desc);
+ return (CRYPTO_UNKNOWN_PROVIDER);
+ }
+
+ delete_kstat(desc);
+
+ if (desc->pd_prov_type == CRYPTO_SW_PROVIDER) {
+ /* Release reference held by kcf_prov_tab_lookup(). */
+ KCF_PROV_REFRELE(desc);
+
+ /*
+ * Wait till the existing requests complete.
+ */
+ mutex_enter(&desc->pd_lock);
+ while (desc->pd_state != KCF_PROV_FREED)
+ cv_wait(&desc->pd_remove_cv, &desc->pd_lock);
+ mutex_exit(&desc->pd_lock);
+ } else {
+ /*
+ * Wait until requests that have been sent to the provider
+ * complete.
+ */
+ mutex_enter(&desc->pd_lock);
+ while (desc->pd_irefcnt > 0)
+ cv_wait(&desc->pd_remove_cv, &desc->pd_lock);
+ mutex_exit(&desc->pd_lock);
+ }
+
+ kcf_do_notify(desc, B_FALSE);
+
+ if (desc->pd_prov_type == CRYPTO_SW_PROVIDER) {
+ /*
+ * This is the only place where kcf_free_provider_desc()
+ * is called directly. KCF_PROV_REFRELE() should free the
+ * structure in all other places.
+ */
+ ASSERT(desc->pd_state == KCF_PROV_FREED &&
+ desc->pd_refcnt == 0);
+ kcf_free_provider_desc(desc);
+ } else {
+ KCF_PROV_REFRELE(desc);
+ }
+
+ return (CRYPTO_SUCCESS);
+}
+
+/*
+ * This routine is used to notify the framework that the state of
+ * a cryptographic provider has changed. Valid state codes are:
+ *
+ * CRYPTO_PROVIDER_READY
+ * The provider indicates that it can process more requests. A provider
+ * will notify with this event if it previously has notified us with a
+ * CRYPTO_PROVIDER_BUSY.
+ *
+ * CRYPTO_PROVIDER_BUSY
+ * The provider can not take more requests.
+ *
+ * CRYPTO_PROVIDER_FAILED
+ * The provider encountered an internal error. The framework will not
+ * be sending any more requests to the provider. The provider may notify
+ * with a CRYPTO_PROVIDER_READY, if it is able to recover from the error.
+ *
+ * This routine can be called from user or interrupt context.
+ */
+void
+crypto_provider_notification(crypto_kcf_provider_handle_t handle, uint_t state)
+{
+ kcf_provider_desc_t *pd;
+
+ /* lookup the provider from the given handle */
+ if ((pd = kcf_prov_tab_lookup((crypto_provider_id_t)handle)) == NULL)
+ return;
+
+ mutex_enter(&pd->pd_lock);
+
+ if (pd->pd_state <= KCF_PROV_VERIFICATION_FAILED)
+ goto out;
+
+ if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ cmn_err(CE_WARN, "crypto_provider_notification: "
+ "logical provider (%x) ignored\n", handle);
+ goto out;
+ }
+ switch (state) {
+ case CRYPTO_PROVIDER_READY:
+ switch (pd->pd_state) {
+ case KCF_PROV_BUSY:
+ pd->pd_state = KCF_PROV_READY;
+ /*
+ * Signal the per-provider taskq threads that they
+ * can start submitting requests.
+ */
+ cv_broadcast(&pd->pd_resume_cv);
+ break;
+
+ case KCF_PROV_FAILED:
+ /*
+ * The provider recovered from the error. Let us
+ * use it now.
+ */
+ pd->pd_state = KCF_PROV_READY;
+ break;
+ default:
+ break;
+ }
+ break;
+
+ case CRYPTO_PROVIDER_BUSY:
+ switch (pd->pd_state) {
+ case KCF_PROV_READY:
+ pd->pd_state = KCF_PROV_BUSY;
+ break;
+ default:
+ break;
+ }
+ break;
+
+ case CRYPTO_PROVIDER_FAILED:
+ /*
+ * We note the failure and return. The per-provider taskq
+ * threads check this flag and start failing the
+ * requests, if it is set. See process_req_hwp() for details.
+ */
+ switch (pd->pd_state) {
+ case KCF_PROV_READY:
+ pd->pd_state = KCF_PROV_FAILED;
+ break;
+
+ case KCF_PROV_BUSY:
+ pd->pd_state = KCF_PROV_FAILED;
+ /*
+ * The per-provider taskq threads may be waiting. We
+ * signal them so that they can start failing requests.
+ */
+ cv_broadcast(&pd->pd_resume_cv);
+ break;
+ default:
+ break;
+ }
+ break;
+ default:
+ break;
+ }
+out:
+ mutex_exit(&pd->pd_lock);
+ KCF_PROV_REFRELE(pd);
+}
+
+/*
+ * This routine is used to notify the framework the result of
+ * an asynchronous request handled by a provider. Valid error
+ * codes are the same as the CRYPTO_* errors defined in common.h.
+ *
+ * This routine can be called from user or interrupt context.
+ */
+void
+crypto_op_notification(crypto_req_handle_t handle, int error)
+{
+ kcf_call_type_t ctype;
+
+ if (handle == NULL)
+ return;
+
+ if ((ctype = GET_REQ_TYPE(handle)) == CRYPTO_SYNCH) {
+ kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)handle;
+
+ if (error != CRYPTO_SUCCESS)
+ sreq->sn_provider->pd_sched_info.ks_nfails++;
+ KCF_PROV_IREFRELE(sreq->sn_provider);
+ kcf_sop_done(sreq, error);
+ } else {
+ kcf_areq_node_t *areq = (kcf_areq_node_t *)handle;
+
+ ASSERT(ctype == CRYPTO_ASYNCH);
+ if (error != CRYPTO_SUCCESS)
+ areq->an_provider->pd_sched_info.ks_nfails++;
+ KCF_PROV_IREFRELE(areq->an_provider);
+ kcf_aop_done(areq, error);
+ }
+}
+
+/*
+ * This routine is used by software providers to determine
+ * whether to use KM_SLEEP or KM_NOSLEEP during memory allocation.
+ * Note that hardware providers can always use KM_SLEEP. So,
+ * they do not need to call this routine.
+ *
+ * This routine can be called from user or interrupt context.
+ */
+int
+crypto_kmflag(crypto_req_handle_t handle)
+{
+ return (REQHNDL2_KMFLAG(handle));
+}
+
+/*
+ * Process the mechanism info structures specified by the provider
+ * during registration. A NULL crypto_provider_info_t indicates
+ * an already initialized provider descriptor.
+ *
+ * Mechanisms are not added to the kernel's mechanism table if the
+ * provider is a logical provider.
+ *
+ * Returns CRYPTO_SUCCESS on success, CRYPTO_ARGUMENTS if one
+ * of the specified mechanisms was malformed, or CRYPTO_HOST_MEMORY
+ * if the table of mechanisms is full.
+ */
+static int
+init_prov_mechs(crypto_provider_info_t *info, kcf_provider_desc_t *desc)
+{
+ uint_t mech_idx;
+ uint_t cleanup_idx;
+ int err = CRYPTO_SUCCESS;
+ kcf_prov_mech_desc_t *pmd;
+ int desc_use_count = 0;
+ int mcount = desc->pd_mech_list_count;
+
+ if (desc->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
+ if (info != NULL) {
+ ASSERT(info->pi_mechanisms != NULL);
+ bcopy(info->pi_mechanisms, desc->pd_mechanisms,
+ sizeof (crypto_mech_info_t) * mcount);
+ }
+ return (CRYPTO_SUCCESS);
+ }
+
+ /*
+ * Copy the mechanism list from the provider info to the provider
+ * descriptor. desc->pd_mechanisms has an extra crypto_mech_info_t
+ * element if the provider has random_ops since we keep an internal
+ * mechanism, SUN_RANDOM, in this case.
+ */
+ if (info != NULL) {
+ if (info->pi_ops_vector->co_random_ops != NULL) {
+ crypto_mech_info_t *rand_mi;
+
+ /*
+ * Need the following check as it is possible to have
+ * a provider that implements just random_ops and has
+ * pi_mechanisms == NULL.
+ */
+ if (info->pi_mechanisms != NULL) {
+ bcopy(info->pi_mechanisms, desc->pd_mechanisms,
+ sizeof (crypto_mech_info_t) * (mcount - 1));
+ }
+ rand_mi = &desc->pd_mechanisms[mcount - 1];
+
+ bzero(rand_mi, sizeof (crypto_mech_info_t));
+ (void) strncpy(rand_mi->cm_mech_name, SUN_RANDOM,
+ CRYPTO_MAX_MECH_NAME);
+ rand_mi->cm_func_group_mask = CRYPTO_FG_RANDOM;
+ } else {
+ ASSERT(info->pi_mechanisms != NULL);
+ bcopy(info->pi_mechanisms, desc->pd_mechanisms,
+ sizeof (crypto_mech_info_t) * mcount);
+ }
+ }
+
+ /*
+ * For each mechanism support by the provider, add the provider
+ * to the corresponding KCF mechanism mech_entry chain.
+ */
+ for (mech_idx = 0; mech_idx < desc->pd_mech_list_count; mech_idx++) {
+ crypto_mech_info_t *mi = &desc->pd_mechanisms[mech_idx];
+
+ if ((mi->cm_mech_flags & CRYPTO_KEYSIZE_UNIT_IN_BITS) &&
+ (mi->cm_mech_flags & CRYPTO_KEYSIZE_UNIT_IN_BYTES)) {
+ err = CRYPTO_ARGUMENTS_BAD;
+ break;
+ }
+
+ if (desc->pd_flags & CRYPTO_HASH_NO_UPDATE &&
+ mi->cm_func_group_mask & CRYPTO_FG_DIGEST) {
+ /*
+ * We ask the provider to specify the limit
+ * per hash mechanism. But, in practice, a
+ * hardware limitation means all hash mechanisms
+ * will have the same maximum size allowed for
+ * input data. So, we make it a per provider
+ * limit to keep it simple.
+ */
+ if (mi->cm_max_input_length == 0) {
+ err = CRYPTO_ARGUMENTS_BAD;
+ break;
+ } else {
+ desc->pd_hash_limit = mi->cm_max_input_length;
+ }
+ }
+
+ if ((err = kcf_add_mech_provider(mech_idx, desc, &pmd)) !=
+ KCF_SUCCESS)
+ break;
+
+ if (pmd == NULL)
+ continue;
+
+ /* The provider will be used for this mechanism */
+ desc_use_count++;
+ }
+
+ /*
+ * Don't allow multiple software providers with disabled mechanisms
+ * to register. Subsequent enabling of mechanisms will result in
+ * an unsupported configuration, i.e. multiple software providers
+ * per mechanism.
+ */
+ if (desc_use_count == 0 && desc->pd_prov_type == CRYPTO_SW_PROVIDER)
+ return (CRYPTO_ARGUMENTS_BAD);
+
+ if (err == KCF_SUCCESS)
+ return (CRYPTO_SUCCESS);
+
+ /*
+ * An error occurred while adding the mechanism, cleanup
+ * and bail.
+ */
+ for (cleanup_idx = 0; cleanup_idx < mech_idx; cleanup_idx++) {
+ kcf_remove_mech_provider(
+ desc->pd_mechanisms[cleanup_idx].cm_mech_name, desc);
+ }
+
+ if (err == KCF_MECH_TAB_FULL)
+ return (CRYPTO_HOST_MEMORY);
+
+ return (CRYPTO_ARGUMENTS_BAD);
+}
+
+/*
+ * Update routine for kstat. Only privileged users are allowed to
+ * access this information, since this information is sensitive.
+ * There are some cryptographic attacks (e.g. traffic analysis)
+ * which can use this information.
+ */
+static int
+kcf_prov_kstat_update(kstat_t *ksp, int rw)
+{
+ kcf_prov_stats_t *ks_data;
+ kcf_provider_desc_t *pd = (kcf_provider_desc_t *)ksp->ks_private;
+
+ if (rw == KSTAT_WRITE)
+ return (EACCES);
+
+ ks_data = ksp->ks_data;
+
+ ks_data->ps_ops_total.value.ui64 =
+ pd->pd_sched_info.ks_ndispatches;
+ ks_data->ps_ops_failed.value.ui64 =
+ pd->pd_sched_info.ks_nfails;
+ ks_data->ps_ops_busy_rval.value.ui64 =
+ pd->pd_sched_info.ks_nbusy_rval;
+ ks_data->ps_ops_passed.value.ui64 =
+ pd->pd_sched_info.ks_ndispatches -
+ pd->pd_sched_info.ks_nfails -
+ pd->pd_sched_info.ks_nbusy_rval;
+
+ return (0);
+}
+
+
+/*
+ * Utility routine called from failure paths in crypto_register_provider()
+ * and from crypto_load_soft_disabled().
+ */
+void
+undo_register_provider(kcf_provider_desc_t *desc, boolean_t remove_prov)
+{
+ uint_t mech_idx;
+
+ /* remove the provider from the mechanisms tables */
+ for (mech_idx = 0; mech_idx < desc->pd_mech_list_count;
+ mech_idx++) {
+ kcf_remove_mech_provider(
+ desc->pd_mechanisms[mech_idx].cm_mech_name, desc);
+ }
+
+ /* remove provider from providers table */
+ if (remove_prov)
+ (void) kcf_prov_tab_rem_provider(desc->pd_prov_id);
+}
+
+/*
+ * Utility routine called from crypto_load_soft_disabled(). Callers
+ * should have done a prior undo_register_provider().
+ */
+void
+redo_register_provider(kcf_provider_desc_t *pd)
+{
+ /* process the mechanisms supported by the provider */
+ (void) init_prov_mechs(NULL, pd);
+
+ /*
+ * Hold provider in providers table. We should not call
+ * kcf_prov_tab_add_provider() here as the provider descriptor
+ * is still valid which means it has an entry in the provider
+ * table.
+ */
+ KCF_PROV_REFHOLD(pd);
+ KCF_PROV_IREFHOLD(pd);
+}
+
+/*
+ * Add provider (p1) to another provider's array of providers (p2).
+ * Hardware and logical providers use this array to cross-reference
+ * each other.
+ */
+static void
+add_provider_to_array(kcf_provider_desc_t *p1, kcf_provider_desc_t *p2)
+{
+ kcf_provider_list_t *new;
+
+ new = kmem_alloc(sizeof (kcf_provider_list_t), KM_SLEEP);
+ mutex_enter(&p2->pd_lock);
+ new->pl_next = p2->pd_provider_list;
+ p2->pd_provider_list = new;
+ KCF_PROV_IREFHOLD(p1);
+ new->pl_provider = p1;
+ mutex_exit(&p2->pd_lock);
+}
+
+/*
+ * Remove provider (p1) from another provider's array of providers (p2).
+ * Hardware and logical providers use this array to cross-reference
+ * each other.
+ */
+static void
+remove_provider_from_array(kcf_provider_desc_t *p1, kcf_provider_desc_t *p2)
+{
+
+ kcf_provider_list_t *pl = NULL, **prev;
+
+ mutex_enter(&p2->pd_lock);
+ for (pl = p2->pd_provider_list, prev = &p2->pd_provider_list;
+ pl != NULL; prev = &pl->pl_next, pl = pl->pl_next) {
+ if (pl->pl_provider == p1) {
+ break;
+ }
+ }
+
+ if (p1 == NULL) {
+ mutex_exit(&p2->pd_lock);
+ return;
+ }
+
+ /* detach and free kcf_provider_list structure */
+ KCF_PROV_IREFRELE(p1);
+ *prev = pl->pl_next;
+ kmem_free(pl, sizeof (*pl));
+ mutex_exit(&p2->pd_lock);
+}
+
+/*
+ * Convert an array of logical provider handles (crypto_provider_id)
+ * stored in a crypto_provider_info structure into an array of provider
+ * descriptors (kcf_provider_desc_t) attached to a logical provider.
+ */
+static void
+process_logical_providers(crypto_provider_info_t *info, kcf_provider_desc_t *hp)
+{
+ kcf_provider_desc_t *lp;
+ crypto_provider_id_t handle;
+ int count = info->pi_logical_provider_count;
+ int i;
+
+ /* add hardware provider to each logical provider */
+ for (i = 0; i < count; i++) {
+ handle = info->pi_logical_providers[i];
+ lp = kcf_prov_tab_lookup((crypto_provider_id_t)handle);
+ if (lp == NULL) {
+ continue;
+ }
+ add_provider_to_array(hp, lp);
+ hp->pd_flags |= KCF_LPROV_MEMBER;
+
+ /*
+ * A hardware provider has to have the provider descriptor of
+ * every logical provider it belongs to, so it can be removed
+ * from the logical provider if the hardware provider
+ * unregisters from the framework.
+ */
+ add_provider_to_array(lp, hp);
+ KCF_PROV_REFRELE(lp);
+ }
+}
+
+/*
+ * This routine removes a provider from all of the logical or
+ * hardware providers it belongs to, and frees the provider's
+ * array of pointers to providers.
+ */
+static void
+remove_provider(kcf_provider_desc_t *pp)
+{
+ kcf_provider_desc_t *p;
+ kcf_provider_list_t *e, *next;
+
+ mutex_enter(&pp->pd_lock);
+ for (e = pp->pd_provider_list; e != NULL; e = next) {
+ p = e->pl_provider;
+ remove_provider_from_array(pp, p);
+ if (p->pd_prov_type == CRYPTO_HW_PROVIDER &&
+ p->pd_provider_list == NULL)
+ p->pd_flags &= ~KCF_LPROV_MEMBER;
+ KCF_PROV_IREFRELE(p);
+ next = e->pl_next;
+ kmem_free(e, sizeof (*e));
+ }
+ pp->pd_provider_list = NULL;
+ mutex_exit(&pp->pd_lock);
+}
+
+/*
+ * Dispatch events as needed for a provider. is_added flag tells
+ * whether the provider is registering or unregistering.
+ */
+void
+kcf_do_notify(kcf_provider_desc_t *prov_desc, boolean_t is_added)
+{
+ int i;
+ crypto_notify_event_change_t ec;
+
+ ASSERT(prov_desc->pd_state > KCF_PROV_VERIFICATION_FAILED);
+
+ /*
+ * Inform interested clients of the mechanisms becoming
+ * available/unavailable. We skip this for logical providers
+ * as they do not affect mechanisms.
+ */
+ if (prov_desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER) {
+ ec.ec_provider_type = prov_desc->pd_prov_type;
+ ec.ec_change = is_added ? CRYPTO_MECH_ADDED :
+ CRYPTO_MECH_REMOVED;
+ for (i = 0; i < prov_desc->pd_mech_list_count; i++) {
+ (void) strncpy(ec.ec_mech_name,
+ prov_desc->pd_mechanisms[i].cm_mech_name,
+ CRYPTO_MAX_MECH_NAME);
+ kcf_walk_ntfylist(CRYPTO_EVENT_MECHS_CHANGED, &ec);
+ }
+
+ }
+
+ /*
+ * Inform interested clients about the new or departing provider.
+ * In case of a logical provider, we need to notify the event only
+ * for the logical provider and not for the underlying
+ * providers which are known by the KCF_LPROV_MEMBER bit.
+ */
+ if (prov_desc->pd_prov_type == CRYPTO_LOGICAL_PROVIDER ||
+ (prov_desc->pd_flags & KCF_LPROV_MEMBER) == 0) {
+ kcf_walk_ntfylist(is_added ? CRYPTO_EVENT_PROVIDER_REGISTERED :
+ CRYPTO_EVENT_PROVIDER_UNREGISTERED, prov_desc);
+ }
+}
+
+static void
+delete_kstat(kcf_provider_desc_t *desc)
+{
+ /* destroy the kstat created for this provider */
+ if (desc->pd_kstat != NULL) {
+ kcf_provider_desc_t *kspd = desc->pd_kstat->ks_private;
+
+ /* release reference held by desc->pd_kstat->ks_private */
+ ASSERT(desc == kspd);
+ kstat_delete(kspd->pd_kstat);
+ desc->pd_kstat = NULL;
+ KCF_PROV_REFRELE(kspd);
+ KCF_PROV_IREFRELE(kspd);
+ }
+}
. "${basedir}/../${SCRIPT_CONFIG}"
else
KERNEL_MODULES=(zlib_deflate zlib_inflate)
-MODULES=(spl splat zavl znvpair zunicode zcommon zfs)
+MODULES=(spl splat zavl znvpair zunicode zcommon icp zfs)
fi
PROG="<define PROG>"
${MODDIR}/nvpair/znvpair.ko \
${MODDIR}/unicode/zunicode.ko \
${MODDIR}/zcommon/zcommon.ko \
+ ${MODDIR}/icp/icp.ko \
${MODDIR}/zfs/zfs.ko \
)
projects / mirror_zfs.git / commitdiff