]>
Commit | Line | Data |
---|---|---|
66be8951 MK |
1 | /* |
2 | * Cryptographic API. | |
3 | * | |
4 | * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using | |
5 | * Supplemental SSE3 instructions. | |
6 | * | |
7 | * This file is based on sha1_generic.c | |
8 | * | |
9 | * Copyright (c) Alan Smithee. | |
10 | * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> | |
11 | * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> | |
12 | * Copyright (c) Mathias Krause <minipli@googlemail.com> | |
7c1da8d0 | 13 | * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com> |
66be8951 MK |
14 | * |
15 | * This program is free software; you can redistribute it and/or modify it | |
16 | * under the terms of the GNU General Public License as published by the Free | |
17 | * Software Foundation; either version 2 of the License, or (at your option) | |
18 | * any later version. | |
19 | * | |
20 | */ | |
21 | ||
22 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
23 | ||
24 | #include <crypto/internal/hash.h> | |
25 | #include <linux/init.h> | |
26 | #include <linux/module.h> | |
27 | #include <linux/mm.h> | |
28 | #include <linux/cryptohash.h> | |
29 | #include <linux/types.h> | |
30 | #include <crypto/sha.h> | |
824b4376 | 31 | #include <crypto/sha1_base.h> |
df6b35f4 | 32 | #include <asm/fpu/api.h> |
66be8951 | 33 | #include <asm/xcr.h> |
669ebabb | 34 | #include <asm/fpu/xstate.h> |
66be8951 MK |
35 | |
36 | ||
37 | asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data, | |
38 | unsigned int rounds); | |
65df5774 | 39 | #ifdef CONFIG_AS_AVX |
66be8951 MK |
40 | asmlinkage void sha1_transform_avx(u32 *digest, const char *data, |
41 | unsigned int rounds); | |
42 | #endif | |
7c1da8d0 | 43 | #ifdef CONFIG_AS_AVX2 |
44 | #define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */ | |
45 | ||
46 | asmlinkage void sha1_transform_avx2(u32 *digest, const char *data, | |
824b4376 | 47 | unsigned int rounds); |
7c1da8d0 | 48 | #endif |
66be8951 | 49 | |
824b4376 | 50 | static void (*sha1_transform_asm)(u32 *, const char *, unsigned int); |
66be8951 MK |
51 | |
52 | static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data, | |
53 | unsigned int len) | |
54 | { | |
55 | struct sha1_state *sctx = shash_desc_ctx(desc); | |
66be8951 | 56 | |
824b4376 AB |
57 | if (!irq_fpu_usable() || |
58 | (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) | |
59 | return crypto_sha1_update(desc, data, len); | |
66be8951 | 60 | |
824b4376 AB |
61 | /* make sure casting to sha1_block_fn() is safe */ |
62 | BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); | |
66be8951 | 63 | |
824b4376 AB |
64 | kernel_fpu_begin(); |
65 | sha1_base_do_update(desc, data, len, | |
66 | (sha1_block_fn *)sha1_transform_asm); | |
67 | kernel_fpu_end(); | |
66be8951 MK |
68 | |
69 | return 0; | |
70 | } | |
71 | ||
824b4376 AB |
72 | static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, |
73 | unsigned int len, u8 *out) | |
66be8951 | 74 | { |
824b4376 AB |
75 | if (!irq_fpu_usable()) |
76 | return crypto_sha1_finup(desc, data, len, out); | |
66be8951 | 77 | |
824b4376 AB |
78 | kernel_fpu_begin(); |
79 | if (len) | |
80 | sha1_base_do_update(desc, data, len, | |
81 | (sha1_block_fn *)sha1_transform_asm); | |
82 | sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_transform_asm); | |
83 | kernel_fpu_end(); | |
66be8951 | 84 | |
824b4376 | 85 | return sha1_base_finish(desc, out); |
66be8951 MK |
86 | } |
87 | ||
824b4376 AB |
88 | /* Add padding and return the message digest. */ |
89 | static int sha1_ssse3_final(struct shash_desc *desc, u8 *out) | |
66be8951 | 90 | { |
824b4376 | 91 | return sha1_ssse3_finup(desc, NULL, 0, out); |
66be8951 MK |
92 | } |
93 | ||
7c1da8d0 | 94 | #ifdef CONFIG_AS_AVX2 |
95 | static void sha1_apply_transform_avx2(u32 *digest, const char *data, | |
96 | unsigned int rounds) | |
97 | { | |
98 | /* Select the optimal transform based on data block size */ | |
99 | if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE) | |
100 | sha1_transform_avx2(digest, data, rounds); | |
101 | else | |
102 | sha1_transform_avx(digest, data, rounds); | |
103 | } | |
104 | #endif | |
105 | ||
66be8951 MK |
106 | static struct shash_alg alg = { |
107 | .digestsize = SHA1_DIGEST_SIZE, | |
824b4376 | 108 | .init = sha1_base_init, |
66be8951 MK |
109 | .update = sha1_ssse3_update, |
110 | .final = sha1_ssse3_final, | |
824b4376 | 111 | .finup = sha1_ssse3_finup, |
66be8951 | 112 | .descsize = sizeof(struct sha1_state), |
66be8951 MK |
113 | .base = { |
114 | .cra_name = "sha1", | |
115 | .cra_driver_name= "sha1-ssse3", | |
116 | .cra_priority = 150, | |
117 | .cra_flags = CRYPTO_ALG_TYPE_SHASH, | |
118 | .cra_blocksize = SHA1_BLOCK_SIZE, | |
119 | .cra_module = THIS_MODULE, | |
120 | } | |
121 | }; | |
122 | ||
65df5774 | 123 | #ifdef CONFIG_AS_AVX |
66be8951 MK |
124 | static bool __init avx_usable(void) |
125 | { | |
126 | u64 xcr0; | |
127 | ||
128 | if (!cpu_has_avx || !cpu_has_osxsave) | |
129 | return false; | |
130 | ||
131 | xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); | |
132 | if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { | |
133 | pr_info("AVX detected but unusable.\n"); | |
134 | ||
135 | return false; | |
136 | } | |
137 | ||
138 | return true; | |
139 | } | |
6ca5afb8 MK |
140 | |
141 | #ifdef CONFIG_AS_AVX2 | |
142 | static bool __init avx2_usable(void) | |
143 | { | |
144 | if (avx_usable() && cpu_has_avx2 && boot_cpu_has(X86_FEATURE_BMI1) && | |
145 | boot_cpu_has(X86_FEATURE_BMI2)) | |
146 | return true; | |
147 | ||
148 | return false; | |
149 | } | |
150 | #endif | |
66be8951 MK |
151 | #endif |
152 | ||
153 | static int __init sha1_ssse3_mod_init(void) | |
154 | { | |
7c1da8d0 | 155 | char *algo_name; |
6ca5afb8 | 156 | |
66be8951 | 157 | /* test for SSSE3 first */ |
7c1da8d0 | 158 | if (cpu_has_ssse3) { |
66be8951 | 159 | sha1_transform_asm = sha1_transform_ssse3; |
7c1da8d0 | 160 | algo_name = "SSSE3"; |
161 | } | |
66be8951 | 162 | |
65df5774 | 163 | #ifdef CONFIG_AS_AVX |
66be8951 | 164 | /* allow AVX to override SSSE3, it's a little faster */ |
7c1da8d0 | 165 | if (avx_usable()) { |
6ca5afb8 MK |
166 | sha1_transform_asm = sha1_transform_avx; |
167 | algo_name = "AVX"; | |
7c1da8d0 | 168 | #ifdef CONFIG_AS_AVX2 |
6ca5afb8 MK |
169 | /* allow AVX2 to override AVX, it's a little faster */ |
170 | if (avx2_usable()) { | |
7c1da8d0 | 171 | sha1_transform_asm = sha1_apply_transform_avx2; |
172 | algo_name = "AVX2"; | |
173 | } | |
174 | #endif | |
175 | } | |
66be8951 MK |
176 | #endif |
177 | ||
178 | if (sha1_transform_asm) { | |
7c1da8d0 | 179 | pr_info("Using %s optimized SHA-1 implementation\n", algo_name); |
66be8951 MK |
180 | return crypto_register_shash(&alg); |
181 | } | |
7c1da8d0 | 182 | pr_info("Neither AVX nor AVX2 nor SSSE3 is available/usable.\n"); |
66be8951 MK |
183 | |
184 | return -ENODEV; | |
185 | } | |
186 | ||
187 | static void __exit sha1_ssse3_mod_fini(void) | |
188 | { | |
189 | crypto_unregister_shash(&alg); | |
190 | } | |
191 | ||
192 | module_init(sha1_ssse3_mod_init); | |
193 | module_exit(sha1_ssse3_mod_fini); | |
194 | ||
195 | MODULE_LICENSE("GPL"); | |
196 | MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated"); | |
197 | ||
5d26a105 | 198 | MODULE_ALIAS_CRYPTO("sha1"); |