2 * Crypto-API module for CRC-32 algorithms implemented with the
3 * z/Architecture Vector Extension Facility.
5 * Copyright IBM Corp. 2015
6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
8 #define KMSG_COMPONENT "crc32-vx"
9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
11 #include <linux/module.h>
12 #include <linux/cpufeature.h>
13 #include <linux/crc32.h>
14 #include <crypto/internal/hash.h>
15 #include <asm/fpu/api.h>
18 #define CRC32_BLOCK_SIZE 1
19 #define CRC32_DIGEST_SIZE 4
22 #define VX_ALIGNMENT 16L
23 #define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
33 /* Prototypes for functions in assembly files */
34 u32
crc32_le_vgfm_16(u32 crc
, unsigned char const *buf
, size_t size
);
35 u32
crc32_be_vgfm_16(u32 crc
, unsigned char const *buf
, size_t size
);
36 u32
crc32c_le_vgfm_16(u32 crc
, unsigned char const *buf
, size_t size
);
39 * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
41 * Creates a function to perform a particular CRC-32 computation. Depending
42 * on the message buffer, the hardware-accelerated or software implementation
43 * is used. Note that the message buffer is aligned to improve fetch
44 * operations of VECTOR LOAD MULTIPLE instructions.
47 #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
48 static u32 __pure ___fname(u32 crc, \
49 unsigned char const *data, size_t datalen) \
51 struct kernel_fpu vxstate; \
52 unsigned long prealign, aligned, remaining; \
54 if (datalen < VX_MIN_LEN + VX_ALIGN_MASK) \
55 return ___crc32_sw(crc, data, datalen); \
57 if ((unsigned long)data & VX_ALIGN_MASK) { \
58 prealign = VX_ALIGNMENT - \
59 ((unsigned long)data & VX_ALIGN_MASK); \
60 datalen -= prealign; \
61 crc = ___crc32_sw(crc, data, prealign); \
62 data = (void *)((unsigned long)data + prealign); \
65 aligned = datalen & ~VX_ALIGN_MASK; \
66 remaining = datalen & VX_ALIGN_MASK; \
68 kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
69 crc = ___crc32_vx(crc, data, aligned); \
70 kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
73 crc = ___crc32_sw(crc, data + aligned, remaining); \
78 DEFINE_CRC32_VX(crc32_le_vx
, crc32_le_vgfm_16
, crc32_le
)
79 DEFINE_CRC32_VX(crc32_be_vx
, crc32_be_vgfm_16
, crc32_be
)
80 DEFINE_CRC32_VX(crc32c_le_vx
, crc32c_le_vgfm_16
, __crc32c_le
)
83 static int crc32_vx_cra_init_zero(struct crypto_tfm
*tfm
)
85 struct crc_ctx
*mctx
= crypto_tfm_ctx(tfm
);
91 static int crc32_vx_cra_init_invert(struct crypto_tfm
*tfm
)
93 struct crc_ctx
*mctx
= crypto_tfm_ctx(tfm
);
99 static int crc32_vx_init(struct shash_desc
*desc
)
101 struct crc_ctx
*mctx
= crypto_shash_ctx(desc
->tfm
);
102 struct crc_desc_ctx
*ctx
= shash_desc_ctx(desc
);
104 ctx
->crc
= mctx
->key
;
108 static int crc32_vx_setkey(struct crypto_shash
*tfm
, const u8
*newkey
,
109 unsigned int newkeylen
)
111 struct crc_ctx
*mctx
= crypto_shash_ctx(tfm
);
113 if (newkeylen
!= sizeof(mctx
->key
)) {
114 crypto_shash_set_flags(tfm
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
117 mctx
->key
= le32_to_cpu(*(__le32
*)newkey
);
121 static int crc32be_vx_setkey(struct crypto_shash
*tfm
, const u8
*newkey
,
122 unsigned int newkeylen
)
124 struct crc_ctx
*mctx
= crypto_shash_ctx(tfm
);
126 if (newkeylen
!= sizeof(mctx
->key
)) {
127 crypto_shash_set_flags(tfm
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
130 mctx
->key
= be32_to_cpu(*(__be32
*)newkey
);
134 static int crc32le_vx_final(struct shash_desc
*desc
, u8
*out
)
136 struct crc_desc_ctx
*ctx
= shash_desc_ctx(desc
);
138 *(__le32
*)out
= cpu_to_le32p(&ctx
->crc
);
142 static int crc32be_vx_final(struct shash_desc
*desc
, u8
*out
)
144 struct crc_desc_ctx
*ctx
= shash_desc_ctx(desc
);
146 *(__be32
*)out
= cpu_to_be32p(&ctx
->crc
);
150 static int crc32c_vx_final(struct shash_desc
*desc
, u8
*out
)
152 struct crc_desc_ctx
*ctx
= shash_desc_ctx(desc
);
155 * Perform a final XOR with 0xFFFFFFFF to be in sync
156 * with the generic crc32c shash implementation.
158 *(__le32
*)out
= ~cpu_to_le32p(&ctx
->crc
);
162 static int __crc32le_vx_finup(u32
*crc
, const u8
*data
, unsigned int len
,
165 *(__le32
*)out
= cpu_to_le32(crc32_le_vx(*crc
, data
, len
));
169 static int __crc32be_vx_finup(u32
*crc
, const u8
*data
, unsigned int len
,
172 *(__be32
*)out
= cpu_to_be32(crc32_be_vx(*crc
, data
, len
));
176 static int __crc32c_vx_finup(u32
*crc
, const u8
*data
, unsigned int len
,
180 * Perform a final XOR with 0xFFFFFFFF to be in sync
181 * with the generic crc32c shash implementation.
183 *(__le32
*)out
= ~cpu_to_le32(crc32c_le_vx(*crc
, data
, len
));
188 #define CRC32_VX_FINUP(alg, func) \
189 static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data, \
190 unsigned int datalen, u8 *out) \
192 return __ ## alg ## _vx_finup(shash_desc_ctx(desc), \
193 data, datalen, out); \
196 CRC32_VX_FINUP(crc32le
, crc32_le_vx
)
197 CRC32_VX_FINUP(crc32be
, crc32_be_vx
)
198 CRC32_VX_FINUP(crc32c
, crc32c_le_vx
)
200 #define CRC32_VX_DIGEST(alg, func) \
201 static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
202 unsigned int len, u8 *out) \
204 return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm), \
208 CRC32_VX_DIGEST(crc32le
, crc32_le_vx
)
209 CRC32_VX_DIGEST(crc32be
, crc32_be_vx
)
210 CRC32_VX_DIGEST(crc32c
, crc32c_le_vx
)
212 #define CRC32_VX_UPDATE(alg, func) \
213 static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
214 unsigned int datalen) \
216 struct crc_desc_ctx *ctx = shash_desc_ctx(desc); \
217 ctx->crc = func(ctx->crc, data, datalen); \
221 CRC32_VX_UPDATE(crc32le
, crc32_le_vx
)
222 CRC32_VX_UPDATE(crc32be
, crc32_be_vx
)
223 CRC32_VX_UPDATE(crc32c
, crc32c_le_vx
)
226 static struct shash_alg crc32_vx_algs
[] = {
229 .init
= crc32_vx_init
,
230 .setkey
= crc32_vx_setkey
,
231 .update
= crc32le_vx_update
,
232 .final
= crc32le_vx_final
,
233 .finup
= crc32le_vx_finup
,
234 .digest
= crc32le_vx_digest
,
235 .descsize
= sizeof(struct crc_desc_ctx
),
236 .digestsize
= CRC32_DIGEST_SIZE
,
239 .cra_driver_name
= "crc32-vx",
241 .cra_blocksize
= CRC32_BLOCK_SIZE
,
242 .cra_ctxsize
= sizeof(struct crc_ctx
),
243 .cra_module
= THIS_MODULE
,
244 .cra_init
= crc32_vx_cra_init_zero
,
249 .init
= crc32_vx_init
,
250 .setkey
= crc32be_vx_setkey
,
251 .update
= crc32be_vx_update
,
252 .final
= crc32be_vx_final
,
253 .finup
= crc32be_vx_finup
,
254 .digest
= crc32be_vx_digest
,
255 .descsize
= sizeof(struct crc_desc_ctx
),
256 .digestsize
= CRC32_DIGEST_SIZE
,
258 .cra_name
= "crc32be",
259 .cra_driver_name
= "crc32be-vx",
261 .cra_blocksize
= CRC32_BLOCK_SIZE
,
262 .cra_ctxsize
= sizeof(struct crc_ctx
),
263 .cra_module
= THIS_MODULE
,
264 .cra_init
= crc32_vx_cra_init_zero
,
269 .init
= crc32_vx_init
,
270 .setkey
= crc32_vx_setkey
,
271 .update
= crc32c_vx_update
,
272 .final
= crc32c_vx_final
,
273 .finup
= crc32c_vx_finup
,
274 .digest
= crc32c_vx_digest
,
275 .descsize
= sizeof(struct crc_desc_ctx
),
276 .digestsize
= CRC32_DIGEST_SIZE
,
278 .cra_name
= "crc32c",
279 .cra_driver_name
= "crc32c-vx",
281 .cra_blocksize
= CRC32_BLOCK_SIZE
,
282 .cra_ctxsize
= sizeof(struct crc_ctx
),
283 .cra_module
= THIS_MODULE
,
284 .cra_init
= crc32_vx_cra_init_invert
,
290 static int __init
crc_vx_mod_init(void)
292 return crypto_register_shashes(crc32_vx_algs
,
293 ARRAY_SIZE(crc32_vx_algs
));
296 static void __exit
crc_vx_mod_exit(void)
298 crypto_unregister_shashes(crc32_vx_algs
, ARRAY_SIZE(crc32_vx_algs
));
301 module_cpu_feature_match(VXRS
, crc_vx_mod_init
);
302 module_exit(crc_vx_mod_exit
);
304 MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
305 MODULE_LICENSE("GPL");
307 MODULE_ALIAS_CRYPTO("crc32");
308 MODULE_ALIAS_CRYPTO("crc32-vx");
309 MODULE_ALIAS_CRYPTO("crc32c");
310 MODULE_ALIAS_CRYPTO("crc32c-vx");