[mirror_ubuntu-bionic-kernel.git] / arch / x86 / include / asm / crypto / glue_helper.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Shared glue code for 128bit block ciphers
 */

#ifndef _CRYPTO_GLUE_HELPER_H
#define _CRYPTO_GLUE_HELPER_H

#include <crypto/internal/skcipher.h>
#include <linux/kernel.h>
#include <asm/fpu/api.h>
#include <crypto/b128ops.h>

typedef void (*common_glue_func_t)(void *ctx, u8 *dst, const u8 *src);
typedef void (*common_glue_cbc_func_t)(void *ctx, u128 *dst, const u128 *src);
typedef void (*common_glue_ctr_func_t)(void *ctx, u128 *dst, const u128 *src,
				       le128 *iv);
typedef void (*common_glue_xts_func_t)(void *ctx, u128 *dst, const u128 *src,
				       le128 *iv);

#define GLUE_FUNC_CAST(fn) ((common_glue_func_t)(fn))
#define GLUE_CBC_FUNC_CAST(fn) ((common_glue_cbc_func_t)(fn))
#define GLUE_CTR_FUNC_CAST(fn) ((common_glue_ctr_func_t)(fn))
#define GLUE_XTS_FUNC_CAST(fn) ((common_glue_xts_func_t)(fn))

struct common_glue_func_entry {
	unsigned int num_blocks; /* number of blocks that @fn will process */
	union {
		common_glue_func_t ecb;
		common_glue_cbc_func_t cbc;
		common_glue_ctr_func_t ctr;
		common_glue_xts_func_t xts;
	} fn_u;
};

struct common_glue_ctx {
	unsigned int num_funcs;
	int fpu_blocks_limit; /* -1 means fpu not needed at all */

	/*
	 * First funcs entry must have largest num_blocks and last funcs entry
	 * must have num_blocks == 1!
	 */
	struct common_glue_func_entry funcs[];
};

static inline bool glue_fpu_begin(unsigned int bsize, int fpu_blocks_limit,
				  struct blkcipher_desc *desc,
				  bool fpu_enabled, unsigned int nbytes)
{
	if (likely(fpu_blocks_limit < 0))
		return false;

	if (fpu_enabled)
		return true;

	/*
	 * Vector-registers are only used when chunk to be processed is large
	 * enough, so do not enable FPU until it is necessary.
	 */
	if (nbytes < bsize * (unsigned int)fpu_blocks_limit)
		return false;

	if (desc) {
		/* prevent sleeping if FPU is in use */
		desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	}

	kernel_fpu_begin();
	return true;
}

static inline bool glue_skwalk_fpu_begin(unsigned int bsize,
					 int fpu_blocks_limit,
					 struct skcipher_walk *walk,
					 bool fpu_enabled, unsigned int nbytes)
{
	if (likely(fpu_blocks_limit < 0))
		return false;

	if (fpu_enabled)
		return true;

	/*
	 * Vector-registers are only used when chunk to be processed is large
	 * enough, so do not enable FPU until it is necessary.
	 */
	if (nbytes < bsize * (unsigned int)fpu_blocks_limit)
		return false;

	/* prevent sleeping if FPU is in use */
	skcipher_walk_atomise(walk);

	kernel_fpu_begin();
	return true;
}

static inline void glue_fpu_end(bool fpu_enabled)
{
	if (fpu_enabled)
		kernel_fpu_end();
}

static inline void le128_to_be128(be128 *dst, const le128 *src)
{
	dst->a = cpu_to_be64(le64_to_cpu(src->a));
	dst->b = cpu_to_be64(le64_to_cpu(src->b));
}

static inline void be128_to_le128(le128 *dst, const be128 *src)
{
	dst->a = cpu_to_le64(be64_to_cpu(src->a));
	dst->b = cpu_to_le64(be64_to_cpu(src->b));
}

static inline void le128_inc(le128 *i)
{
	u64 a = le64_to_cpu(i->a);
	u64 b = le64_to_cpu(i->b);

	b++;
	if (!b)
		a++;

	i->a = cpu_to_le64(a);
	i->b = cpu_to_le64(b);
}

extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
				 struct blkcipher_desc *desc,
				 struct scatterlist *dst,
				 struct scatterlist *src, unsigned int nbytes);

extern int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
				   struct blkcipher_desc *desc,
				   struct scatterlist *dst,
				   struct scatterlist *src,
				   unsigned int nbytes);

extern int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
				   struct blkcipher_desc *desc,
				   struct scatterlist *dst,
				   struct scatterlist *src,
				   unsigned int nbytes);

extern int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
				 struct blkcipher_desc *desc,
				 struct scatterlist *dst,
				 struct scatterlist *src, unsigned int nbytes);

extern int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
				 struct blkcipher_desc *desc,
				 struct scatterlist *dst,
				 struct scatterlist *src, unsigned int nbytes,
				 common_glue_func_t tweak_fn, void *tweak_ctx,
				 void *crypt_ctx);

extern int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
				 struct blkcipher_desc *desc,
				 struct scatterlist *dst,
				 struct scatterlist *src, unsigned int nbytes,
				 common_glue_func_t tweak_fn, void *tweak_ctx,
				 void *crypt_ctx);

extern int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
			       struct skcipher_request *req,
			       common_glue_func_t tweak_fn, void *tweak_ctx,
			       void *crypt_ctx);

extern void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src,
				      le128 *iv, common_glue_func_t fn);

#endif /* _CRYPTO_GLUE_HELPER_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
596d8750 JK	2	/*
	3	* Shared glue code for 128bit block ciphers
	4	*/
	5
	6	#ifndef _CRYPTO_GLUE_HELPER_H
	7	#define _CRYPTO_GLUE_HELPER_H
	8
065ce327	9	#include <crypto/internal/skcipher.h>
596d8750	10	#include <linux/kernel.h>
df6b35f4	11	#include <asm/fpu/api.h>
596d8750 JK	12	#include <crypto/b128ops.h>
	13
	14	typedef void (common_glue_func_t)(void ctx, u8 dst, const u8 src);
	15	typedef void (common_glue_cbc_func_t)(void ctx, u128 dst, const u128 src);
	16	typedef void (common_glue_ctr_func_t)(void ctx, u128 dst, const u128 src,
58990986	17	le128 *iv);
a05248ed JK	18	typedef void (common_glue_xts_func_t)(void ctx, u128 dst, const u128 src,
a05248ed JK	19	le128 *iv);
596d8750 JK	20
	21	#define GLUE_FUNC_CAST(fn) ((common_glue_func_t)(fn))
	22	#define GLUE_CBC_FUNC_CAST(fn) ((common_glue_cbc_func_t)(fn))
	23	#define GLUE_CTR_FUNC_CAST(fn) ((common_glue_ctr_func_t)(fn))
a05248ed	24	#define GLUE_XTS_FUNC_CAST(fn) ((common_glue_xts_func_t)(fn))
596d8750 JK	25
	26	struct common_glue_func_entry {
	27	unsigned int num_blocks; /* number of blocks that @fn will process */
	28	union {
	29	common_glue_func_t ecb;
	30	common_glue_cbc_func_t cbc;
	31	common_glue_ctr_func_t ctr;
a05248ed	32	common_glue_xts_func_t xts;
596d8750 JK	33	} fn_u;
	34	};
	35
	36	struct common_glue_ctx {
	37	unsigned int num_funcs;
	38	int fpu_blocks_limit; /* -1 means fpu not needed at all */
	39
	40	/*
	41	* First funcs entry must have largest num_blocks and last funcs entry
	42	* must have num_blocks == 1!
	43	*/
	44	struct common_glue_func_entry funcs[];
	45	};
	46
	47	static inline bool glue_fpu_begin(unsigned int bsize, int fpu_blocks_limit,
	48	struct blkcipher_desc *desc,
	49	bool fpu_enabled, unsigned int nbytes)
	50	{
	51	if (likely(fpu_blocks_limit < 0))
	52	return false;
	53
	54	if (fpu_enabled)
	55	return true;
	56
	57	/*
	58	* Vector-registers are only used when chunk to be processed is large
	59	* enough, so do not enable FPU until it is necessary.
	60	*/
	61	if (nbytes < bsize * (unsigned int)fpu_blocks_limit)
	62	return false;
	63
	64	if (desc) {
	65	/* prevent sleeping if FPU is in use */
	66	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	67	}
	68
	69	kernel_fpu_begin();
	70	return true;
	71	}
	72
065ce327 HX	73	static inline bool glue_skwalk_fpu_begin(unsigned int bsize,
	74	int fpu_blocks_limit,
	75	struct skcipher_walk *walk,
	76	bool fpu_enabled, unsigned int nbytes)
	77	{
	78	if (likely(fpu_blocks_limit < 0))
	79	return false;
	80
	81	if (fpu_enabled)
	82	return true;
	83
	84	/*
	85	* Vector-registers are only used when chunk to be processed is large
	86	* enough, so do not enable FPU until it is necessary.
	87	*/
	88	if (nbytes < bsize * (unsigned int)fpu_blocks_limit)
	89	return false;
	90
	91	/* prevent sleeping if FPU is in use */
	92	skcipher_walk_atomise(walk);
	93
	94	kernel_fpu_begin();
	95	return true;
	96	}
	97
596d8750 JK	98	static inline void glue_fpu_end(bool fpu_enabled)
	99	{
	100	if (fpu_enabled)
	101	kernel_fpu_end();
	102	}
	103
58990986	104	static inline void le128_to_be128(be128 dst, const le128 src)
596d8750	105	{
58990986 JK	106	dst->a = cpu_to_be64(le64_to_cpu(src->a));
58990986 JK	107	dst->b = cpu_to_be64(le64_to_cpu(src->b));
596d8750 JK	108	}
596d8750 JK	109
58990986	110	static inline void be128_to_le128(le128 dst, const be128 src)
596d8750	111	{
58990986 JK	112	dst->a = cpu_to_le64(be64_to_cpu(src->a));
58990986 JK	113	dst->b = cpu_to_le64(be64_to_cpu(src->b));
596d8750 JK	114	}
596d8750 JK	115
58990986	116	static inline void le128_inc(le128 *i)
596d8750	117	{
58990986 JK	118	u64 a = le64_to_cpu(i->a);
	119	u64 b = le64_to_cpu(i->b);
	120
	121	b++;
	122	if (!b)
	123	a++;
	124
	125	i->a = cpu_to_le64(a);
	126	i->b = cpu_to_le64(b);
596d8750 JK	127	}
	128
	129	extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
	130	struct blkcipher_desc *desc,
	131	struct scatterlist *dst,
	132	struct scatterlist *src, unsigned int nbytes);
	133
	134	extern int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
	135	struct blkcipher_desc *desc,
	136	struct scatterlist *dst,
	137	struct scatterlist *src,
	138	unsigned int nbytes);
	139
	140	extern int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
	141	struct blkcipher_desc *desc,
	142	struct scatterlist *dst,
	143	struct scatterlist *src,
	144	unsigned int nbytes);
	145
	146	extern int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
	147	struct blkcipher_desc *desc,
	148	struct scatterlist *dst,
	149	struct scatterlist *src, unsigned int nbytes);
	150
a05248ed JK	151	extern int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
	152	struct blkcipher_desc *desc,
	153	struct scatterlist *dst,
	154	struct scatterlist *src, unsigned int nbytes,
	155	common_glue_func_t tweak_fn, void *tweak_ctx,
	156	void *crypt_ctx);
	157
065ce327 HX	158	extern int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
	159	struct blkcipher_desc *desc,
	160	struct scatterlist *dst,
	161	struct scatterlist *src, unsigned int nbytes,
	162	common_glue_func_t tweak_fn, void *tweak_ctx,
	163	void *crypt_ctx);
	164
	165	extern int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
	166	struct skcipher_request *req,
	167	common_glue_func_t tweak_fn, void *tweak_ctx,
	168	void *crypt_ctx);
	169
a05248ed JK	170	extern void glue_xts_crypt_128bit_one(void ctx, u128 dst, const u128 *src,
	171	le128 *iv, common_glue_func_t fn);
	172
596d8750	173	#endif /* _CRYPTO_GLUE_HELPER_H */