[mirror_ubuntu-artful-kernel.git] / arch / s390 / crypto / aes_s390.c

/*
 * Cryptographic API.
 *
 * s390 implementation of the AES Cipher Algorithm.
 *
 * s390 Version:
 *   Copyright IBM Corp. 2005,2007
 *   Author(s): Jan Glauber (jang@de.ibm.com)
 *		Sebastian Siewior (sebastian@breakpoint.cc> SW-Fallback
 *
 * Derived from "crypto/aes_generic.c"
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 */

#define KMSG_COMPONENT "aes_s390"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/init.h>
#include "crypt_s390.h"

#define AES_KEYLEN_128		1
#define AES_KEYLEN_192		2
#define AES_KEYLEN_256		4

static char keylen_flag = 0;

struct s390_aes_ctx {
	u8 iv[AES_BLOCK_SIZE];
	u8 key[AES_MAX_KEY_SIZE];
	long enc;
	long dec;
	int key_len;
	union {
		struct crypto_blkcipher *blk;
		struct crypto_cipher *cip;
	} fallback;
};

/*
 * Check if the key_len is supported by the HW.
 * Returns 0 if it is, a positive number if it is not and software fallback is
 * required or a negative number in case the key size is not valid
 */
static int need_fallback(unsigned int key_len)
{
	switch (key_len) {
	case 16:
		if (!(keylen_flag & AES_KEYLEN_128))
			return 1;
		break;
	case 24:
		if (!(keylen_flag & AES_KEYLEN_192))
			return 1;
		break;
	case 32:
		if (!(keylen_flag & AES_KEYLEN_256))
			return 1;
		break;
	default:
		return -1;
		break;
	}
	return 0;
}

static int setkey_fallback_cip(struct crypto_tfm *tfm, const u8 *in_key,
		unsigned int key_len)
{
	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	int ret;

	sctx->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
	sctx->fallback.blk->base.crt_flags |= (tfm->crt_flags &
			CRYPTO_TFM_REQ_MASK);

	ret = crypto_cipher_setkey(sctx->fallback.cip, in_key, key_len);
	if (ret) {
		tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
		tfm->crt_flags |= (sctx->fallback.blk->base.crt_flags &
				CRYPTO_TFM_RES_MASK);
	}
	return ret;
}

static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
		       unsigned int key_len)
{
	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	u32 *flags = &tfm->crt_flags;
	int ret;

	ret = need_fallback(key_len);
	if (ret < 0) {
		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
		return -EINVAL;
	}

	sctx->key_len = key_len;
	if (!ret) {
		memcpy(sctx->key, in_key, key_len);
		return 0;
	}

	return setkey_fallback_cip(tfm, in_key, key_len);
}

static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
	const struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);

	if (unlikely(need_fallback(sctx->key_len))) {
		crypto_cipher_encrypt_one(sctx->fallback.cip, out, in);
		return;
	}

	switch (sctx->key_len) {
	case 16:
		crypt_s390_km(KM_AES_128_ENCRYPT, &sctx->key, out, in,
			      AES_BLOCK_SIZE);
		break;
	case 24:
		crypt_s390_km(KM_AES_192_ENCRYPT, &sctx->key, out, in,
			      AES_BLOCK_SIZE);
		break;
	case 32:
		crypt_s390_km(KM_AES_256_ENCRYPT, &sctx->key, out, in,
			      AES_BLOCK_SIZE);
		break;
	}
}

static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
	const struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);

	if (unlikely(need_fallback(sctx->key_len))) {
		crypto_cipher_decrypt_one(sctx->fallback.cip, out, in);
		return;
	}

	switch (sctx->key_len) {
	case 16:
		crypt_s390_km(KM_AES_128_DECRYPT, &sctx->key, out, in,
			      AES_BLOCK_SIZE);
		break;
	case 24:
		crypt_s390_km(KM_AES_192_DECRYPT, &sctx->key, out, in,
			      AES_BLOCK_SIZE);
		break;
	case 32:
		crypt_s390_km(KM_AES_256_DECRYPT, &sctx->key, out, in,
			      AES_BLOCK_SIZE);
		break;
	}
}

static int fallback_init_cip(struct crypto_tfm *tfm)
{
	const char *name = tfm->__crt_alg->cra_name;
	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);

	sctx->fallback.cip = crypto_alloc_cipher(name, 0,
			CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);

	if (IS_ERR(sctx->fallback.cip)) {
		pr_err("Allocating AES fallback algorithm %s failed\n",
		       name);
		return PTR_ERR(sctx->fallback.cip);
	}

	return 0;
}

static void fallback_exit_cip(struct crypto_tfm *tfm)
{
	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);

	crypto_free_cipher(sctx->fallback.cip);
	sctx->fallback.cip = NULL;
}

static struct crypto_alg aes_alg = {
	.cra_name		=	"aes",
	.cra_driver_name	=	"aes-s390",
	.cra_priority		=	CRYPT_S390_PRIORITY,
	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER |
					CRYPTO_ALG_NEED_FALLBACK,
	.cra_blocksize		=	AES_BLOCK_SIZE,
	.cra_ctxsize		=	sizeof(struct s390_aes_ctx),
	.cra_module		=	THIS_MODULE,
	.cra_list		=	LIST_HEAD_INIT(aes_alg.cra_list),
	.cra_init               =       fallback_init_cip,
	.cra_exit               =       fallback_exit_cip,
	.cra_u			=	{
		.cipher = {
			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
			.cia_setkey		=	aes_set_key,
			.cia_encrypt		=	aes_encrypt,
			.cia_decrypt		=	aes_decrypt,
		}
	}
};

static int setkey_fallback_blk(struct crypto_tfm *tfm, const u8 *key,
		unsigned int len)
{
	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	unsigned int ret;

	sctx->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
	sctx->fallback.blk->base.crt_flags |= (tfm->crt_flags &
			CRYPTO_TFM_REQ_MASK);

	ret = crypto_blkcipher_setkey(sctx->fallback.blk, key, len);
	if (ret) {
		tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
		tfm->crt_flags |= (sctx->fallback.blk->base.crt_flags &
				CRYPTO_TFM_RES_MASK);
	}
	return ret;
}

static int fallback_blk_dec(struct blkcipher_desc *desc,
		struct scatterlist *dst, struct scatterlist *src,
		unsigned int nbytes)
{
	unsigned int ret;
	struct crypto_blkcipher *tfm;
	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);

	tfm = desc->tfm;
	desc->tfm = sctx->fallback.blk;

	ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes);

	desc->tfm = tfm;
	return ret;
}

static int fallback_blk_enc(struct blkcipher_desc *desc,
		struct scatterlist *dst, struct scatterlist *src,
		unsigned int nbytes)
{
	unsigned int ret;
	struct crypto_blkcipher *tfm;
	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);

	tfm = desc->tfm;
	desc->tfm = sctx->fallback.blk;

	ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes);

	desc->tfm = tfm;
	return ret;
}

static int ecb_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
			   unsigned int key_len)
{
	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	int ret;

	ret = need_fallback(key_len);
	if (ret > 0) {
		sctx->key_len = key_len;
		return setkey_fallback_blk(tfm, in_key, key_len);
	}

	switch (key_len) {
	case 16:
		sctx->enc = KM_AES_128_ENCRYPT;
		sctx->dec = KM_AES_128_DECRYPT;
		break;
	case 24:
		sctx->enc = KM_AES_192_ENCRYPT;
		sctx->dec = KM_AES_192_DECRYPT;
		break;
	case 32:
		sctx->enc = KM_AES_256_ENCRYPT;
		sctx->dec = KM_AES_256_DECRYPT;
		break;
	}

	return aes_set_key(tfm, in_key, key_len);
}

static int ecb_aes_crypt(struct blkcipher_desc *desc, long func, void *param,
			 struct blkcipher_walk *walk)
{
	int ret = blkcipher_walk_virt(desc, walk);
	unsigned int nbytes;

	while ((nbytes = walk->nbytes)) {
		/* only use complete blocks */
		unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1);
		u8 *out = walk->dst.virt.addr;
		u8 *in = walk->src.virt.addr;

		ret = crypt_s390_km(func, param, out, in, n);
		BUG_ON((ret < 0) || (ret != n));

		nbytes &= AES_BLOCK_SIZE - 1;
		ret = blkcipher_walk_done(desc, walk, nbytes);
	}

	return ret;
}

static int ecb_aes_encrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
	struct blkcipher_walk walk;

	if (unlikely(need_fallback(sctx->key_len)))
		return fallback_blk_enc(desc, dst, src, nbytes);

	blkcipher_walk_init(&walk, dst, src, nbytes);
	return ecb_aes_crypt(desc, sctx->enc, sctx->key, &walk);
}

static int ecb_aes_decrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
	struct blkcipher_walk walk;

	if (unlikely(need_fallback(sctx->key_len)))
		return fallback_blk_dec(desc, dst, src, nbytes);

	blkcipher_walk_init(&walk, dst, src, nbytes);
	return ecb_aes_crypt(desc, sctx->dec, sctx->key, &walk);
}

static int fallback_init_blk(struct crypto_tfm *tfm)
{
	const char *name = tfm->__crt_alg->cra_name;
	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);

	sctx->fallback.blk = crypto_alloc_blkcipher(name, 0,
			CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);

	if (IS_ERR(sctx->fallback.blk)) {
		pr_err("Allocating AES fallback algorithm %s failed\n",
		       name);
		return PTR_ERR(sctx->fallback.blk);
	}

	return 0;
}

static void fallback_exit_blk(struct crypto_tfm *tfm)
{
	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);

	crypto_free_blkcipher(sctx->fallback.blk);
	sctx->fallback.blk = NULL;
}

static struct crypto_alg ecb_aes_alg = {
	.cra_name		=	"ecb(aes)",
	.cra_driver_name	=	"ecb-aes-s390",
	.cra_priority		=	CRYPT_S390_COMPOSITE_PRIORITY,
	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER |
					CRYPTO_ALG_NEED_FALLBACK,
	.cra_blocksize		=	AES_BLOCK_SIZE,
	.cra_ctxsize		=	sizeof(struct s390_aes_ctx),
	.cra_type		=	&crypto_blkcipher_type,
	.cra_module		=	THIS_MODULE,
	.cra_list		=	LIST_HEAD_INIT(ecb_aes_alg.cra_list),
	.cra_init		=	fallback_init_blk,
	.cra_exit		=	fallback_exit_blk,
	.cra_u			=	{
		.blkcipher = {
			.min_keysize		=	AES_MIN_KEY_SIZE,
			.max_keysize		=	AES_MAX_KEY_SIZE,
			.setkey			=	ecb_aes_set_key,
			.encrypt		=	ecb_aes_encrypt,
			.decrypt		=	ecb_aes_decrypt,
		}
	}
};

static int cbc_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
			   unsigned int key_len)
{
	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	int ret;

	ret = need_fallback(key_len);
	if (ret > 0) {
		sctx->key_len = key_len;
		return setkey_fallback_blk(tfm, in_key, key_len);
	}

	switch (key_len) {
	case 16:
		sctx->enc = KMC_AES_128_ENCRYPT;
		sctx->dec = KMC_AES_128_DECRYPT;
		break;
	case 24:
		sctx->enc = KMC_AES_192_ENCRYPT;
		sctx->dec = KMC_AES_192_DECRYPT;
		break;
	case 32:
		sctx->enc = KMC_AES_256_ENCRYPT;
		sctx->dec = KMC_AES_256_DECRYPT;
		break;
	}

	return aes_set_key(tfm, in_key, key_len);
}

static int cbc_aes_crypt(struct blkcipher_desc *desc, long func, void *param,
			 struct blkcipher_walk *walk)
{
	int ret = blkcipher_walk_virt(desc, walk);
	unsigned int nbytes = walk->nbytes;

	if (!nbytes)
		goto out;

	memcpy(param, walk->iv, AES_BLOCK_SIZE);
	do {
		/* only use complete blocks */
		unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1);
		u8 *out = walk->dst.virt.addr;
		u8 *in = walk->src.virt.addr;

		ret = crypt_s390_kmc(func, param, out, in, n);
		BUG_ON((ret < 0) || (ret != n));

		nbytes &= AES_BLOCK_SIZE - 1;
		ret = blkcipher_walk_done(desc, walk, nbytes);
	} while ((nbytes = walk->nbytes));
	memcpy(walk->iv, param, AES_BLOCK_SIZE);

out:
	return ret;
}

static int cbc_aes_encrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
	struct blkcipher_walk walk;

	if (unlikely(need_fallback(sctx->key_len)))
		return fallback_blk_enc(desc, dst, src, nbytes);

	blkcipher_walk_init(&walk, dst, src, nbytes);
	return cbc_aes_crypt(desc, sctx->enc, sctx->iv, &walk);
}

static int cbc_aes_decrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
	struct blkcipher_walk walk;

	if (unlikely(need_fallback(sctx->key_len)))
		return fallback_blk_dec(desc, dst, src, nbytes);

	blkcipher_walk_init(&walk, dst, src, nbytes);
	return cbc_aes_crypt(desc, sctx->dec, sctx->iv, &walk);
}

static struct crypto_alg cbc_aes_alg = {
	.cra_name		=	"cbc(aes)",
	.cra_driver_name	=	"cbc-aes-s390",
	.cra_priority		=	CRYPT_S390_COMPOSITE_PRIORITY,
	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER |
					CRYPTO_ALG_NEED_FALLBACK,
	.cra_blocksize		=	AES_BLOCK_SIZE,
	.cra_ctxsize		=	sizeof(struct s390_aes_ctx),
	.cra_type		=	&crypto_blkcipher_type,
	.cra_module		=	THIS_MODULE,
	.cra_list		=	LIST_HEAD_INIT(cbc_aes_alg.cra_list),
	.cra_init		=	fallback_init_blk,
	.cra_exit		=	fallback_exit_blk,
	.cra_u			=	{
		.blkcipher = {
			.min_keysize		=	AES_MIN_KEY_SIZE,
			.max_keysize		=	AES_MAX_KEY_SIZE,
			.ivsize			=	AES_BLOCK_SIZE,
			.setkey			=	cbc_aes_set_key,
			.encrypt		=	cbc_aes_encrypt,
			.decrypt		=	cbc_aes_decrypt,
		}
	}
};

static int __init aes_s390_init(void)
{
	int ret;

	if (crypt_s390_func_available(KM_AES_128_ENCRYPT))
		keylen_flag |= AES_KEYLEN_128;
	if (crypt_s390_func_available(KM_AES_192_ENCRYPT))
		keylen_flag |= AES_KEYLEN_192;
	if (crypt_s390_func_available(KM_AES_256_ENCRYPT))
		keylen_flag |= AES_KEYLEN_256;

	if (!keylen_flag)
		return -EOPNOTSUPP;

	/* z9 109 and z9 BC/EC only support 128 bit key length */
	if (keylen_flag == AES_KEYLEN_128)
		pr_info("AES hardware acceleration is only available for"
			" 128-bit keys\n");

	ret = crypto_register_alg(&aes_alg);
	if (ret)
		goto aes_err;

	ret = crypto_register_alg(&ecb_aes_alg);
	if (ret)
		goto ecb_aes_err;

	ret = crypto_register_alg(&cbc_aes_alg);
	if (ret)
		goto cbc_aes_err;

out:
	return ret;

cbc_aes_err:
	crypto_unregister_alg(&ecb_aes_alg);
ecb_aes_err:
	crypto_unregister_alg(&aes_alg);
aes_err:
	goto out;
}

static void __exit aes_s390_fini(void)
{
	crypto_unregister_alg(&cbc_aes_alg);
	crypto_unregister_alg(&ecb_aes_alg);
	crypto_unregister_alg(&aes_alg);
}

module_init(aes_s390_init);
module_exit(aes_s390_fini);

MODULE_ALIAS("aes-all");

MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm");
MODULE_LICENSE("GPL");
Commit	Line	Data
bf754ae8 JG	1	/*
	2	* Cryptographic API.
	3	*
	4	* s390 implementation of the AES Cipher Algorithm.
	5	*
	6	* s390 Version:
86aa9fc2	7	* Copyright IBM Corp. 2005,2007
bf754ae8	8	* Author(s): Jan Glauber (jang@de.ibm.com)
b0c3e75d	9	* Sebastian Siewior (sebastian@breakpoint.cc> SW-Fallback
bf754ae8	10	*
f8246af0	11	* Derived from "crypto/aes_generic.c"
bf754ae8 JG	12	*
	13	* This program is free software; you can redistribute it and/or modify it
	14	* under the terms of the GNU General Public License as published by the Free
	15	* Software Foundation; either version 2 of the License, or (at your option)
	16	* any later version.
	17	*
	18	*/
	19
39f09392 JG	20	#define KMSG_COMPONENT "aes_s390"
	21	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
	22
89e12654	23	#include <crypto/aes.h>
a9e62fad	24	#include <crypto/algapi.h>
b0c3e75d	25	#include <linux/err.h>
bf754ae8 JG	26	#include <linux/module.h>
bf754ae8 JG	27	#include <linux/init.h>
bf754ae8 JG	28	#include "crypt_s390.h"
bf754ae8 JG	29
86aa9fc2 JG	30	#define AES_KEYLEN_128 1
	31	#define AES_KEYLEN_192 2
	32	#define AES_KEYLEN_256 4
	33
	34	static char keylen_flag = 0;
bf754ae8 JG	35
	36	struct s390_aes_ctx {
	37	u8 iv[AES_BLOCK_SIZE];
	38	u8 key[AES_MAX_KEY_SIZE];
a9e62fad HX	39	long enc;
a9e62fad HX	40	long dec;
bf754ae8	41	int key_len;
b0c3e75d SS	42	union {
	43	struct crypto_blkcipher *blk;
	44	struct crypto_cipher *cip;
	45	} fallback;
bf754ae8 JG	46	};
bf754ae8 JG	47
b0c3e75d SS	48	/*
	49	* Check if the key_len is supported by the HW.
	50	* Returns 0 if it is, a positive number if it is not and software fallback is
	51	* required or a negative number in case the key size is not valid
	52	*/
	53	static int need_fallback(unsigned int key_len)
bf754ae8	54	{
bf754ae8 JG	55	switch (key_len) {
bf754ae8 JG	56	case 16:
86aa9fc2	57	if (!(keylen_flag & AES_KEYLEN_128))
b0c3e75d	58	return 1;
bf754ae8 JG	59	break;
bf754ae8 JG	60	case 24:
86aa9fc2	61	if (!(keylen_flag & AES_KEYLEN_192))
b0c3e75d	62	return 1;
bf754ae8 JG	63	break;
bf754ae8 JG	64	case 32:
86aa9fc2	65	if (!(keylen_flag & AES_KEYLEN_256))
b0c3e75d	66	return 1;
bf754ae8 JG	67	break;
bf754ae8 JG	68	default:
b0c3e75d	69	return -1;
bf754ae8 JG	70	break;
bf754ae8 JG	71	}
b0c3e75d SS	72	return 0;
	73	}
	74
	75	static int setkey_fallback_cip(struct crypto_tfm tfm, const u8 in_key,
	76	unsigned int key_len)
	77	{
	78	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	79	int ret;
	80
	81	sctx->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
	82	sctx->fallback.blk->base.crt_flags \|= (tfm->crt_flags &
	83	CRYPTO_TFM_REQ_MASK);
	84
	85	ret = crypto_cipher_setkey(sctx->fallback.cip, in_key, key_len);
	86	if (ret) {
	87	tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
	88	tfm->crt_flags \|= (sctx->fallback.blk->base.crt_flags &
	89	CRYPTO_TFM_RES_MASK);
	90	}
	91	return ret;
	92	}
	93
	94	static int aes_set_key(struct crypto_tfm tfm, const u8 in_key,
	95	unsigned int key_len)
	96	{
	97	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	98	u32 *flags = &tfm->crt_flags;
	99	int ret;
	100
	101	ret = need_fallback(key_len);
	102	if (ret < 0) {
	103	*flags \|= CRYPTO_TFM_RES_BAD_KEY_LEN;
	104	return -EINVAL;
	105	}
bf754ae8 JG	106
bf754ae8 JG	107	sctx->key_len = key_len;
b0c3e75d SS	108	if (!ret) {
	109	memcpy(sctx->key, in_key, key_len);
	110	return 0;
	111	}
	112
	113	return setkey_fallback_cip(tfm, in_key, key_len);
bf754ae8 JG	114	}
bf754ae8 JG	115
6c2bb98b	116	static void aes_encrypt(struct crypto_tfm tfm, u8 out, const u8 *in)
bf754ae8	117	{
6c2bb98b	118	const struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
bf754ae8	119
b0c3e75d SS	120	if (unlikely(need_fallback(sctx->key_len))) {
	121	crypto_cipher_encrypt_one(sctx->fallback.cip, out, in);
	122	return;
	123	}
	124
bf754ae8 JG	125	switch (sctx->key_len) {
	126	case 16:
	127	crypt_s390_km(KM_AES_128_ENCRYPT, &sctx->key, out, in,
	128	AES_BLOCK_SIZE);
	129	break;
	130	case 24:
	131	crypt_s390_km(KM_AES_192_ENCRYPT, &sctx->key, out, in,
	132	AES_BLOCK_SIZE);
	133	break;
	134	case 32:
	135	crypt_s390_km(KM_AES_256_ENCRYPT, &sctx->key, out, in,
	136	AES_BLOCK_SIZE);
	137	break;
	138	}
	139	}
	140
6c2bb98b	141	static void aes_decrypt(struct crypto_tfm tfm, u8 out, const u8 *in)
bf754ae8	142	{
6c2bb98b	143	const struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
bf754ae8	144
b0c3e75d SS	145	if (unlikely(need_fallback(sctx->key_len))) {
	146	crypto_cipher_decrypt_one(sctx->fallback.cip, out, in);
	147	return;
	148	}
	149
bf754ae8 JG	150	switch (sctx->key_len) {
	151	case 16:
	152	crypt_s390_km(KM_AES_128_DECRYPT, &sctx->key, out, in,
	153	AES_BLOCK_SIZE);
	154	break;
	155	case 24:
	156	crypt_s390_km(KM_AES_192_DECRYPT, &sctx->key, out, in,
	157	AES_BLOCK_SIZE);
	158	break;
	159	case 32:
	160	crypt_s390_km(KM_AES_256_DECRYPT, &sctx->key, out, in,
	161	AES_BLOCK_SIZE);
	162	break;
	163	}
	164	}
	165
b0c3e75d SS	166	static int fallback_init_cip(struct crypto_tfm *tfm)
	167	{
	168	const char *name = tfm->__crt_alg->cra_name;
	169	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	170
	171	sctx->fallback.cip = crypto_alloc_cipher(name, 0,
	172	CRYPTO_ALG_ASYNC \| CRYPTO_ALG_NEED_FALLBACK);
	173
	174	if (IS_ERR(sctx->fallback.cip)) {
39f09392 JG	175	pr_err("Allocating AES fallback algorithm %s failed\n",
39f09392 JG	176	name);
b59cdcb3	177	return PTR_ERR(sctx->fallback.cip);
b0c3e75d SS	178	}
	179
	180	return 0;
	181	}
	182
	183	static void fallback_exit_cip(struct crypto_tfm *tfm)
	184	{
	185	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	186
	187	crypto_free_cipher(sctx->fallback.cip);
	188	sctx->fallback.cip = NULL;
	189	}
bf754ae8 JG	190
	191	static struct crypto_alg aes_alg = {
	192	.cra_name = "aes",
65b75c36 HX	193	.cra_driver_name = "aes-s390",
65b75c36 HX	194	.cra_priority = CRYPT_S390_PRIORITY,
f67d1369 JG	195	.cra_flags = CRYPTO_ALG_TYPE_CIPHER \|
f67d1369 JG	196	CRYPTO_ALG_NEED_FALLBACK,
bf754ae8 JG	197	.cra_blocksize = AES_BLOCK_SIZE,
	198	.cra_ctxsize = sizeof(struct s390_aes_ctx),
	199	.cra_module = THIS_MODULE,
	200	.cra_list = LIST_HEAD_INIT(aes_alg.cra_list),
b0c3e75d SS	201	.cra_init = fallback_init_cip,
b0c3e75d SS	202	.cra_exit = fallback_exit_cip,
bf754ae8 JG	203	.cra_u = {
	204	.cipher = {
	205	.cia_min_keysize = AES_MIN_KEY_SIZE,
	206	.cia_max_keysize = AES_MAX_KEY_SIZE,
	207	.cia_setkey = aes_set_key,
	208	.cia_encrypt = aes_encrypt,
	209	.cia_decrypt = aes_decrypt,
bf754ae8 JG	210	}
	211	}
	212	};
	213
b0c3e75d SS	214	static int setkey_fallback_blk(struct crypto_tfm tfm, const u8 key,
	215	unsigned int len)
	216	{
	217	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	218	unsigned int ret;
	219
	220	sctx->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
	221	sctx->fallback.blk->base.crt_flags \|= (tfm->crt_flags &
	222	CRYPTO_TFM_REQ_MASK);
	223
	224	ret = crypto_blkcipher_setkey(sctx->fallback.blk, key, len);
	225	if (ret) {
	226	tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
	227	tfm->crt_flags \|= (sctx->fallback.blk->base.crt_flags &
	228	CRYPTO_TFM_RES_MASK);
	229	}
	230	return ret;
	231	}
	232
	233	static int fallback_blk_dec(struct blkcipher_desc *desc,
	234	struct scatterlist dst, struct scatterlist src,
	235	unsigned int nbytes)
	236	{
	237	unsigned int ret;
	238	struct crypto_blkcipher *tfm;
	239	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
	240
b0c3e75d SS	241	tfm = desc->tfm;
	242	desc->tfm = sctx->fallback.blk;
	243
2d74d405	244	ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes);
b0c3e75d SS	245
	246	desc->tfm = tfm;
	247	return ret;
	248	}
	249
	250	static int fallback_blk_enc(struct blkcipher_desc *desc,
	251	struct scatterlist dst, struct scatterlist src,
	252	unsigned int nbytes)
	253	{
	254	unsigned int ret;
	255	struct crypto_blkcipher *tfm;
	256	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
	257
b0c3e75d SS	258	tfm = desc->tfm;
	259	desc->tfm = sctx->fallback.blk;
	260
2d74d405	261	ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes);
b0c3e75d SS	262
	263	desc->tfm = tfm;
	264	return ret;
	265	}
	266
a9e62fad HX	267	static int ecb_aes_set_key(struct crypto_tfm tfm, const u8 in_key,
	268	unsigned int key_len)
	269	{
	270	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
b0c3e75d SS	271	int ret;
	272
	273	ret = need_fallback(key_len);
	274	if (ret > 0) {
	275	sctx->key_len = key_len;
	276	return setkey_fallback_blk(tfm, in_key, key_len);
	277	}
a9e62fad HX	278
	279	switch (key_len) {
	280	case 16:
	281	sctx->enc = KM_AES_128_ENCRYPT;
	282	sctx->dec = KM_AES_128_DECRYPT;
	283	break;
	284	case 24:
	285	sctx->enc = KM_AES_192_ENCRYPT;
	286	sctx->dec = KM_AES_192_DECRYPT;
	287	break;
	288	case 32:
	289	sctx->enc = KM_AES_256_ENCRYPT;
	290	sctx->dec = KM_AES_256_DECRYPT;
	291	break;
	292	}
	293
	294	return aes_set_key(tfm, in_key, key_len);
	295	}
	296
	297	static int ecb_aes_crypt(struct blkcipher_desc desc, long func, void param,
	298	struct blkcipher_walk *walk)
	299	{
	300	int ret = blkcipher_walk_virt(desc, walk);
	301	unsigned int nbytes;
	302
	303	while ((nbytes = walk->nbytes)) {
	304	/* only use complete blocks */
	305	unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1);
	306	u8 *out = walk->dst.virt.addr;
	307	u8 *in = walk->src.virt.addr;
	308
	309	ret = crypt_s390_km(func, param, out, in, n);
	310	BUG_ON((ret < 0) \|\| (ret != n));
	311
	312	nbytes &= AES_BLOCK_SIZE - 1;
	313	ret = blkcipher_walk_done(desc, walk, nbytes);
	314	}
	315
	316	return ret;
	317	}
	318
	319	static int ecb_aes_encrypt(struct blkcipher_desc *desc,
	320	struct scatterlist dst, struct scatterlist src,
	321	unsigned int nbytes)
	322	{
	323	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
	324	struct blkcipher_walk walk;
	325
b0c3e75d SS	326	if (unlikely(need_fallback(sctx->key_len)))
	327	return fallback_blk_enc(desc, dst, src, nbytes);
	328
a9e62fad HX	329	blkcipher_walk_init(&walk, dst, src, nbytes);
	330	return ecb_aes_crypt(desc, sctx->enc, sctx->key, &walk);
	331	}
	332
	333	static int ecb_aes_decrypt(struct blkcipher_desc *desc,
	334	struct scatterlist dst, struct scatterlist src,
	335	unsigned int nbytes)
	336	{
	337	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
	338	struct blkcipher_walk walk;
	339
b0c3e75d SS	340	if (unlikely(need_fallback(sctx->key_len)))
	341	return fallback_blk_dec(desc, dst, src, nbytes);
	342
a9e62fad HX	343	blkcipher_walk_init(&walk, dst, src, nbytes);
	344	return ecb_aes_crypt(desc, sctx->dec, sctx->key, &walk);
	345	}
	346
b0c3e75d SS	347	static int fallback_init_blk(struct crypto_tfm *tfm)
	348	{
	349	const char *name = tfm->__crt_alg->cra_name;
	350	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	351
	352	sctx->fallback.blk = crypto_alloc_blkcipher(name, 0,
	353	CRYPTO_ALG_ASYNC \| CRYPTO_ALG_NEED_FALLBACK);
	354
	355	if (IS_ERR(sctx->fallback.blk)) {
39f09392 JG	356	pr_err("Allocating AES fallback algorithm %s failed\n",
39f09392 JG	357	name);
b0c3e75d SS	358	return PTR_ERR(sctx->fallback.blk);
	359	}
	360
	361	return 0;
	362	}
	363
	364	static void fallback_exit_blk(struct crypto_tfm *tfm)
	365	{
	366	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
	367
	368	crypto_free_blkcipher(sctx->fallback.blk);
	369	sctx->fallback.blk = NULL;
	370	}
	371
a9e62fad HX	372	static struct crypto_alg ecb_aes_alg = {
	373	.cra_name = "ecb(aes)",
	374	.cra_driver_name = "ecb-aes-s390",
	375	.cra_priority = CRYPT_S390_COMPOSITE_PRIORITY,
f67d1369 JG	376	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
f67d1369 JG	377	CRYPTO_ALG_NEED_FALLBACK,
a9e62fad HX	378	.cra_blocksize = AES_BLOCK_SIZE,
	379	.cra_ctxsize = sizeof(struct s390_aes_ctx),
	380	.cra_type = &crypto_blkcipher_type,
	381	.cra_module = THIS_MODULE,
	382	.cra_list = LIST_HEAD_INIT(ecb_aes_alg.cra_list),
b0c3e75d SS	383	.cra_init = fallback_init_blk,
b0c3e75d SS	384	.cra_exit = fallback_exit_blk,
a9e62fad HX	385	.cra_u = {
	386	.blkcipher = {
	387	.min_keysize = AES_MIN_KEY_SIZE,
	388	.max_keysize = AES_MAX_KEY_SIZE,
	389	.setkey = ecb_aes_set_key,
	390	.encrypt = ecb_aes_encrypt,
	391	.decrypt = ecb_aes_decrypt,
	392	}
	393	}
	394	};
	395
	396	static int cbc_aes_set_key(struct crypto_tfm tfm, const u8 in_key,
	397	unsigned int key_len)
	398	{
	399	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
b0c3e75d SS	400	int ret;
	401
	402	ret = need_fallback(key_len);
	403	if (ret > 0) {
	404	sctx->key_len = key_len;
	405	return setkey_fallback_blk(tfm, in_key, key_len);
	406	}
a9e62fad HX	407
	408	switch (key_len) {
	409	case 16:
	410	sctx->enc = KMC_AES_128_ENCRYPT;
	411	sctx->dec = KMC_AES_128_DECRYPT;
	412	break;
	413	case 24:
	414	sctx->enc = KMC_AES_192_ENCRYPT;
	415	sctx->dec = KMC_AES_192_DECRYPT;
	416	break;
	417	case 32:
	418	sctx->enc = KMC_AES_256_ENCRYPT;
	419	sctx->dec = KMC_AES_256_DECRYPT;
	420	break;
	421	}
	422
	423	return aes_set_key(tfm, in_key, key_len);
	424	}
	425
	426	static int cbc_aes_crypt(struct blkcipher_desc desc, long func, void param,
	427	struct blkcipher_walk *walk)
	428	{
	429	int ret = blkcipher_walk_virt(desc, walk);
	430	unsigned int nbytes = walk->nbytes;
	431
	432	if (!nbytes)
	433	goto out;
	434
	435	memcpy(param, walk->iv, AES_BLOCK_SIZE);
	436	do {
	437	/* only use complete blocks */
	438	unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1);
	439	u8 *out = walk->dst.virt.addr;
	440	u8 *in = walk->src.virt.addr;
	441
	442	ret = crypt_s390_kmc(func, param, out, in, n);
	443	BUG_ON((ret < 0) \|\| (ret != n));
	444
	445	nbytes &= AES_BLOCK_SIZE - 1;
	446	ret = blkcipher_walk_done(desc, walk, nbytes);
	447	} while ((nbytes = walk->nbytes));
	448	memcpy(walk->iv, param, AES_BLOCK_SIZE);
	449
	450	out:
	451	return ret;
	452	}
	453
	454	static int cbc_aes_encrypt(struct blkcipher_desc *desc,
	455	struct scatterlist dst, struct scatterlist src,
	456	unsigned int nbytes)
	457	{
	458	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
	459	struct blkcipher_walk walk;
	460
b0c3e75d SS	461	if (unlikely(need_fallback(sctx->key_len)))
	462	return fallback_blk_enc(desc, dst, src, nbytes);
	463
a9e62fad HX	464	blkcipher_walk_init(&walk, dst, src, nbytes);
	465	return cbc_aes_crypt(desc, sctx->enc, sctx->iv, &walk);
	466	}
	467
	468	static int cbc_aes_decrypt(struct blkcipher_desc *desc,
	469	struct scatterlist dst, struct scatterlist src,
	470	unsigned int nbytes)
	471	{
	472	struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
	473	struct blkcipher_walk walk;
	474
b0c3e75d SS	475	if (unlikely(need_fallback(sctx->key_len)))
	476	return fallback_blk_dec(desc, dst, src, nbytes);
	477
a9e62fad HX	478	blkcipher_walk_init(&walk, dst, src, nbytes);
	479	return cbc_aes_crypt(desc, sctx->dec, sctx->iv, &walk);
	480	}
	481
	482	static struct crypto_alg cbc_aes_alg = {
	483	.cra_name = "cbc(aes)",
	484	.cra_driver_name = "cbc-aes-s390",
	485	.cra_priority = CRYPT_S390_COMPOSITE_PRIORITY,
f67d1369 JG	486	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
f67d1369 JG	487	CRYPTO_ALG_NEED_FALLBACK,
a9e62fad HX	488	.cra_blocksize = AES_BLOCK_SIZE,
	489	.cra_ctxsize = sizeof(struct s390_aes_ctx),
	490	.cra_type = &crypto_blkcipher_type,
	491	.cra_module = THIS_MODULE,
	492	.cra_list = LIST_HEAD_INIT(cbc_aes_alg.cra_list),
b0c3e75d SS	493	.cra_init = fallback_init_blk,
b0c3e75d SS	494	.cra_exit = fallback_exit_blk,
a9e62fad HX	495	.cra_u = {
	496	.blkcipher = {
	497	.min_keysize = AES_MIN_KEY_SIZE,
	498	.max_keysize = AES_MAX_KEY_SIZE,
	499	.ivsize = AES_BLOCK_SIZE,
	500	.setkey = cbc_aes_set_key,
	501	.encrypt = cbc_aes_encrypt,
	502	.decrypt = cbc_aes_decrypt,
	503	}
	504	}
	505	};
	506
9f7819c1	507	static int __init aes_s390_init(void)
bf754ae8 JG	508	{
	509	int ret;
	510
	511	if (crypt_s390_func_available(KM_AES_128_ENCRYPT))
86aa9fc2	512	keylen_flag \|= AES_KEYLEN_128;
bf754ae8	513	if (crypt_s390_func_available(KM_AES_192_ENCRYPT))
86aa9fc2	514	keylen_flag \|= AES_KEYLEN_192;
bf754ae8	515	if (crypt_s390_func_available(KM_AES_256_ENCRYPT))
86aa9fc2 JG	516	keylen_flag \|= AES_KEYLEN_256;
	517
	518	if (!keylen_flag)
	519	return -EOPNOTSUPP;
bf754ae8	520
86aa9fc2	521	/* z9 109 and z9 BC/EC only support 128 bit key length */
b0c3e75d	522	if (keylen_flag == AES_KEYLEN_128)
39f09392 JG	523	pr_info("AES hardware acceleration is only available for"
39f09392 JG	524	" 128-bit keys\n");
bf754ae8 JG	525
bf754ae8 JG	526	ret = crypto_register_alg(&aes_alg);
86aa9fc2	527	if (ret)
a9e62fad	528	goto aes_err;
a9e62fad HX	529
a9e62fad HX	530	ret = crypto_register_alg(&ecb_aes_alg);
86aa9fc2	531	if (ret)
a9e62fad	532	goto ecb_aes_err;
a9e62fad HX	533
a9e62fad HX	534	ret = crypto_register_alg(&cbc_aes_alg);
86aa9fc2	535	if (ret)
a9e62fad	536	goto cbc_aes_err;
a9e62fad HX	537
a9e62fad HX	538	out:
bf754ae8	539	return ret;
a9e62fad HX	540
	541	cbc_aes_err:
	542	crypto_unregister_alg(&ecb_aes_alg);
	543	ecb_aes_err:
	544	crypto_unregister_alg(&aes_alg);
	545	aes_err:
	546	goto out;
bf754ae8 JG	547	}
bf754ae8 JG	548
9f7819c1	549	static void __exit aes_s390_fini(void)
bf754ae8	550	{
a9e62fad HX	551	crypto_unregister_alg(&cbc_aes_alg);
a9e62fad HX	552	crypto_unregister_alg(&ecb_aes_alg);
bf754ae8 JG	553	crypto_unregister_alg(&aes_alg);
	554	}
	555
9f7819c1 HC	556	module_init(aes_s390_init);
9f7819c1 HC	557	module_exit(aes_s390_fini);
bf754ae8	558
a760a665	559	MODULE_ALIAS("aes-all");
bf754ae8 JG	560
	561	MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm");
	562	MODULE_LICENSE("GPL");