[mirror_ubuntu-artful-kernel.git] / arch / x86 / crypto / twofish_glue_3way.c

/*
 * Glue Code for 3-way parallel assembler optimized version of Twofish
 *
 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <asm/processor.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>
#include <crypto/algapi.h>
#include <crypto/twofish.h>
#include <crypto/b128ops.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>

/* regular block cipher functions from twofish_x86_64 module */
asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
				const u8 *src);
asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
				const u8 *src);

/* 3-way parallel cipher functions */
asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
				       const u8 *src, bool xor);
asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src);

static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
					const u8 *src)
{
	__twofish_enc_blk_3way(ctx, dst, src, false);
}

static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
					    const u8 *src)
{
	__twofish_enc_blk_3way(ctx, dst, src, true);
}

static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
		     void (*fn)(struct twofish_ctx *, u8 *, const u8 *),
		     void (*fn_3way)(struct twofish_ctx *, u8 *, const u8 *))
{
	struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	unsigned int bsize = TF_BLOCK_SIZE;
	unsigned int nbytes;
	int err;

	err = blkcipher_walk_virt(desc, walk);

	while ((nbytes = walk->nbytes)) {
		u8 *wsrc = walk->src.virt.addr;
		u8 *wdst = walk->dst.virt.addr;

		/* Process three block batch */
		if (nbytes >= bsize * 3) {
			do {
				fn_3way(ctx, wdst, wsrc);

				wsrc += bsize * 3;
				wdst += bsize * 3;
				nbytes -= bsize * 3;
			} while (nbytes >= bsize * 3);

			if (nbytes < bsize)
				goto done;
		}

		/* Handle leftovers */
		do {
			fn(ctx, wdst, wsrc);

			wsrc += bsize;
			wdst += bsize;
			nbytes -= bsize;
		} while (nbytes >= bsize);

done:
		err = blkcipher_walk_done(desc, walk, nbytes);
	}

	return err;
}

static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct blkcipher_walk walk;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way);
}

static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct blkcipher_walk walk;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way);
}

static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
				  struct blkcipher_walk *walk)
{
	struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	unsigned int bsize = TF_BLOCK_SIZE;
	unsigned int nbytes = walk->nbytes;
	u128 *src = (u128 *)walk->src.virt.addr;
	u128 *dst = (u128 *)walk->dst.virt.addr;
	u128 *iv = (u128 *)walk->iv;

	do {
		u128_xor(dst, src, iv);
		twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
		iv = dst;

		src += 1;
		dst += 1;
		nbytes -= bsize;
	} while (nbytes >= bsize);

	u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
	return nbytes;
}

static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct blkcipher_walk walk;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	while ((nbytes = walk.nbytes)) {
		nbytes = __cbc_encrypt(desc, &walk);
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	return err;
}

static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
				  struct blkcipher_walk *walk)
{
	struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	unsigned int bsize = TF_BLOCK_SIZE;
	unsigned int nbytes = walk->nbytes;
	u128 *src = (u128 *)walk->src.virt.addr;
	u128 *dst = (u128 *)walk->dst.virt.addr;
	u128 ivs[3 - 1];
	u128 last_iv;

	/* Start of the last block. */
	src += nbytes / bsize - 1;
	dst += nbytes / bsize - 1;

	last_iv = *src;

	/* Process three block batch */
	if (nbytes >= bsize * 3) {
		do {
			nbytes -= bsize * (3 - 1);
			src -= 3 - 1;
			dst -= 3 - 1;

			ivs[0] = src[0];
			ivs[1] = src[1];

			twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);

			u128_xor(dst + 1, dst + 1, ivs + 0);
			u128_xor(dst + 2, dst + 2, ivs + 1);

			nbytes -= bsize;
			if (nbytes < bsize)
				goto done;

			u128_xor(dst, dst, src - 1);
			src -= 1;
			dst -= 1;
		} while (nbytes >= bsize * 3);

		if (nbytes < bsize)
			goto done;
	}

	/* Handle leftovers */
	for (;;) {
		twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src);

		nbytes -= bsize;
		if (nbytes < bsize)
			break;

		u128_xor(dst, dst, src - 1);
		src -= 1;
		dst -= 1;
	}

done:
	u128_xor(dst, dst, (u128 *)walk->iv);
	*(u128 *)walk->iv = last_iv;

	return nbytes;
}

static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct blkcipher_walk walk;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	while ((nbytes = walk.nbytes)) {
		nbytes = __cbc_decrypt(desc, &walk);
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	return err;
}

static inline void u128_to_be128(be128 *dst, const u128 *src)
{
	dst->a = cpu_to_be64(src->a);
	dst->b = cpu_to_be64(src->b);
}

static inline void be128_to_u128(u128 *dst, const be128 *src)
{
	dst->a = be64_to_cpu(src->a);
	dst->b = be64_to_cpu(src->b);
}

static inline void u128_inc(u128 *i)
{
	i->b++;
	if (!i->b)
		i->a++;
}

static void ctr_crypt_final(struct blkcipher_desc *desc,
			    struct blkcipher_walk *walk)
{
	struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	u8 *ctrblk = walk->iv;
	u8 keystream[TF_BLOCK_SIZE];
	u8 *src = walk->src.virt.addr;
	u8 *dst = walk->dst.virt.addr;
	unsigned int nbytes = walk->nbytes;

	twofish_enc_blk(ctx, keystream, ctrblk);
	crypto_xor(keystream, src, nbytes);
	memcpy(dst, keystream, nbytes);

	crypto_inc(ctrblk, TF_BLOCK_SIZE);
}

static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
				struct blkcipher_walk *walk)
{
	struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	unsigned int bsize = TF_BLOCK_SIZE;
	unsigned int nbytes = walk->nbytes;
	u128 *src = (u128 *)walk->src.virt.addr;
	u128 *dst = (u128 *)walk->dst.virt.addr;
	u128 ctrblk;
	be128 ctrblocks[3];

	be128_to_u128(&ctrblk, (be128 *)walk->iv);

	/* Process three block batch */
	if (nbytes >= bsize * 3) {
		do {
			if (dst != src) {
				dst[0] = src[0];
				dst[1] = src[1];
				dst[2] = src[2];
			}

			/* create ctrblks for parallel encrypt */
			u128_to_be128(&ctrblocks[0], &ctrblk);
			u128_inc(&ctrblk);
			u128_to_be128(&ctrblocks[1], &ctrblk);
			u128_inc(&ctrblk);
			u128_to_be128(&ctrblocks[2], &ctrblk);
			u128_inc(&ctrblk);

			twofish_enc_blk_xor_3way(ctx, (u8 *)dst,
						 (u8 *)ctrblocks);

			src += 3;
			dst += 3;
			nbytes -= bsize * 3;
		} while (nbytes >= bsize * 3);

		if (nbytes < bsize)
			goto done;
	}

	/* Handle leftovers */
	do {
		if (dst != src)
			*dst = *src;

		u128_to_be128(&ctrblocks[0], &ctrblk);
		u128_inc(&ctrblk);

		twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
		u128_xor(dst, dst, (u128 *)ctrblocks);

		src += 1;
		dst += 1;
		nbytes -= bsize;
	} while (nbytes >= bsize);

done:
	u128_to_be128((be128 *)walk->iv, &ctrblk);
	return nbytes;
}

static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		     struct scatterlist *src, unsigned int nbytes)
{
	struct blkcipher_walk walk;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE);

	while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) {
		nbytes = __ctr_crypt(desc, &walk);
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	if (walk.nbytes) {
		ctr_crypt_final(desc, &walk);
		err = blkcipher_walk_done(desc, &walk, 0);
	}

	return err;
}

static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
{
	const unsigned int bsize = TF_BLOCK_SIZE;
	struct twofish_ctx *ctx = priv;
	int i;

	if (nbytes == 3 * bsize) {
		twofish_enc_blk_3way(ctx, srcdst, srcdst);
		return;
	}

	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
		twofish_enc_blk(ctx, srcdst, srcdst);
}

static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
{
	const unsigned int bsize = TF_BLOCK_SIZE;
	struct twofish_ctx *ctx = priv;
	int i;

	if (nbytes == 3 * bsize) {
		twofish_dec_blk_3way(ctx, srcdst, srcdst);
		return;
	}

	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
		twofish_dec_blk(ctx, srcdst, srcdst);
}

struct twofish_lrw_ctx {
	struct lrw_table_ctx lrw_table;
	struct twofish_ctx twofish_ctx;
};

static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
	int err;

	err = __twofish_setkey(&ctx->twofish_ctx, key, keylen - TF_BLOCK_SIZE,
			       &tfm->crt_flags);
	if (err)
		return err;

	return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE);
}

static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[3];
	struct lrw_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.table_ctx = &ctx->lrw_table,
		.crypt_ctx = &ctx->twofish_ctx,
		.crypt_fn = encrypt_callback,
	};

	return lrw_crypt(desc, dst, src, nbytes, &req);
}

static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[3];
	struct lrw_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.table_ctx = &ctx->lrw_table,
		.crypt_ctx = &ctx->twofish_ctx,
		.crypt_fn = decrypt_callback,
	};

	return lrw_crypt(desc, dst, src, nbytes, &req);
}

static void lrw_exit_tfm(struct crypto_tfm *tfm)
{
	struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);

	lrw_free_table(&ctx->lrw_table);
}

struct twofish_xts_ctx {
	struct twofish_ctx tweak_ctx;
	struct twofish_ctx crypt_ctx;
};

static int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm);
	u32 *flags = &tfm->crt_flags;
	int err;

	/* key consists of keys of equal size concatenated, therefore
	 * the length must be even
	 */
	if (keylen % 2) {
		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
		return -EINVAL;
	}

	/* first half of xts-key is for crypt */
	err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
	if (err)
		return err;

	/* second half of xts-key is for tweak */
	return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
				flags);
}

static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[3];
	struct xts_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.tweak_ctx = &ctx->tweak_ctx,
		.tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
		.crypt_ctx = &ctx->crypt_ctx,
		.crypt_fn = encrypt_callback,
	};

	return xts_crypt(desc, dst, src, nbytes, &req);
}

static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[3];
	struct xts_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.tweak_ctx = &ctx->tweak_ctx,
		.tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
		.crypt_ctx = &ctx->crypt_ctx,
		.crypt_fn = decrypt_callback,
	};

	return xts_crypt(desc, dst, src, nbytes, &req);
}

static struct crypto_alg tf_algs[5] = { {
	.cra_name		= "ecb(twofish)",
	.cra_driver_name	= "ecb-twofish-3way",
	.cra_priority		= 300,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct twofish_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_list		= LIST_HEAD_INIT(tf_algs[0].cra_list),
	.cra_u = {
		.blkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE,
			.setkey		= twofish_setkey,
			.encrypt	= ecb_encrypt,
			.decrypt	= ecb_decrypt,
		},
	},
}, {
	.cra_name		= "cbc(twofish)",
	.cra_driver_name	= "cbc-twofish-3way",
	.cra_priority		= 300,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct twofish_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_list		= LIST_HEAD_INIT(tf_algs[1].cra_list),
	.cra_u = {
		.blkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= twofish_setkey,
			.encrypt	= cbc_encrypt,
			.decrypt	= cbc_decrypt,
		},
	},
}, {
	.cra_name		= "ctr(twofish)",
	.cra_driver_name	= "ctr-twofish-3way",
	.cra_priority		= 300,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= 1,
	.cra_ctxsize		= sizeof(struct twofish_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_list		= LIST_HEAD_INIT(tf_algs[2].cra_list),
	.cra_u = {
		.blkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= twofish_setkey,
			.encrypt	= ctr_crypt,
			.decrypt	= ctr_crypt,
		},
	},
}, {
	.cra_name		= "lrw(twofish)",
	.cra_driver_name	= "lrw-twofish-3way",
	.cra_priority		= 300,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct twofish_lrw_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_list		= LIST_HEAD_INIT(tf_algs[3].cra_list),
	.cra_exit		= lrw_exit_tfm,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE + TF_BLOCK_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE + TF_BLOCK_SIZE,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= lrw_twofish_setkey,
			.encrypt	= lrw_encrypt,
			.decrypt	= lrw_decrypt,
		},
	},
}, {
	.cra_name		= "xts(twofish)",
	.cra_driver_name	= "xts-twofish-3way",
	.cra_priority		= 300,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct twofish_xts_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_list		= LIST_HEAD_INIT(tf_algs[4].cra_list),
	.cra_u = {
		.blkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE * 2,
			.max_keysize	= TF_MAX_KEY_SIZE * 2,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= xts_twofish_setkey,
			.encrypt	= xts_encrypt,
			.decrypt	= xts_decrypt,
		},
	},
} };

static bool is_blacklisted_cpu(void)
{
	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
		return false;

	if (boot_cpu_data.x86 == 0x06 &&
		(boot_cpu_data.x86_model == 0x1c ||
		 boot_cpu_data.x86_model == 0x26 ||
		 boot_cpu_data.x86_model == 0x36)) {
		/*
		 * On Atom, twofish-3way is slower than original assembler
		 * implementation. Twofish-3way trades off some performance in
		 * storing blocks in 64bit registers to allow three blocks to
		 * be processed parallel. Parallel operation then allows gaining
		 * more performance than was trade off, on out-of-order CPUs.
		 * However Atom does not benefit from this parallellism and
		 * should be blacklisted.
		 */
		return true;
	}

	if (boot_cpu_data.x86 == 0x0f) {
		/*
		 * On Pentium 4, twofish-3way is slower than original assembler
		 * implementation because excessive uses of 64bit rotate and
		 * left-shifts (which are really slow on P4) needed to store and
		 * handle 128bit block in two 64bit registers.
		 */
		return true;
	}

	return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init init(void)
{
	if (!force && is_blacklisted_cpu()) {
		printk(KERN_INFO
			"twofish-x86_64-3way: performance on this CPU "
			"would be suboptimal: disabling "
			"twofish-x86_64-3way.\n");
		return -ENODEV;
	}

	return crypto_register_algs(tf_algs, ARRAY_SIZE(tf_algs));
}

static void __exit fini(void)
{
	crypto_unregister_algs(tf_algs, ARRAY_SIZE(tf_algs));
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
MODULE_ALIAS("twofish");
MODULE_ALIAS("twofish-asm");
Commit	Line	Data
8280daad JK	1	/*
	2	* Glue Code for 3-way parallel assembler optimized version of Twofish
	3	*
	4	* Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
	5	*
	6	* CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
	7	* Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
	8	* CTR part based on code (crypto/ctr.c) by:
	9	* (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of the GNU General Public License as published by
	13	* the Free Software Foundation; either version 2 of the License, or
	14	* (at your option) any later version.
	15	*
	16	* This program is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	19	* GNU General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with this program; if not, write to the Free Software
	23	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	24	* USA
	25	*
	26	*/
	27
a522ee85	28	#include <asm/processor.h>
8280daad JK	29	#include <linux/crypto.h>
	30	#include <linux/init.h>
	31	#include <linux/module.h>
	32	#include <linux/types.h>
	33	#include <crypto/algapi.h>
	34	#include <crypto/twofish.h>
	35	#include <crypto/b128ops.h>
81559f9a	36	#include <crypto/lrw.h>
bae6d303	37	#include <crypto/xts.h>
81559f9a	38
8280daad JK	39	/* regular block cipher functions from twofish_x86_64 module */
	40	asmlinkage void twofish_enc_blk(struct twofish_ctx ctx, u8 dst,
	41	const u8 *src);
	42	asmlinkage void twofish_dec_blk(struct twofish_ctx ctx, u8 dst,
	43	const u8 *src);
	44
	45	/* 3-way parallel cipher functions */
	46	asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx ctx, u8 dst,
	47	const u8 *src, bool xor);
	48	asmlinkage void twofish_dec_blk_3way(struct twofish_ctx ctx, u8 dst,
	49	const u8 *src);
	50
	51	static inline void twofish_enc_blk_3way(struct twofish_ctx ctx, u8 dst,
	52	const u8 *src)
	53	{
	54	__twofish_enc_blk_3way(ctx, dst, src, false);
	55	}
	56
	57	static inline void twofish_enc_blk_xor_3way(struct twofish_ctx ctx, u8 dst,
	58	const u8 *src)
	59	{
	60	__twofish_enc_blk_3way(ctx, dst, src, true);
	61	}
	62
	63	static int ecb_crypt(struct blkcipher_desc desc, struct blkcipher_walk walk,
	64	void (fn)(struct twofish_ctx , u8 , const u8 ),
	65	void (fn_3way)(struct twofish_ctx , u8 , const u8 ))
	66	{
	67	struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	68	unsigned int bsize = TF_BLOCK_SIZE;
	69	unsigned int nbytes;
	70	int err;
	71
	72	err = blkcipher_walk_virt(desc, walk);
	73
	74	while ((nbytes = walk->nbytes)) {
	75	u8 *wsrc = walk->src.virt.addr;
	76	u8 *wdst = walk->dst.virt.addr;
	77
	78	/* Process three block batch */
	79	if (nbytes >= bsize * 3) {
	80	do {
	81	fn_3way(ctx, wdst, wsrc);
	82
	83	wsrc += bsize * 3;
	84	wdst += bsize * 3;
	85	nbytes -= bsize * 3;
	86	} while (nbytes >= bsize * 3);
	87
	88	if (nbytes < bsize)
	89	goto done;
	90	}
	91
	92	/* Handle leftovers */
	93	do {
	94	fn(ctx, wdst, wsrc);
	95
	96	wsrc += bsize;
	97	wdst += bsize;
	98	nbytes -= bsize;
	99	} while (nbytes >= bsize);
	100
	101	done:
	102	err = blkcipher_walk_done(desc, walk, nbytes);
103	}
104
105	return err;
106	}
107
108	static int ecb_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
109	struct scatterlist *src, unsigned int nbytes)
110	{
111	struct blkcipher_walk walk;
112
113	blkcipher_walk_init(&walk, dst, src, nbytes);
114	return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way);
115	}
116
117	static int ecb_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
118	struct scatterlist *src, unsigned int nbytes)
119	{
120	struct blkcipher_walk walk;
121
122	blkcipher_walk_init(&walk, dst, src, nbytes);
123	return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way);
124	}
125
8280daad JK	126	static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
	127	struct blkcipher_walk *walk)
	128	{
	129	struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	130	unsigned int bsize = TF_BLOCK_SIZE;
	131	unsigned int nbytes = walk->nbytes;
	132	u128 src = (u128 )walk->src.virt.addr;
	133	u128 dst = (u128 )walk->dst.virt.addr;
	134	u128 iv = (u128 )walk->iv;
	135
	136	do {
	137	u128_xor(dst, src, iv);
	138	twofish_enc_blk(ctx, (u8 )dst, (u8 )dst);
	139	iv = dst;
	140
	141	src += 1;
	142	dst += 1;
	143	nbytes -= bsize;
	144	} while (nbytes >= bsize);
	145
	146	u128_xor((u128 )walk->iv, (u128 )walk->iv, iv);
	147	return nbytes;
	148	}
	149
	150	static int cbc_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	151	struct scatterlist *src, unsigned int nbytes)
	152	{
	153	struct blkcipher_walk walk;
	154	int err;
	155
	156	blkcipher_walk_init(&walk, dst, src, nbytes);
	157	err = blkcipher_walk_virt(desc, &walk);
	158
	159	while ((nbytes = walk.nbytes)) {
	160	nbytes = __cbc_encrypt(desc, &walk);
	161	err = blkcipher_walk_done(desc, &walk, nbytes);
	162	}
	163
	164	return err;
	165	}
	166
	167	static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
	168	struct blkcipher_walk *walk)
	169	{
	170	struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	171	unsigned int bsize = TF_BLOCK_SIZE;
	172	unsigned int nbytes = walk->nbytes;
	173	u128 src = (u128 )walk->src.virt.addr;
	174	u128 dst = (u128 )walk->dst.virt.addr;
	175	u128 ivs[3 - 1];
	176	u128 last_iv;
	177
	178	/* Start of the last block. */
	179	src += nbytes / bsize - 1;
	180	dst += nbytes / bsize - 1;
	181
	182	last_iv = *src;
	183
	184	/* Process three block batch */
	185	if (nbytes >= bsize * 3) {
	186	do {
	187	nbytes -= bsize * (3 - 1);
	188	src -= 3 - 1;
	189	dst -= 3 - 1;
190
191	ivs[0] = src[0];
192	ivs[1] = src[1];
193
194	twofish_dec_blk_3way(ctx, (u8 )dst, (u8 )src);
195
196	u128_xor(dst + 1, dst + 1, ivs + 0);
197	u128_xor(dst + 2, dst + 2, ivs + 1);
198
199	nbytes -= bsize;
200	if (nbytes < bsize)
201	goto done;
202
203	u128_xor(dst, dst, src - 1);
204	src -= 1;
205	dst -= 1;
206	} while (nbytes >= bsize * 3);
207
208	if (nbytes < bsize)
209	goto done;
210	}
211
212	/* Handle leftovers */
213	for (;;) {
214	twofish_dec_blk(ctx, (u8 )dst, (u8 )src);
215
216	nbytes -= bsize;
217	if (nbytes < bsize)
218	break;
219
220	u128_xor(dst, dst, src - 1);
221	src -= 1;
222	dst -= 1;
223	}
224
225	done:
226	u128_xor(dst, dst, (u128 *)walk->iv);
227	(u128 )walk->iv = last_iv;
228
229	return nbytes;
230	}
231
232	static int cbc_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
233	struct scatterlist *src, unsigned int nbytes)
234	{
235	struct blkcipher_walk walk;
236	int err;
237
238	blkcipher_walk_init(&walk, dst, src, nbytes);
239	err = blkcipher_walk_virt(desc, &walk);
240
241	while ((nbytes = walk.nbytes)) {
242	nbytes = __cbc_decrypt(desc, &walk);
243	err = blkcipher_walk_done(desc, &walk, nbytes);
244	}
245
246	return err;
247	}
248
8280daad JK	249	static inline void u128_to_be128(be128 dst, const u128 src)
	250	{
	251	dst->a = cpu_to_be64(src->a);
	252	dst->b = cpu_to_be64(src->b);
	253	}
	254
	255	static inline void be128_to_u128(u128 dst, const be128 src)
	256	{
	257	dst->a = be64_to_cpu(src->a);
	258	dst->b = be64_to_cpu(src->b);
	259	}
	260
	261	static inline void u128_inc(u128 *i)
	262	{
	263	i->b++;
	264	if (!i->b)
	265	i->a++;
	266	}
	267
	268	static void ctr_crypt_final(struct blkcipher_desc *desc,
	269	struct blkcipher_walk *walk)
	270	{
	271	struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	272	u8 *ctrblk = walk->iv;
	273	u8 keystream[TF_BLOCK_SIZE];
	274	u8 *src = walk->src.virt.addr;
	275	u8 *dst = walk->dst.virt.addr;
	276	unsigned int nbytes = walk->nbytes;
	277
	278	twofish_enc_blk(ctx, keystream, ctrblk);
	279	crypto_xor(keystream, src, nbytes);
	280	memcpy(dst, keystream, nbytes);
	281
	282	crypto_inc(ctrblk, TF_BLOCK_SIZE);
	283	}
	284
	285	static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
	286	struct blkcipher_walk *walk)
	287	{
	288	struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	289	unsigned int bsize = TF_BLOCK_SIZE;
	290	unsigned int nbytes = walk->nbytes;
	291	u128 src = (u128 )walk->src.virt.addr;
	292	u128 dst = (u128 )walk->dst.virt.addr;
	293	u128 ctrblk;
	294	be128 ctrblocks[3];
	295
	296	be128_to_u128(&ctrblk, (be128 *)walk->iv);
	297
	298	/* Process three block batch */
	299	if (nbytes >= bsize * 3) {
	300	do {
	301	if (dst != src) {
	302	dst[0] = src[0];
	303	dst[1] = src[1];
	304	dst[2] = src[2];
	305	}
	306
	307	/* create ctrblks for parallel encrypt */
	308	u128_to_be128(&ctrblocks[0], &ctrblk);
	309	u128_inc(&ctrblk);
	310	u128_to_be128(&ctrblocks[1], &ctrblk);
	311	u128_inc(&ctrblk);
	312	u128_to_be128(&ctrblocks[2], &ctrblk);
313	u128_inc(&ctrblk);
314
315	twofish_enc_blk_xor_3way(ctx, (u8 *)dst,
316	(u8 *)ctrblocks);
317
318	src += 3;
319	dst += 3;
320	nbytes -= bsize * 3;
321	} while (nbytes >= bsize * 3);
322
323	if (nbytes < bsize)
324	goto done;
325	}
326
327	/* Handle leftovers */
328	do {
329	if (dst != src)
330	dst = src;
331
332	u128_to_be128(&ctrblocks[0], &ctrblk);
333	u128_inc(&ctrblk);
334
335	twofish_enc_blk(ctx, (u8 )ctrblocks, (u8 )ctrblocks);
336	u128_xor(dst, dst, (u128 *)ctrblocks);
337
338	src += 1;
339	dst += 1;
340	nbytes -= bsize;
341	} while (nbytes >= bsize);
342
343	done:
344	u128_to_be128((be128 *)walk->iv, &ctrblk);
345	return nbytes;
346	}
347
348	static int ctr_crypt(struct blkcipher_desc desc, struct scatterlist dst,
349	struct scatterlist *src, unsigned int nbytes)
350	{
351	struct blkcipher_walk walk;
352	int err;
353
354	blkcipher_walk_init(&walk, dst, src, nbytes);
355	err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE);
356
357	while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) {
358	nbytes = __ctr_crypt(desc, &walk);
359	err = blkcipher_walk_done(desc, &walk, nbytes);
360	}
361
362	if (walk.nbytes) {
363	ctr_crypt_final(desc, &walk);
364	err = blkcipher_walk_done(desc, &walk, 0);
365	}
366
367	return err;
368	}
369
81559f9a JK	370	static void encrypt_callback(void priv, u8 srcdst, unsigned int nbytes)
	371	{
	372	const unsigned int bsize = TF_BLOCK_SIZE;
	373	struct twofish_ctx *ctx = priv;
	374	int i;
	375
	376	if (nbytes == 3 * bsize) {
	377	twofish_enc_blk_3way(ctx, srcdst, srcdst);
	378	return;
	379	}
	380
	381	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
	382	twofish_enc_blk(ctx, srcdst, srcdst);
	383	}
	384
	385	static void decrypt_callback(void priv, u8 srcdst, unsigned int nbytes)
	386	{
	387	const unsigned int bsize = TF_BLOCK_SIZE;
	388	struct twofish_ctx *ctx = priv;
	389	int i;
	390
	391	if (nbytes == 3 * bsize) {
	392	twofish_dec_blk_3way(ctx, srcdst, srcdst);
	393	return;
	394	}
	395
	396	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
	397	twofish_dec_blk(ctx, srcdst, srcdst);
	398	}
	399
	400	struct twofish_lrw_ctx {
	401	struct lrw_table_ctx lrw_table;
	402	struct twofish_ctx twofish_ctx;
	403	};
	404
	405	static int lrw_twofish_setkey(struct crypto_tfm tfm, const u8 key,
	406	unsigned int keylen)
	407	{
	408	struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
	409	int err;
	410
	411	err = __twofish_setkey(&ctx->twofish_ctx, key, keylen - TF_BLOCK_SIZE,
	412	&tfm->crt_flags);
	413	if (err)
	414	return err;
	415
	416	return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE);
	417	}
	418
	419	static int lrw_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	420	struct scatterlist *src, unsigned int nbytes)
	421	{
	422	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	423	be128 buf[3];
	424	struct lrw_crypt_req req = {
	425	.tbuf = buf,
	426	.tbuflen = sizeof(buf),
	427
	428	.table_ctx = &ctx->lrw_table,
	429	.crypt_ctx = &ctx->twofish_ctx,
	430	.crypt_fn = encrypt_callback,
	431	};
	432
	433	return lrw_crypt(desc, dst, src, nbytes, &req);
434	}
435
436	static int lrw_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
437	struct scatterlist *src, unsigned int nbytes)
438	{
439	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
440	be128 buf[3];
441	struct lrw_crypt_req req = {
442	.tbuf = buf,
443	.tbuflen = sizeof(buf),
444
445	.table_ctx = &ctx->lrw_table,
446	.crypt_ctx = &ctx->twofish_ctx,
447	.crypt_fn = decrypt_callback,
448	};
449
450	return lrw_crypt(desc, dst, src, nbytes, &req);
451	}
452
453	static void lrw_exit_tfm(struct crypto_tfm *tfm)
454	{
455	struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
456
457	lrw_free_table(&ctx->lrw_table);
458	}
459
bae6d303 JK	460	struct twofish_xts_ctx {
	461	struct twofish_ctx tweak_ctx;
	462	struct twofish_ctx crypt_ctx;
	463	};
	464
	465	static int xts_twofish_setkey(struct crypto_tfm tfm, const u8 key,
	466	unsigned int keylen)
	467	{
	468	struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm);
	469	u32 *flags = &tfm->crt_flags;
	470	int err;
	471
	472	/* key consists of keys of equal size concatenated, therefore
	473	* the length must be even
	474	*/
	475	if (keylen % 2) {
	476	*flags \|= CRYPTO_TFM_RES_BAD_KEY_LEN;
	477	return -EINVAL;
	478	}
	479
	480	/* first half of xts-key is for crypt */
	481	err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
	482	if (err)
	483	return err;
	484
	485	/* second half of xts-key is for tweak */
	486	return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
	487	flags);
	488	}
	489
	490	static int xts_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	491	struct scatterlist *src, unsigned int nbytes)
	492	{
	493	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	494	be128 buf[3];
	495	struct xts_crypt_req req = {
	496	.tbuf = buf,
	497	.tbuflen = sizeof(buf),
	498
	499	.tweak_ctx = &ctx->tweak_ctx,
	500	.tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
	501	.crypt_ctx = &ctx->crypt_ctx,
	502	.crypt_fn = encrypt_callback,
	503	};
	504
	505	return xts_crypt(desc, dst, src, nbytes, &req);
	506	}
	507
	508	static int xts_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	509	struct scatterlist *src, unsigned int nbytes)
	510	{
	511	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	512	be128 buf[3];
	513	struct xts_crypt_req req = {
	514	.tbuf = buf,
	515	.tbuflen = sizeof(buf),
	516
	517	.tweak_ctx = &ctx->tweak_ctx,
	518	.tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
	519	.crypt_ctx = &ctx->crypt_ctx,
	520	.crypt_fn = decrypt_callback,
	521	};
	522
	523	return xts_crypt(desc, dst, src, nbytes, &req);
524	}
525
53709dde JK	526	static struct crypto_alg tf_algs[5] = { {
	527	.cra_name = "ecb(twofish)",
	528	.cra_driver_name = "ecb-twofish-3way",
	529	.cra_priority = 300,
	530	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	531	.cra_blocksize = TF_BLOCK_SIZE,
	532	.cra_ctxsize = sizeof(struct twofish_ctx),
	533	.cra_alignmask = 0,
	534	.cra_type = &crypto_blkcipher_type,
	535	.cra_module = THIS_MODULE,
	536	.cra_list = LIST_HEAD_INIT(tf_algs[0].cra_list),
	537	.cra_u = {
	538	.blkcipher = {
	539	.min_keysize = TF_MIN_KEY_SIZE,
	540	.max_keysize = TF_MAX_KEY_SIZE,
	541	.setkey = twofish_setkey,
	542	.encrypt = ecb_encrypt,
	543	.decrypt = ecb_decrypt,
	544	},
	545	},
	546	}, {
	547	.cra_name = "cbc(twofish)",
	548	.cra_driver_name = "cbc-twofish-3way",
	549	.cra_priority = 300,
	550	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	551	.cra_blocksize = TF_BLOCK_SIZE,
	552	.cra_ctxsize = sizeof(struct twofish_ctx),
	553	.cra_alignmask = 0,
	554	.cra_type = &crypto_blkcipher_type,
	555	.cra_module = THIS_MODULE,
	556	.cra_list = LIST_HEAD_INIT(tf_algs[1].cra_list),
	557	.cra_u = {
	558	.blkcipher = {
	559	.min_keysize = TF_MIN_KEY_SIZE,
	560	.max_keysize = TF_MAX_KEY_SIZE,
	561	.ivsize = TF_BLOCK_SIZE,
	562	.setkey = twofish_setkey,
	563	.encrypt = cbc_encrypt,
	564	.decrypt = cbc_decrypt,
	565	},
	566	},
	567	}, {
	568	.cra_name = "ctr(twofish)",
	569	.cra_driver_name = "ctr-twofish-3way",
	570	.cra_priority = 300,
	571	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	572	.cra_blocksize = 1,
	573	.cra_ctxsize = sizeof(struct twofish_ctx),
	574	.cra_alignmask = 0,
	575	.cra_type = &crypto_blkcipher_type,
	576	.cra_module = THIS_MODULE,
	577	.cra_list = LIST_HEAD_INIT(tf_algs[2].cra_list),
	578	.cra_u = {
	579	.blkcipher = {
	580	.min_keysize = TF_MIN_KEY_SIZE,
	581	.max_keysize = TF_MAX_KEY_SIZE,
	582	.ivsize = TF_BLOCK_SIZE,
	583	.setkey = twofish_setkey,
	584	.encrypt = ctr_crypt,
	585	.decrypt = ctr_crypt,
	586	},
	587	},
	588	}, {
	589	.cra_name = "lrw(twofish)",
590	.cra_driver_name = "lrw-twofish-3way",
591	.cra_priority = 300,
592	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
593	.cra_blocksize = TF_BLOCK_SIZE,
594	.cra_ctxsize = sizeof(struct twofish_lrw_ctx),
595	.cra_alignmask = 0,
596	.cra_type = &crypto_blkcipher_type,
597	.cra_module = THIS_MODULE,
598	.cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list),
599	.cra_exit = lrw_exit_tfm,
600	.cra_u = {
601	.blkcipher = {
602	.min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE,
603	.max_keysize = TF_MAX_KEY_SIZE + TF_BLOCK_SIZE,
604	.ivsize = TF_BLOCK_SIZE,
605	.setkey = lrw_twofish_setkey,
606	.encrypt = lrw_encrypt,
607	.decrypt = lrw_decrypt,
608	},
609	},
610	}, {
bae6d303 JK	611	.cra_name = "xts(twofish)",
	612	.cra_driver_name = "xts-twofish-3way",
	613	.cra_priority = 300,
	614	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	615	.cra_blocksize = TF_BLOCK_SIZE,
	616	.cra_ctxsize = sizeof(struct twofish_xts_ctx),
	617	.cra_alignmask = 0,
	618	.cra_type = &crypto_blkcipher_type,
	619	.cra_module = THIS_MODULE,
53709dde	620	.cra_list = LIST_HEAD_INIT(tf_algs[4].cra_list),
bae6d303 JK	621	.cra_u = {
	622	.blkcipher = {
	623	.min_keysize = TF_MIN_KEY_SIZE * 2,
	624	.max_keysize = TF_MAX_KEY_SIZE * 2,
	625	.ivsize = TF_BLOCK_SIZE,
	626	.setkey = xts_twofish_setkey,
	627	.encrypt = xts_encrypt,
	628	.decrypt = xts_decrypt,
	629	},
	630	},
53709dde	631	} };
bae6d303	632
a522ee85 JK	633	static bool is_blacklisted_cpu(void)
	634	{
	635	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
	636	return false;
	637
	638	if (boot_cpu_data.x86 == 0x06 &&
	639	(boot_cpu_data.x86_model == 0x1c \|\|
	640	boot_cpu_data.x86_model == 0x26 \|\|
	641	boot_cpu_data.x86_model == 0x36)) {
	642	/*
	643	* On Atom, twofish-3way is slower than original assembler
	644	* implementation. Twofish-3way trades off some performance in
	645	* storing blocks in 64bit registers to allow three blocks to
	646	* be processed parallel. Parallel operation then allows gaining
	647	* more performance than was trade off, on out-of-order CPUs.
	648	* However Atom does not benefit from this parallellism and
	649	* should be blacklisted.
	650	*/
	651	return true;
	652	}
	653
	654	if (boot_cpu_data.x86 == 0x0f) {
	655	/*
	656	* On Pentium 4, twofish-3way is slower than original assembler
	657	* implementation because excessive uses of 64bit rotate and
	658	* left-shifts (which are really slow on P4) needed to store and
	659	* handle 128bit block in two 64bit registers.
	660	*/
	661	return true;
	662	}
	663
	664	return false;
	665	}
	666
	667	static int force;
	668	module_param(force, int, 0);
	669	MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
	670
ff0a70fe	671	static int __init init(void)
8280daad	672	{
a522ee85 JK	673	if (!force && is_blacklisted_cpu()) {
	674	printk(KERN_INFO
	675	"twofish-x86_64-3way: performance on this CPU "
	676	"would be suboptimal: disabling "
	677	"twofish-x86_64-3way.\n");
	678	return -ENODEV;
	679	}
	680
53709dde	681	return crypto_register_algs(tf_algs, ARRAY_SIZE(tf_algs));
8280daad JK	682	}
8280daad JK	683
ff0a70fe	684	static void __exit fini(void)
8280daad	685	{
53709dde	686	crypto_unregister_algs(tf_algs, ARRAY_SIZE(tf_algs));
8280daad JK	687	}
	688
	689	module_init(init);
	690	module_exit(fini);
	691
	692	MODULE_LICENSE("GPL");
	693	MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
	694	MODULE_ALIAS("twofish");
	695	MODULE_ALIAS("twofish-asm");