arch/powerpc/crypto/crct10dif-vpmsum_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Calculate a CRC T10-DIF with vpmsum acceleration
   4  *
   5  * Copyright 2017, Daniel Axtens, IBM Corporation.
   6  * [based on crc32c-vpmsum_glue.c]
   7  */
   8
   9 #include <linux/crc-t10dif.h>
  10 #include <crypto/internal/hash.h>
  11 #include <crypto/internal/simd.h>
  12 #include <linux/init.h>
  13 #include <linux/module.h>
  14 #include <linux/string.h>
  15 #include <linux/kernel.h>
  16 #include <linux/cpufeature.h>
  17 #include <asm/simd.h>
  18 #include <asm/switch_to.h>
  19
  20 #define VMX_ALIGN               16
  21 #define VMX_ALIGN_MASK          (VMX_ALIGN-1)
  22
  23 #define VECTOR_BREAKPOINT       64
  24
  25 u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len);
  26
  27 static u16 crct10dif_vpmsum(u16 crci, unsigned char const *p, size_t len)
  28 {
  29         unsigned int prealign;
  30         unsigned int tail;
  31         u32 crc = crci;
  32
  33         if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable())
  34                 return crc_t10dif_generic(crc, p, len);
  35
  36         if ((unsigned long)p & VMX_ALIGN_MASK) {
  37                 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
  38                 crc = crc_t10dif_generic(crc, p, prealign);
  39                 len -= prealign;
  40                 p += prealign;
  41         }
  42
  43         if (len & ~VMX_ALIGN_MASK) {
  44                 crc <<= 16;
  45                 preempt_disable();
  46                 pagefault_disable();
  47                 enable_kernel_altivec();
  48                 crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
  49                 disable_kernel_altivec();
  50                 pagefault_enable();
  51                 preempt_enable();
  52                 crc >>= 16;
  53         }
  54
  55         tail = len & VMX_ALIGN_MASK;
  56         if (tail) {
  57                 p += len & ~VMX_ALIGN_MASK;
  58                 crc = crc_t10dif_generic(crc, p, tail);
  59         }
  60
  61         return crc & 0xffff;
  62 }
  63
  64 static int crct10dif_vpmsum_init(struct shash_desc *desc)
  65 {
  66         u16 *crc = shash_desc_ctx(desc);
  67
  68         *crc = 0;
  69         return 0;
  70 }
  71
  72 static int crct10dif_vpmsum_update(struct shash_desc *desc, const u8 *data,
  73                             unsigned int length)
  74 {
  75         u16 *crc = shash_desc_ctx(desc);
  76
  77         *crc = crct10dif_vpmsum(*crc, data, length);
  78
  79         return 0;
  80 }
  81
  82
  83 static int crct10dif_vpmsum_final(struct shash_desc *desc, u8 *out)
  84 {
  85         u16 *crcp = shash_desc_ctx(desc);
  86
  87         *(u16 *)out = *crcp;
  88         return 0;
  89 }
  90
  91 static struct shash_alg alg = {
  92         .init           = crct10dif_vpmsum_init,
  93         .update         = crct10dif_vpmsum_update,
  94         .final          = crct10dif_vpmsum_final,
  95         .descsize       = CRC_T10DIF_DIGEST_SIZE,
  96         .digestsize     = CRC_T10DIF_DIGEST_SIZE,
  97         .base           = {
  98                 .cra_name               = "crct10dif",
  99                 .cra_driver_name        = "crct10dif-vpmsum",
 100                 .cra_priority           = 200,
 101                 .cra_blocksize          = CRC_T10DIF_BLOCK_SIZE,
 102                 .cra_module             = THIS_MODULE,
 103         }
 104 };
 105
 106 static int __init crct10dif_vpmsum_mod_init(void)
 107 {
 108         if (!cpu_has_feature(CPU_FTR_ARCH_207S))
 109                 return -ENODEV;
 110
 111         return crypto_register_shash(&alg);
 112 }
 113
 114 static void __exit crct10dif_vpmsum_mod_fini(void)
 115 {
 116         crypto_unregister_shash(&alg);
 117 }
 118
 119 module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crct10dif_vpmsum_mod_init);
 120 module_exit(crct10dif_vpmsum_mod_fini);
 121
 122 MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>");
 123 MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions");
 124 MODULE_LICENSE("GPL");
 125 MODULE_ALIAS_CRYPTO("crct10dif");
 126 MODULE_ALIAS_CRYPTO("crct10dif-vpmsum");