ceph/src/seastar/dpdk/lib/librte_net/rte_net_crc.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <stdint.h>
  37
  38 #include <rte_cpuflags.h>
  39 #include <rte_common.h>
  40 #include <rte_net_crc.h>
  41
  42 #if defined(RTE_ARCH_X86_64)                            \
  43         && defined(RTE_MACHINE_CPUFLAG_SSE4_2)          \
  44         && defined(RTE_MACHINE_CPUFLAG_PCLMULQDQ)
  45 #define X86_64_SSE42_PCLMULQDQ     1
  46 #endif
  47
  48 #ifdef X86_64_SSE42_PCLMULQDQ
  49 #include <net_crc_sse.h>
  50 #endif
  51
  52 /* crc tables */
  53 static uint32_t crc32_eth_lut[CRC_LUT_SIZE];
  54 static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE];
  55
  56 static uint32_t
  57 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len);
  58
  59 static uint32_t
  60 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len);
  61
  62 typedef uint32_t
  63 (*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len);
  64
  65 static rte_net_crc_handler *handlers;
  66
  67 static rte_net_crc_handler handlers_scalar[] = {
  68         [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler,
  69         [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler,
  70 };
  71
  72 #ifdef X86_64_SSE42_PCLMULQDQ
  73 static rte_net_crc_handler handlers_sse42[] = {
  74         [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler,
  75         [RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler,
  76 };
  77 #endif
  78
  79 /**
  80  * Reflect the bits about the middle
  81  *
  82  * @param val
  83  *   value to be reflected
  84  *
  85  * @return
  86  *   reflected value
  87  */
  88 static uint32_t
  89 reflect_32bits(uint32_t val)
  90 {
  91         uint32_t i, res = 0;
  92
  93         for (i = 0; i < 32; i++)
  94                 if ((val & (1 << i)) != 0)
  95                         res |= (uint32_t)(1 << (31 - i));
  96
  97         return res;
  98 }
  99
 100 static void
 101 crc32_eth_init_lut(uint32_t poly,
 102         uint32_t *lut)
 103 {
 104         uint32_t i, j;
 105
 106         for (i = 0; i < CRC_LUT_SIZE; i++) {
 107                 uint32_t crc = reflect_32bits(i);
 108
 109                 for (j = 0; j < 8; j++) {
 110                         if (crc & 0x80000000L)
 111                                 crc = (crc << 1) ^ poly;
 112                         else
 113                                 crc <<= 1;
 114                 }
 115                 lut[i] = reflect_32bits(crc);
 116         }
 117 }
 118
 119 static inline __attribute__((always_inline)) uint32_t
 120 crc32_eth_calc_lut(const uint8_t *data,
 121         uint32_t data_len,
 122         uint32_t crc,
 123         const uint32_t *lut)
 124 {
 125         while (data_len--)
 126                 crc = lut[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
 127
 128         return crc;
 129 }
 130
 131 static void
 132 rte_net_crc_scalar_init(void)
 133 {
 134         /* 32-bit crc init */
 135         crc32_eth_init_lut(CRC32_ETH_POLYNOMIAL, crc32_eth_lut);
 136
 137         /* 16-bit CRC init */
 138         crc32_eth_init_lut(CRC16_CCITT_POLYNOMIAL << 16, crc16_ccitt_lut);
 139 }
 140
 141 static inline uint32_t
 142 rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len)
 143 {
 144         /* return 16-bit CRC value */
 145         return (uint16_t)~crc32_eth_calc_lut(data,
 146                 data_len,
 147                 0xffff,
 148                 crc16_ccitt_lut);
 149 }
 150
 151 static inline uint32_t
 152 rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len)
 153 {
 154         /* return 32-bit CRC value */
 155         return ~crc32_eth_calc_lut(data,
 156                 data_len,
 157                 0xffffffffUL,
 158                 crc32_eth_lut);
 159 }
 160
 161 void
 162 rte_net_crc_set_alg(enum rte_net_crc_alg alg)
 163 {
 164         switch (alg) {
 165         case RTE_NET_CRC_SSE42:
 166 #ifdef X86_64_SSE42_PCLMULQDQ
 167                 handlers = handlers_sse42;
 168 #else
 169                 alg = RTE_NET_CRC_SCALAR;
 170 #endif
 171                 break;
 172         case RTE_NET_CRC_SCALAR:
 173         default:
 174                 handlers = handlers_scalar;
 175                 break;
 176         }
 177 }
 178
 179 uint32_t
 180 rte_net_crc_calc(const void *data,
 181         uint32_t data_len,
 182         enum rte_net_crc_type type)
 183 {
 184         uint32_t ret;
 185         rte_net_crc_handler f_handle;
 186
 187         f_handle = handlers[type];
 188         ret = f_handle(data, data_len);
 189
 190         return ret;
 191 }
 192
 193 /* Select highest available crc algorithm as default one */
 194 static inline void __attribute__((constructor))
 195 rte_net_crc_init(void)
 196 {
 197         enum rte_net_crc_alg alg = RTE_NET_CRC_SCALAR;
 198
 199         rte_net_crc_scalar_init();
 200
 201 #ifdef X86_64_SSE42_PCLMULQDQ
 202                 alg = RTE_NET_CRC_SSE42;
 203                 rte_net_crc_sse42_init();
 204 #endif
 205
 206         rte_net_crc_set_alg(alg);
 207 }