]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
7c673cae FG |
4 | * Copyright (c) Intel Corporation. |
5 | * All rights reserved. | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * * Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * * Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * * Neither the name of Intel Corporation nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | ||
11fdf7f2 | 34 | #include "spdk/crc32.h" |
7c673cae | 35 | |
9f95a23c TL |
36 | #if defined(__aarch64__) || defined(__AARCH64__) |
37 | #ifdef __ARM_FEATURE_CRC32 | |
38 | #define SPDK_HAVE_ARM_CRC | |
39 | #include <arm_acle.h> | |
40 | #endif | |
41 | #endif | |
42 | ||
11fdf7f2 | 43 | #if defined(__x86_64__) && defined(__SSE4_2__) |
9f95a23c TL |
44 | #ifdef SPDK_CONFIG_ISAL |
45 | #define SPDK_HAVE_ISAL | |
46 | #include <isa-l/include/crc.h> | |
47 | #else | |
48 | #define SPDK_HAVE_SSE4_2 | |
11fdf7f2 | 49 | #include <x86intrin.h> |
9f95a23c TL |
50 | #endif |
51 | #endif | |
52 | ||
53 | #ifdef SPDK_HAVE_ISAL | |
54 | ||
55 | uint32_t | |
56 | spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) | |
57 | { | |
58 | return crc32_iscsi((unsigned char *)buf, len, crc); | |
59 | } | |
60 | ||
61 | #elif defined(SPDK_HAVE_SSE4_2) | |
7c673cae | 62 | |
11fdf7f2 TL |
63 | uint32_t |
64 | spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) | |
7c673cae | 65 | { |
11fdf7f2 TL |
66 | uint64_t crc_tmp64; |
67 | size_t count; | |
7c673cae | 68 | |
11fdf7f2 TL |
69 | /* _mm_crc32_u64() needs a 64-bit intermediate value */ |
70 | crc_tmp64 = crc; | |
7c673cae | 71 | |
11fdf7f2 TL |
72 | /* Process as much of the buffer as possible in 64-bit blocks. */ |
73 | count = len / 8; | |
74 | while (count--) { | |
75 | uint64_t block; | |
7c673cae | 76 | |
11fdf7f2 TL |
77 | /* |
78 | * Use memcpy() to avoid unaligned loads, which are undefined behavior in C. | |
79 | * The compiler will optimize out the memcpy() in release builds. | |
80 | */ | |
81 | memcpy(&block, buf, sizeof(block)); | |
82 | crc_tmp64 = _mm_crc32_u64(crc_tmp64, block); | |
83 | buf += sizeof(block); | |
84 | } | |
85 | crc = (uint32_t)crc_tmp64; | |
7c673cae | 86 | |
11fdf7f2 TL |
87 | /* Handle any trailing bytes. */ |
88 | count = len & 7; | |
89 | while (count--) { | |
90 | crc = _mm_crc32_u8(crc, *(const uint8_t *)buf); | |
91 | buf++; | |
7c673cae FG |
92 | } |
93 | ||
11fdf7f2 | 94 | return crc; |
7c673cae FG |
95 | } |
96 | ||
9f95a23c TL |
97 | #elif defined(SPDK_HAVE_ARM_CRC) |
98 | ||
99 | uint32_t | |
100 | spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) | |
101 | { | |
102 | size_t count; | |
103 | ||
104 | count = len / 8; | |
105 | while (count--) { | |
106 | uint64_t block; | |
107 | ||
108 | memcpy(&block, buf, sizeof(block)); | |
109 | crc = __crc32cd(crc, block); | |
110 | buf += sizeof(block); | |
111 | } | |
112 | ||
113 | count = len & 7; | |
114 | while (count--) { | |
115 | crc = __crc32cb(crc, *(const uint8_t *)buf); | |
116 | buf++; | |
117 | } | |
118 | ||
119 | return crc; | |
120 | } | |
121 | ||
122 | #else /* Neither SSE 4.2 nor ARM CRC32 instructions available */ | |
7c673cae | 123 | |
11fdf7f2 | 124 | static struct spdk_crc32_table g_crc32c_table; |
7c673cae | 125 | |
11fdf7f2 TL |
126 | __attribute__((constructor)) static void |
127 | spdk_crc32c_init(void) | |
128 | { | |
129 | spdk_crc32_table_init(&g_crc32c_table, SPDK_CRC32C_POLYNOMIAL_REFLECT); | |
130 | } | |
7c673cae | 131 | |
11fdf7f2 TL |
132 | uint32_t |
133 | spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) | |
134 | { | |
135 | return spdk_crc32_update(&g_crc32c_table, buf, len, crc); | |
7c673cae | 136 | } |
11fdf7f2 TL |
137 | |
138 | #endif |