]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright(c) 2015 Cavium networks. All rights reserved. | |
5 | * All rights reserved. | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * * Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * * Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * * Neither the name of Cavium networks nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | ||
34 | #ifndef _RTE_CRC_ARM64_H_ | |
35 | #define _RTE_CRC_ARM64_H_ | |
36 | ||
37 | /** | |
38 | * @file | |
39 | * | |
40 | * RTE CRC arm64 Hash | |
41 | */ | |
42 | ||
43 | #ifdef __cplusplus | |
44 | extern "C" { | |
45 | #endif | |
46 | ||
47 | #include <stdint.h> | |
48 | #include <rte_cpuflags.h> | |
49 | #include <rte_branch_prediction.h> | |
50 | #include <rte_common.h> | |
51 | ||
52 | static inline uint32_t | |
53 | crc32c_arm64_u8(uint8_t data, uint32_t init_val) | |
54 | { | |
55 | asm(".arch armv8-a+crc"); | |
56 | __asm__ volatile( | |
57 | "crc32cb %w[crc], %w[crc], %w[value]" | |
58 | : [crc] "+r" (init_val) | |
59 | : [value] "r" (data)); | |
60 | return init_val; | |
61 | } | |
62 | ||
63 | static inline uint32_t | |
64 | crc32c_arm64_u16(uint16_t data, uint32_t init_val) | |
65 | { | |
66 | asm(".arch armv8-a+crc"); | |
67 | __asm__ volatile( | |
68 | "crc32ch %w[crc], %w[crc], %w[value]" | |
69 | : [crc] "+r" (init_val) | |
70 | : [value] "r" (data)); | |
71 | return init_val; | |
72 | } | |
73 | ||
74 | static inline uint32_t | |
75 | crc32c_arm64_u32(uint32_t data, uint32_t init_val) | |
76 | { | |
77 | asm(".arch armv8-a+crc"); | |
78 | __asm__ volatile( | |
79 | "crc32cw %w[crc], %w[crc], %w[value]" | |
80 | : [crc] "+r" (init_val) | |
81 | : [value] "r" (data)); | |
82 | return init_val; | |
83 | } | |
84 | ||
85 | static inline uint32_t | |
86 | crc32c_arm64_u64(uint64_t data, uint32_t init_val) | |
87 | { | |
88 | asm(".arch armv8-a+crc"); | |
89 | __asm__ volatile( | |
90 | "crc32cx %w[crc], %w[crc], %x[value]" | |
91 | : [crc] "+r" (init_val) | |
92 | : [value] "r" (data)); | |
93 | return init_val; | |
94 | } | |
95 | ||
96 | /** | |
97 | * Allow or disallow use of arm64 SIMD instrinsics for CRC32 hash | |
98 | * calculation. | |
99 | * | |
100 | * @param alg | |
101 | * An OR of following flags: | |
102 | * - (CRC32_SW) Don't use arm64 crc intrinsics | |
103 | * - (CRC32_ARM64) Use ARMv8 CRC intrinsic if available | |
104 | * | |
105 | */ | |
106 | static inline void | |
107 | rte_hash_crc_set_alg(uint8_t alg) | |
108 | { | |
109 | switch (alg) { | |
110 | case CRC32_ARM64: | |
111 | if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_CRC32)) | |
112 | alg = CRC32_SW; | |
113 | case CRC32_SW: | |
114 | crc32_alg = alg; | |
115 | default: | |
116 | break; | |
117 | } | |
118 | } | |
119 | ||
120 | /* Setting the best available algorithm */ | |
121 | static inline void __attribute__((constructor)) | |
122 | rte_hash_crc_init_alg(void) | |
123 | { | |
124 | rte_hash_crc_set_alg(CRC32_ARM64); | |
125 | } | |
126 | ||
127 | /** | |
128 | * Use single crc32 instruction to perform a hash on a 1 byte value. | |
129 | * Fall back to software crc32 implementation in case arm64 crc intrinsics is | |
130 | * not supported | |
131 | * | |
132 | * @param data | |
133 | * Data to perform hash on. | |
134 | * @param init_val | |
135 | * Value to initialise hash generator. | |
136 | * @return | |
137 | * 32bit calculated hash value. | |
138 | */ | |
139 | static inline uint32_t | |
140 | rte_hash_crc_1byte(uint8_t data, uint32_t init_val) | |
141 | { | |
142 | if (likely(crc32_alg & CRC32_ARM64)) | |
143 | return crc32c_arm64_u8(data, init_val); | |
144 | ||
145 | return crc32c_1byte(data, init_val); | |
146 | } | |
147 | ||
148 | /** | |
149 | * Use single crc32 instruction to perform a hash on a 2 bytes value. | |
150 | * Fall back to software crc32 implementation in case arm64 crc intrinsics is | |
151 | * not supported | |
152 | * | |
153 | * @param data | |
154 | * Data to perform hash on. | |
155 | * @param init_val | |
156 | * Value to initialise hash generator. | |
157 | * @return | |
158 | * 32bit calculated hash value. | |
159 | */ | |
160 | static inline uint32_t | |
161 | rte_hash_crc_2byte(uint16_t data, uint32_t init_val) | |
162 | { | |
163 | if (likely(crc32_alg & CRC32_ARM64)) | |
164 | return crc32c_arm64_u16(data, init_val); | |
165 | ||
166 | return crc32c_2bytes(data, init_val); | |
167 | } | |
168 | ||
169 | /** | |
170 | * Use single crc32 instruction to perform a hash on a 4 byte value. | |
171 | * Fall back to software crc32 implementation in case arm64 crc intrinsics is | |
172 | * not supported | |
173 | * | |
174 | * @param data | |
175 | * Data to perform hash on. | |
176 | * @param init_val | |
177 | * Value to initialise hash generator. | |
178 | * @return | |
179 | * 32bit calculated hash value. | |
180 | */ | |
181 | static inline uint32_t | |
182 | rte_hash_crc_4byte(uint32_t data, uint32_t init_val) | |
183 | { | |
184 | if (likely(crc32_alg & CRC32_ARM64)) | |
185 | return crc32c_arm64_u32(data, init_val); | |
186 | ||
187 | return crc32c_1word(data, init_val); | |
188 | } | |
189 | ||
190 | /** | |
191 | * Use single crc32 instruction to perform a hash on a 8 byte value. | |
192 | * Fall back to software crc32 implementation in case arm64 crc intrinsics is | |
193 | * not supported | |
194 | * | |
195 | * @param data | |
196 | * Data to perform hash on. | |
197 | * @param init_val | |
198 | * Value to initialise hash generator. | |
199 | * @return | |
200 | * 32bit calculated hash value. | |
201 | */ | |
202 | static inline uint32_t | |
203 | rte_hash_crc_8byte(uint64_t data, uint32_t init_val) | |
204 | { | |
205 | if (likely(crc32_alg == CRC32_ARM64)) | |
206 | return crc32c_arm64_u64(data, init_val); | |
207 | ||
208 | return crc32c_2words(data, init_val); | |
209 | } | |
210 | ||
211 | #ifdef __cplusplus | |
212 | } | |
213 | #endif | |
214 | ||
215 | #endif /* _RTE_CRC_ARM64_H_ */ |