4 * Copyright (C) Cavium networks Ltd. 2015.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of Cavium networks nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 struct _neon_acl_const
{
37 rte_xmm_t xmm_shuffle_input
;
38 rte_xmm_t xmm_index_mask
;
40 } neon_acl_const
__attribute__((aligned(RTE_CACHE_LINE_SIZE
))) = {
42 .u32
= {0x00000000, 0x04040404, 0x08080808, 0x0c0c0c0c}
45 .u32
= {RTE_ACL_NODE_INDEX
, RTE_ACL_NODE_INDEX
,
46 RTE_ACL_NODE_INDEX
, RTE_ACL_NODE_INDEX
}
49 .u32
= {0xffffff00, 0xffffff04, 0xffffff08, 0xffffff0c}
54 * Resolve priority for multiple results (neon version).
55 * This consists comparing the priority of the current traversal with the
56 * running set of results for the packet.
57 * For each result, keep a running array of the result (rule number) and
58 * its priority for each category.
61 resolve_priority_neon(uint64_t transition
, int n
, const struct rte_acl_ctx
*ctx
,
63 const struct rte_acl_match_results
*p
,
67 int32x4_t results
, priority
, results1
, priority1
;
69 int32_t *saved_results
, *saved_priority
;
71 for (x
= 0; x
< categories
; x
+= RTE_ACL_RESULTS_MULTIPLIER
) {
72 saved_results
= (int32_t *)(&parms
[n
].cmplt
->results
[x
]);
73 saved_priority
= (int32_t *)(&parms
[n
].cmplt
->priority
[x
]);
75 /* get results and priorities for completed trie */
77 (const int32_t *)&p
[transition
].results
[x
]);
79 (const int32_t *)&p
[transition
].priority
[x
]);
81 /* if this is not the first completed trie */
82 if (parms
[n
].cmplt
->count
!= ctx
->num_tries
) {
83 /* get running best results and their priorities */
84 results1
= vld1q_s32(saved_results
);
85 priority1
= vld1q_s32(saved_priority
);
87 /* select results that are highest priority */
88 selector
= vcgtq_s32(priority1
, priority
);
89 results
= vbslq_s32(selector
, results1
, results
);
90 priority
= vbslq_s32(selector
, priority1
, priority
);
93 /* save running best results and their priorities */
94 vst1q_s32(saved_results
, results
);
95 vst1q_s32(saved_priority
, priority
);
100 * Check for any match in 4 transitions
102 static inline __attribute__((always_inline
)) uint32_t
103 check_any_match_x4(uint64_t val
[])
105 return (val
[0] | val
[1] | val
[2] | val
[3]) & RTE_ACL_NODE_MATCH
;
108 static inline __attribute__((always_inline
)) void
109 acl_match_check_x4(int slot
, const struct rte_acl_ctx
*ctx
, struct parms
*parms
,
110 struct acl_flow_data
*flows
, uint64_t transitions
[])
112 while (check_any_match_x4(transitions
)) {
113 transitions
[0] = acl_match_check(transitions
[0], slot
, ctx
,
114 parms
, flows
, resolve_priority_neon
);
115 transitions
[1] = acl_match_check(transitions
[1], slot
+ 1, ctx
,
116 parms
, flows
, resolve_priority_neon
);
117 transitions
[2] = acl_match_check(transitions
[2], slot
+ 2, ctx
,
118 parms
, flows
, resolve_priority_neon
);
119 transitions
[3] = acl_match_check(transitions
[3], slot
+ 3, ctx
,
120 parms
, flows
, resolve_priority_neon
);
125 * Process 4 transitions (in 2 NEON Q registers) in parallel
127 static inline __attribute__((always_inline
)) int32x4_t
128 transition4(int32x4_t next_input
, const uint64_t *trans
, uint64_t transitions
[])
130 int32x4x2_t tr_hi_lo
;
132 uint32x4_t index_msk
, node_type
, addr
;
133 uint32x4_t dfa_msk
, mask
, quad_ofs
, dfa_ofs
;
135 /* Move low 32 into tr_hi_lo.val[0] and high 32 into tr_hi_lo.val[1] */
136 tr_hi_lo
= vld2q_s32((const int32_t *)transitions
);
138 /* Calculate the address (array index) for all 4 transitions. */
140 index_msk
= vld1q_u32((const uint32_t *)&neon_acl_const
.xmm_index_mask
);
142 /* Calc node type and node addr */
143 node_type
= vbicq_s32(tr_hi_lo
.val
[0], index_msk
);
144 addr
= vandq_s32(tr_hi_lo
.val
[0], index_msk
);
147 t
= veorq_s32(node_type
, node_type
);
149 /* mask for DFA type(0) nodes */
150 dfa_msk
= vceqq_u32(node_type
, t
);
152 mask
= vld1q_s32((const int32_t *)&neon_acl_const
.xmm_shuffle_input
);
153 in
= vqtbl1q_u8((uint8x16_t
)next_input
, (uint8x16_t
)mask
);
155 /* DFA calculations. */
156 r
= vshrq_n_u32(in
, 30); /* div by 64 */
157 mask
= vld1q_s32((const int32_t *)&neon_acl_const
.range_base
);
158 r
= vaddq_u8(r
, mask
);
159 t
= vshrq_n_u32(in
, 24);
160 r
= vqtbl1q_u8((uint8x16_t
)tr_hi_lo
.val
[1], (uint8x16_t
)r
);
161 dfa_ofs
= vsubq_s32(t
, r
);
163 /* QUAD/SINGLE calculations. */
164 t
= vcgtq_s8(in
, tr_hi_lo
.val
[1]);
167 quad_ofs
= vpaddlq_u16(t
);
169 /* blend DFA and QUAD/SINGLE. */
170 t
= vbslq_u8(dfa_msk
, dfa_ofs
, quad_ofs
);
172 /* calculate address for next transitions */
173 addr
= vaddq_u32(addr
, t
);
175 /* Fill next transitions */
176 transitions
[0] = trans
[vgetq_lane_u32(addr
, 0)];
177 transitions
[1] = trans
[vgetq_lane_u32(addr
, 1)];
178 transitions
[2] = trans
[vgetq_lane_u32(addr
, 2)];
179 transitions
[3] = trans
[vgetq_lane_u32(addr
, 3)];
181 return vshrq_n_u32(next_input
, CHAR_BIT
);
185 * Execute trie traversal with 8 traversals in parallel
188 search_neon_8(const struct rte_acl_ctx
*ctx
, const uint8_t **data
,
189 uint32_t *results
, uint32_t total_packets
, uint32_t categories
)
192 struct acl_flow_data flows
;
193 uint64_t index_array
[8];
194 struct completion cmplt
[8];
195 struct parms parms
[8];
196 int32x4_t input0
, input1
;
198 acl_set_flow(&flows
, cmplt
, RTE_DIM(cmplt
), data
, results
,
199 total_packets
, categories
, ctx
->trans_table
);
201 for (n
= 0; n
< 8; n
++) {
203 index_array
[n
] = acl_start_next_trie(&flows
, parms
, n
, ctx
);
206 /* Check for any matches. */
207 acl_match_check_x4(0, ctx
, parms
, &flows
, &index_array
[0]);
208 acl_match_check_x4(4, ctx
, parms
, &flows
, &index_array
[4]);
210 while (flows
.started
> 0) {
211 /* Gather 4 bytes of input data for each stream. */
212 input0
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 0), input0
, 0);
213 input1
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 4), input1
, 0);
215 input0
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 1), input0
, 1);
216 input1
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 5), input1
, 1);
218 input0
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 2), input0
, 2);
219 input1
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 6), input1
, 2);
221 input0
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 3), input0
, 3);
222 input1
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 7), input1
, 3);
224 /* Process the 4 bytes of input on each stream. */
226 input0
= transition4(input0
, flows
.trans
, &index_array
[0]);
227 input1
= transition4(input1
, flows
.trans
, &index_array
[4]);
229 input0
= transition4(input0
, flows
.trans
, &index_array
[0]);
230 input1
= transition4(input1
, flows
.trans
, &index_array
[4]);
232 input0
= transition4(input0
, flows
.trans
, &index_array
[0]);
233 input1
= transition4(input1
, flows
.trans
, &index_array
[4]);
235 input0
= transition4(input0
, flows
.trans
, &index_array
[0]);
236 input1
= transition4(input1
, flows
.trans
, &index_array
[4]);
238 /* Check for any matches. */
239 acl_match_check_x4(0, ctx
, parms
, &flows
, &index_array
[0]);
240 acl_match_check_x4(4, ctx
, parms
, &flows
, &index_array
[4]);
247 * Execute trie traversal with 4 traversals in parallel
250 search_neon_4(const struct rte_acl_ctx
*ctx
, const uint8_t **data
,
251 uint32_t *results
, int total_packets
, uint32_t categories
)
254 struct acl_flow_data flows
;
255 uint64_t index_array
[4];
256 struct completion cmplt
[4];
257 struct parms parms
[4];
260 acl_set_flow(&flows
, cmplt
, RTE_DIM(cmplt
), data
, results
,
261 total_packets
, categories
, ctx
->trans_table
);
263 for (n
= 0; n
< 4; n
++) {
265 index_array
[n
] = acl_start_next_trie(&flows
, parms
, n
, ctx
);
268 /* Check for any matches. */
269 acl_match_check_x4(0, ctx
, parms
, &flows
, index_array
);
271 while (flows
.started
> 0) {
272 /* Gather 4 bytes of input data for each stream. */
273 input
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 0), input
, 0);
274 input
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 1), input
, 1);
275 input
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 2), input
, 2);
276 input
= vsetq_lane_s32(GET_NEXT_4BYTES(parms
, 3), input
, 3);
278 /* Process the 4 bytes of input on each stream. */
279 input
= transition4(input
, flows
.trans
, index_array
);
280 input
= transition4(input
, flows
.trans
, index_array
);
281 input
= transition4(input
, flows
.trans
, index_array
);
282 input
= transition4(input
, flows
.trans
, index_array
);
284 /* Check for any matches. */
285 acl_match_check_x4(0, ctx
, parms
, &flows
, index_array
);