1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2017 Intel Corporation
5 #include <rte_malloc.h>
6 #include <rte_cycles.h>
7 #include <rte_crypto.h>
8 #include <rte_cryptodev.h>
10 #include "cperf_test_latency.h"
11 #include "cperf_ops.h"
12 #include "cperf_test_common.h"
14 struct cperf_op_result
{
17 enum rte_crypto_op_status status
;
20 struct cperf_latency_ctx
{
25 struct rte_mempool
*pool
;
27 struct rte_cryptodev_sym_session
*sess
;
29 cperf_populate_ops_t populate_ops
;
31 uint32_t src_buf_offset
;
32 uint32_t dst_buf_offset
;
34 const struct cperf_options
*options
;
35 const struct cperf_test_vector
*test_vector
;
36 struct cperf_op_result
*res
;
40 struct cperf_op_result
*result
;
43 #define max(a, b) (a > b ? (uint64_t)a : (uint64_t)b)
44 #define min(a, b) (a < b ? (uint64_t)a : (uint64_t)b)
47 cperf_latency_test_free(struct cperf_latency_ctx
*ctx
)
51 rte_cryptodev_sym_session_clear(ctx
->dev_id
, ctx
->sess
);
52 rte_cryptodev_sym_session_free(ctx
->sess
);
56 rte_mempool_free(ctx
->pool
);
64 cperf_latency_test_constructor(struct rte_mempool
*sess_mp
,
65 uint8_t dev_id
, uint16_t qp_id
,
66 const struct cperf_options
*options
,
67 const struct cperf_test_vector
*test_vector
,
68 const struct cperf_op_fns
*op_fns
)
70 struct cperf_latency_ctx
*ctx
= NULL
;
71 size_t extra_op_priv_size
= sizeof(struct priv_op_data
);
73 ctx
= rte_malloc(NULL
, sizeof(struct cperf_latency_ctx
), 0);
80 ctx
->populate_ops
= op_fns
->populate_ops
;
81 ctx
->options
= options
;
82 ctx
->test_vector
= test_vector
;
84 /* IV goes at the end of the crypto operation */
85 uint16_t iv_offset
= sizeof(struct rte_crypto_op
) +
86 sizeof(struct rte_crypto_sym_op
) +
87 sizeof(struct cperf_op_result
*);
89 ctx
->sess
= op_fns
->sess_create(sess_mp
, dev_id
, options
, test_vector
,
91 if (ctx
->sess
== NULL
)
94 if (cperf_alloc_common_memory(options
, test_vector
, dev_id
, qp_id
,
96 &ctx
->src_buf_offset
, &ctx
->dst_buf_offset
,
100 ctx
->res
= rte_malloc(NULL
, sizeof(struct cperf_op_result
) *
101 ctx
->options
->total_ops
, 0);
103 if (ctx
->res
== NULL
)
108 cperf_latency_test_free(ctx
);
114 store_timestamp(struct rte_crypto_op
*op
, uint64_t timestamp
)
116 struct priv_op_data
*priv_data
;
118 priv_data
= (struct priv_op_data
*) (op
->sym
+ 1);
119 priv_data
->result
->status
= op
->status
;
120 priv_data
->result
->tsc_end
= timestamp
;
124 cperf_latency_test_runner(void *arg
)
126 struct cperf_latency_ctx
*ctx
= arg
;
127 uint16_t test_burst_size
;
128 uint8_t burst_size_idx
= 0;
129 uint32_t imix_idx
= 0;
131 static int only_once
;
136 struct rte_crypto_op
*ops
[ctx
->options
->max_burst_size
];
137 struct rte_crypto_op
*ops_processed
[ctx
->options
->max_burst_size
];
139 struct priv_op_data
*priv_data
;
141 uint32_t lcore
= rte_lcore_id();
143 #ifdef CPERF_LINEARIZATION_ENABLE
144 struct rte_cryptodev_info dev_info
;
147 /* Check if source mbufs require coalescing */
148 if (ctx
->options
->segment_sz
< ctx
->options
->max_buffer_size
) {
149 rte_cryptodev_info_get(ctx
->dev_id
, &dev_info
);
150 if ((dev_info
.feature_flags
&
151 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER
) == 0)
154 #endif /* CPERF_LINEARIZATION_ENABLE */
156 ctx
->lcore_id
= lcore
;
158 /* Warm up the host CPU before starting the test */
159 for (i
= 0; i
< ctx
->options
->total_ops
; i
++)
160 rte_cryptodev_enqueue_burst(ctx
->dev_id
, ctx
->qp_id
, NULL
, 0);
162 /* Get first size from range or list */
163 if (ctx
->options
->inc_burst_size
!= 0)
164 test_burst_size
= ctx
->options
->min_burst_size
;
166 test_burst_size
= ctx
->options
->burst_size_list
[0];
168 uint16_t iv_offset
= sizeof(struct rte_crypto_op
) +
169 sizeof(struct rte_crypto_sym_op
) +
170 sizeof(struct cperf_op_result
*);
172 while (test_burst_size
<= ctx
->options
->max_burst_size
) {
173 uint64_t ops_enqd
= 0, ops_deqd
= 0;
176 uint64_t tsc_val
, tsc_end
, tsc_start
;
177 uint64_t tsc_max
= 0, tsc_min
= ~0UL, tsc_tot
= 0, tsc_idx
= 0;
178 uint64_t enqd_max
= 0, enqd_min
= ~0UL, enqd_tot
= 0;
179 uint64_t deqd_max
= 0, deqd_min
= ~0UL, deqd_tot
= 0;
181 while (enqd_tot
< ctx
->options
->total_ops
) {
183 uint16_t burst_size
= ((enqd_tot
+ test_burst_size
)
184 <= ctx
->options
->total_ops
) ?
186 ctx
->options
->total_ops
-
189 /* Allocate objects containing crypto operations and mbufs */
190 if (rte_mempool_get_bulk(ctx
->pool
, (void **)ops
,
193 "Failed to allocate more crypto operations "
194 "from the crypto operation pool.\n"
195 "Consider increasing the pool size "
200 /* Setup crypto op, attach mbuf etc */
201 (ctx
->populate_ops
)(ops
, ctx
->src_buf_offset
,
203 burst_size
, ctx
->sess
, ctx
->options
,
204 ctx
->test_vector
, iv_offset
,
207 tsc_start
= rte_rdtsc_precise();
209 #ifdef CPERF_LINEARIZATION_ENABLE
211 /* PMD doesn't support scatter-gather and source buffer
213 * We need to linearize it before enqueuing.
215 for (i
= 0; i
< burst_size
; i
++)
216 rte_pktmbuf_linearize(ops
[i
]->sym
->m_src
);
218 #endif /* CPERF_LINEARIZATION_ENABLE */
220 /* Enqueue burst of ops on crypto device */
221 ops_enqd
= rte_cryptodev_enqueue_burst(ctx
->dev_id
, ctx
->qp_id
,
224 /* Dequeue processed burst of ops from crypto device */
225 ops_deqd
= rte_cryptodev_dequeue_burst(ctx
->dev_id
, ctx
->qp_id
,
226 ops_processed
, test_burst_size
);
228 tsc_end
= rte_rdtsc_precise();
230 /* Free memory for not enqueued operations */
231 if (ops_enqd
!= burst_size
)
232 rte_mempool_put_bulk(ctx
->pool
,
233 (void **)&ops
[ops_enqd
],
234 burst_size
- ops_enqd
);
236 for (i
= 0; i
< ops_enqd
; i
++) {
237 ctx
->res
[tsc_idx
].tsc_start
= tsc_start
;
239 * Private data structure starts after the end of the
240 * rte_crypto_sym_op structure.
242 priv_data
= (struct priv_op_data
*) (ops
[i
]->sym
+ 1);
243 priv_data
->result
= (void *)&ctx
->res
[tsc_idx
];
247 if (likely(ops_deqd
)) {
248 /* Free crypto ops so they can be reused. */
249 for (i
= 0; i
< ops_deqd
; i
++)
250 store_timestamp(ops_processed
[i
], tsc_end
);
252 rte_mempool_put_bulk(ctx
->pool
,
253 (void **)ops_processed
, ops_deqd
);
255 deqd_tot
+= ops_deqd
;
256 deqd_max
= max(ops_deqd
, deqd_max
);
257 deqd_min
= min(ops_deqd
, deqd_min
);
260 enqd_tot
+= ops_enqd
;
261 enqd_max
= max(ops_enqd
, enqd_max
);
262 enqd_min
= min(ops_enqd
, enqd_min
);
267 /* Dequeue any operations still in the crypto device */
268 while (deqd_tot
< ctx
->options
->total_ops
) {
269 /* Sending 0 length burst to flush sw crypto device */
270 rte_cryptodev_enqueue_burst(ctx
->dev_id
, ctx
->qp_id
, NULL
, 0);
273 ops_deqd
= rte_cryptodev_dequeue_burst(ctx
->dev_id
, ctx
->qp_id
,
274 ops_processed
, test_burst_size
);
276 tsc_end
= rte_rdtsc_precise();
279 for (i
= 0; i
< ops_deqd
; i
++)
280 store_timestamp(ops_processed
[i
], tsc_end
);
282 rte_mempool_put_bulk(ctx
->pool
,
283 (void **)ops_processed
, ops_deqd
);
285 deqd_tot
+= ops_deqd
;
286 deqd_max
= max(ops_deqd
, deqd_max
);
287 deqd_min
= min(ops_deqd
, deqd_min
);
291 for (i
= 0; i
< tsc_idx
; i
++) {
292 tsc_val
= ctx
->res
[i
].tsc_end
- ctx
->res
[i
].tsc_start
;
293 tsc_max
= max(tsc_val
, tsc_max
);
294 tsc_min
= min(tsc_val
, tsc_min
);
298 double time_tot
, time_avg
, time_max
, time_min
;
300 const uint64_t tunit
= 1000000; /* us */
301 const uint64_t tsc_hz
= rte_get_tsc_hz();
303 uint64_t enqd_avg
= enqd_tot
/ b_idx
;
304 uint64_t deqd_avg
= deqd_tot
/ b_idx
;
305 uint64_t tsc_avg
= tsc_tot
/ tsc_idx
;
307 time_tot
= tunit
*(double)(tsc_tot
) / tsc_hz
;
308 time_avg
= tunit
*(double)(tsc_avg
) / tsc_hz
;
309 time_max
= tunit
*(double)(tsc_max
) / tsc_hz
;
310 time_min
= tunit
*(double)(tsc_min
) / tsc_hz
;
312 if (ctx
->options
->csv
) {
314 printf("\n# lcore, Buffer Size, Burst Size, Pakt Seq #, "
315 "Packet Size, cycles, time (us)");
317 for (i
= 0; i
< ctx
->options
->total_ops
; i
++) {
319 printf("\n%u;%u;%u;%"PRIu64
";%"PRIu64
";%.3f",
320 ctx
->lcore_id
, ctx
->options
->test_buffer_size
,
321 test_burst_size
, i
+ 1,
322 ctx
->res
[i
].tsc_end
- ctx
->res
[i
].tsc_start
,
323 tunit
* (double) (ctx
->res
[i
].tsc_end
324 - ctx
->res
[i
].tsc_start
)
330 printf("\n# Device %d on lcore %u\n", ctx
->dev_id
,
332 printf("\n# total operations: %u", ctx
->options
->total_ops
);
333 printf("\n# Buffer size: %u", ctx
->options
->test_buffer_size
);
334 printf("\n# Burst size: %u", test_burst_size
);
335 printf("\n# Number of bursts: %"PRIu64
,
339 printf("\n# \t Total\t Average\t "
340 "Maximum\t Minimum");
341 printf("\n# enqueued\t%12"PRIu64
"\t%10"PRIu64
"\t"
342 "%10"PRIu64
"\t%10"PRIu64
, enqd_tot
,
343 enqd_avg
, enqd_max
, enqd_min
);
344 printf("\n# dequeued\t%12"PRIu64
"\t%10"PRIu64
"\t"
345 "%10"PRIu64
"\t%10"PRIu64
, deqd_tot
,
346 deqd_avg
, deqd_max
, deqd_min
);
347 printf("\n# cycles\t%12"PRIu64
"\t%10"PRIu64
"\t"
348 "%10"PRIu64
"\t%10"PRIu64
, tsc_tot
,
349 tsc_avg
, tsc_max
, tsc_min
);
350 printf("\n# time [us]\t%12.0f\t%10.3f\t%10.3f\t%10.3f",
351 time_tot
, time_avg
, time_max
, time_min
);
356 /* Get next size from range or list */
357 if (ctx
->options
->inc_burst_size
!= 0)
358 test_burst_size
+= ctx
->options
->inc_burst_size
;
360 if (++burst_size_idx
== ctx
->options
->burst_size_count
)
363 ctx
->options
->burst_size_list
[burst_size_idx
];
371 cperf_latency_test_destructor(void *arg
)
373 struct cperf_latency_ctx
*ctx
= arg
;
378 cperf_latency_test_free(ctx
);