1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
6 #include <rte_common.h>
7 #include <rte_malloc.h>
9 #include <rte_cryptodev.h>
10 #include <rte_cryptodev_pmd.h>
11 #include <rte_reorder.h>
13 #include "scheduler_pmd_private.h"
15 /** attaching the slaves predefined by scheduler's EAL options */
17 scheduler_attach_init_slave(struct rte_cryptodev
*dev
)
19 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
20 uint8_t scheduler_id
= dev
->data
->dev_id
;
23 for (i
= sched_ctx
->nb_init_slaves
- 1; i
>= 0; i
--) {
24 const char *dev_name
= sched_ctx
->init_slave_names
[i
];
25 struct rte_cryptodev
*slave_dev
=
26 rte_cryptodev_pmd_get_named_dev(dev_name
);
30 CR_SCHED_LOG(ERR
, "Failed to locate slave dev %s",
35 status
= rte_cryptodev_scheduler_slave_attach(
36 scheduler_id
, slave_dev
->data
->dev_id
);
39 CR_SCHED_LOG(ERR
, "Failed to attach slave cryptodev %u",
40 slave_dev
->data
->dev_id
);
44 CR_SCHED_LOG(INFO
, "Scheduler %s attached slave %s",
46 sched_ctx
->init_slave_names
[i
]);
48 rte_free(sched_ctx
->init_slave_names
[i
]);
49 sched_ctx
->init_slave_names
[i
] = NULL
;
51 sched_ctx
->nb_init_slaves
-= 1;
56 /** Configure device */
58 scheduler_pmd_config(struct rte_cryptodev
*dev
,
59 struct rte_cryptodev_config
*config
)
61 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
65 /* although scheduler_attach_init_slave presents multiple times,
66 * there will be only 1 meaningful execution.
68 ret
= scheduler_attach_init_slave(dev
);
72 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
73 uint8_t slave_dev_id
= sched_ctx
->slaves
[i
].dev_id
;
75 ret
= rte_cryptodev_configure(slave_dev_id
, config
);
84 update_order_ring(struct rte_cryptodev
*dev
, uint16_t qp_id
)
86 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
87 struct scheduler_qp_ctx
*qp_ctx
= dev
->data
->queue_pairs
[qp_id
];
89 if (sched_ctx
->reordering_enabled
) {
90 char order_ring_name
[RTE_CRYPTODEV_NAME_MAX_LEN
];
91 uint32_t buff_size
= rte_align32pow2(
92 sched_ctx
->nb_slaves
* PER_SLAVE_BUFF_SIZE
);
94 if (qp_ctx
->order_ring
) {
95 rte_ring_free(qp_ctx
->order_ring
);
96 qp_ctx
->order_ring
= NULL
;
102 if (snprintf(order_ring_name
, RTE_CRYPTODEV_NAME_MAX_LEN
,
103 "%s_rb_%u_%u", RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD
),
104 dev
->data
->dev_id
, qp_id
) < 0) {
105 CR_SCHED_LOG(ERR
, "failed to create unique reorder buffer"
110 qp_ctx
->order_ring
= rte_ring_create(order_ring_name
,
111 buff_size
, rte_socket_id(),
112 RING_F_SP_ENQ
| RING_F_SC_DEQ
);
113 if (!qp_ctx
->order_ring
) {
114 CR_SCHED_LOG(ERR
, "failed to create order ring");
118 if (qp_ctx
->order_ring
) {
119 rte_ring_free(qp_ctx
->order_ring
);
120 qp_ctx
->order_ring
= NULL
;
129 scheduler_pmd_start(struct rte_cryptodev
*dev
)
131 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
135 if (dev
->data
->dev_started
)
138 /* although scheduler_attach_init_slave presents multiple times,
139 * there will be only 1 meaningful execution.
141 ret
= scheduler_attach_init_slave(dev
);
145 for (i
= 0; i
< dev
->data
->nb_queue_pairs
; i
++) {
146 ret
= update_order_ring(dev
, i
);
148 CR_SCHED_LOG(ERR
, "Failed to update reorder buffer");
153 if (sched_ctx
->mode
== CDEV_SCHED_MODE_NOT_SET
) {
154 CR_SCHED_LOG(ERR
, "Scheduler mode is not set");
158 if (!sched_ctx
->nb_slaves
) {
159 CR_SCHED_LOG(ERR
, "No slave in the scheduler");
163 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx
->ops
.slave_attach
, -ENOTSUP
);
165 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
166 uint8_t slave_dev_id
= sched_ctx
->slaves
[i
].dev_id
;
168 if ((*sched_ctx
->ops
.slave_attach
)(dev
, slave_dev_id
) < 0) {
169 CR_SCHED_LOG(ERR
, "Failed to attach slave");
174 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx
->ops
.scheduler_start
, -ENOTSUP
);
176 if ((*sched_ctx
->ops
.scheduler_start
)(dev
) < 0) {
177 CR_SCHED_LOG(ERR
, "Scheduler start failed");
181 /* start all slaves */
182 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
183 uint8_t slave_dev_id
= sched_ctx
->slaves
[i
].dev_id
;
184 struct rte_cryptodev
*slave_dev
=
185 rte_cryptodev_pmd_get_dev(slave_dev_id
);
187 ret
= (*slave_dev
->dev_ops
->dev_start
)(slave_dev
);
189 CR_SCHED_LOG(ERR
, "Failed to start slave dev %u",
200 scheduler_pmd_stop(struct rte_cryptodev
*dev
)
202 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
205 if (!dev
->data
->dev_started
)
208 /* stop all slaves first */
209 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
210 uint8_t slave_dev_id
= sched_ctx
->slaves
[i
].dev_id
;
211 struct rte_cryptodev
*slave_dev
=
212 rte_cryptodev_pmd_get_dev(slave_dev_id
);
214 (*slave_dev
->dev_ops
->dev_stop
)(slave_dev
);
217 if (*sched_ctx
->ops
.scheduler_stop
)
218 (*sched_ctx
->ops
.scheduler_stop
)(dev
);
220 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
221 uint8_t slave_dev_id
= sched_ctx
->slaves
[i
].dev_id
;
223 if (*sched_ctx
->ops
.slave_detach
)
224 (*sched_ctx
->ops
.slave_detach
)(dev
, slave_dev_id
);
230 scheduler_pmd_close(struct rte_cryptodev
*dev
)
232 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
236 /* the dev should be stopped before being closed */
237 if (dev
->data
->dev_started
)
240 /* close all slaves first */
241 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
242 uint8_t slave_dev_id
= sched_ctx
->slaves
[i
].dev_id
;
243 struct rte_cryptodev
*slave_dev
=
244 rte_cryptodev_pmd_get_dev(slave_dev_id
);
246 ret
= (*slave_dev
->dev_ops
->dev_close
)(slave_dev
);
251 for (i
= 0; i
< dev
->data
->nb_queue_pairs
; i
++) {
252 struct scheduler_qp_ctx
*qp_ctx
= dev
->data
->queue_pairs
[i
];
254 if (qp_ctx
->order_ring
) {
255 rte_ring_free(qp_ctx
->order_ring
);
256 qp_ctx
->order_ring
= NULL
;
259 if (qp_ctx
->private_qp_ctx
) {
260 rte_free(qp_ctx
->private_qp_ctx
);
261 qp_ctx
->private_qp_ctx
= NULL
;
265 if (sched_ctx
->private_ctx
) {
266 rte_free(sched_ctx
->private_ctx
);
267 sched_ctx
->private_ctx
= NULL
;
270 if (sched_ctx
->capabilities
) {
271 rte_free(sched_ctx
->capabilities
);
272 sched_ctx
->capabilities
= NULL
;
278 /** Get device statistics */
280 scheduler_pmd_stats_get(struct rte_cryptodev
*dev
,
281 struct rte_cryptodev_stats
*stats
)
283 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
286 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
287 uint8_t slave_dev_id
= sched_ctx
->slaves
[i
].dev_id
;
288 struct rte_cryptodev
*slave_dev
=
289 rte_cryptodev_pmd_get_dev(slave_dev_id
);
290 struct rte_cryptodev_stats slave_stats
= {0};
292 (*slave_dev
->dev_ops
->stats_get
)(slave_dev
, &slave_stats
);
294 stats
->enqueued_count
+= slave_stats
.enqueued_count
;
295 stats
->dequeued_count
+= slave_stats
.dequeued_count
;
297 stats
->enqueue_err_count
+= slave_stats
.enqueue_err_count
;
298 stats
->dequeue_err_count
+= slave_stats
.dequeue_err_count
;
302 /** Reset device statistics */
304 scheduler_pmd_stats_reset(struct rte_cryptodev
*dev
)
306 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
309 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
310 uint8_t slave_dev_id
= sched_ctx
->slaves
[i
].dev_id
;
311 struct rte_cryptodev
*slave_dev
=
312 rte_cryptodev_pmd_get_dev(slave_dev_id
);
314 (*slave_dev
->dev_ops
->stats_reset
)(slave_dev
);
318 /** Get device info */
320 scheduler_pmd_info_get(struct rte_cryptodev
*dev
,
321 struct rte_cryptodev_info
*dev_info
)
323 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
324 uint32_t max_nb_sess
= 0;
325 uint16_t headroom_sz
= 0;
326 uint16_t tailroom_sz
= 0;
332 /* although scheduler_attach_init_slave presents multiple times,
333 * there will be only 1 meaningful execution.
335 scheduler_attach_init_slave(dev
);
337 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
338 uint8_t slave_dev_id
= sched_ctx
->slaves
[i
].dev_id
;
339 struct rte_cryptodev_info slave_info
;
341 rte_cryptodev_info_get(slave_dev_id
, &slave_info
);
342 uint32_t dev_max_sess
= slave_info
.sym
.max_nb_sessions
;
343 if (dev_max_sess
!= 0) {
344 if (max_nb_sess
== 0 || dev_max_sess
< max_nb_sess
)
345 max_nb_sess
= slave_info
.sym
.max_nb_sessions
;
348 /* Get the max headroom requirement among slave PMDs */
349 headroom_sz
= slave_info
.min_mbuf_headroom_req
>
351 slave_info
.min_mbuf_headroom_req
:
354 /* Get the max tailroom requirement among slave PMDs */
355 tailroom_sz
= slave_info
.min_mbuf_tailroom_req
>
357 slave_info
.min_mbuf_tailroom_req
:
361 dev_info
->driver_id
= dev
->driver_id
;
362 dev_info
->feature_flags
= dev
->feature_flags
;
363 dev_info
->capabilities
= sched_ctx
->capabilities
;
364 dev_info
->max_nb_queue_pairs
= sched_ctx
->max_nb_queue_pairs
;
365 dev_info
->min_mbuf_headroom_req
= headroom_sz
;
366 dev_info
->min_mbuf_tailroom_req
= tailroom_sz
;
367 dev_info
->sym
.max_nb_sessions
= max_nb_sess
;
370 /** Release queue pair */
372 scheduler_pmd_qp_release(struct rte_cryptodev
*dev
, uint16_t qp_id
)
374 struct scheduler_qp_ctx
*qp_ctx
= dev
->data
->queue_pairs
[qp_id
];
379 if (qp_ctx
->order_ring
)
380 rte_ring_free(qp_ctx
->order_ring
);
381 if (qp_ctx
->private_qp_ctx
)
382 rte_free(qp_ctx
->private_qp_ctx
);
385 dev
->data
->queue_pairs
[qp_id
] = NULL
;
390 /** Setup a queue pair */
392 scheduler_pmd_qp_setup(struct rte_cryptodev
*dev
, uint16_t qp_id
,
393 const struct rte_cryptodev_qp_conf
*qp_conf
, int socket_id
)
395 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
396 struct scheduler_qp_ctx
*qp_ctx
;
397 char name
[RTE_CRYPTODEV_NAME_MAX_LEN
];
401 if (snprintf(name
, RTE_CRYPTODEV_NAME_MAX_LEN
,
402 "CRYTO_SCHE PMD %u QP %u",
403 dev
->data
->dev_id
, qp_id
) < 0) {
404 CR_SCHED_LOG(ERR
, "Failed to create unique queue pair name");
408 /* Free memory prior to re-allocation if needed. */
409 if (dev
->data
->queue_pairs
[qp_id
] != NULL
)
410 scheduler_pmd_qp_release(dev
, qp_id
);
412 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
413 uint8_t slave_id
= sched_ctx
->slaves
[i
].dev_id
;
416 * All slaves will share the same session mempool
417 * for session-less operations, so the objects
418 * must be big enough for all the drivers used.
420 ret
= rte_cryptodev_queue_pair_setup(slave_id
, qp_id
,
426 /* Allocate the queue pair data structure. */
427 qp_ctx
= rte_zmalloc_socket(name
, sizeof(*qp_ctx
), RTE_CACHE_LINE_SIZE
,
432 /* The actual available object number = nb_descriptors - 1 */
433 qp_ctx
->max_nb_objs
= qp_conf
->nb_descriptors
- 1;
435 dev
->data
->queue_pairs
[qp_id
] = qp_ctx
;
437 /* although scheduler_attach_init_slave presents multiple times,
438 * there will be only 1 meaningful execution.
440 ret
= scheduler_attach_init_slave(dev
);
442 CR_SCHED_LOG(ERR
, "Failed to attach slave");
443 scheduler_pmd_qp_release(dev
, qp_id
);
447 if (*sched_ctx
->ops
.config_queue_pair
) {
448 if ((*sched_ctx
->ops
.config_queue_pair
)(dev
, qp_id
) < 0) {
449 CR_SCHED_LOG(ERR
, "Unable to configure queue pair");
457 /** Return the number of allocated queue pairs */
459 scheduler_pmd_qp_count(struct rte_cryptodev
*dev
)
461 return dev
->data
->nb_queue_pairs
;
465 scheduler_pmd_sym_session_get_size(struct rte_cryptodev
*dev __rte_unused
)
467 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
469 uint32_t max_priv_sess_size
= 0;
471 /* Check what is the maximum private session size for all slaves */
472 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
473 uint8_t slave_dev_id
= sched_ctx
->slaves
[i
].dev_id
;
474 struct rte_cryptodev
*dev
= &rte_cryptodevs
[slave_dev_id
];
475 uint32_t priv_sess_size
= (*dev
->dev_ops
->sym_session_get_size
)(dev
);
477 if (max_priv_sess_size
< priv_sess_size
)
478 max_priv_sess_size
= priv_sess_size
;
481 return max_priv_sess_size
;
485 scheduler_pmd_sym_session_configure(struct rte_cryptodev
*dev
,
486 struct rte_crypto_sym_xform
*xform
,
487 struct rte_cryptodev_sym_session
*sess
,
488 struct rte_mempool
*mempool
)
490 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
494 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
495 struct scheduler_slave
*slave
= &sched_ctx
->slaves
[i
];
497 ret
= rte_cryptodev_sym_session_init(slave
->dev_id
, sess
,
500 CR_SCHED_LOG(ERR
, "unable to config sym session");
508 /** Clear the memory of session so it doesn't leave key material behind */
510 scheduler_pmd_sym_session_clear(struct rte_cryptodev
*dev
,
511 struct rte_cryptodev_sym_session
*sess
)
513 struct scheduler_ctx
*sched_ctx
= dev
->data
->dev_private
;
516 /* Clear private data of slaves */
517 for (i
= 0; i
< sched_ctx
->nb_slaves
; i
++) {
518 struct scheduler_slave
*slave
= &sched_ctx
->slaves
[i
];
520 rte_cryptodev_sym_session_clear(slave
->dev_id
, sess
);
524 static struct rte_cryptodev_ops scheduler_pmd_ops
= {
525 .dev_configure
= scheduler_pmd_config
,
526 .dev_start
= scheduler_pmd_start
,
527 .dev_stop
= scheduler_pmd_stop
,
528 .dev_close
= scheduler_pmd_close
,
530 .stats_get
= scheduler_pmd_stats_get
,
531 .stats_reset
= scheduler_pmd_stats_reset
,
533 .dev_infos_get
= scheduler_pmd_info_get
,
535 .queue_pair_setup
= scheduler_pmd_qp_setup
,
536 .queue_pair_release
= scheduler_pmd_qp_release
,
537 .queue_pair_count
= scheduler_pmd_qp_count
,
539 .sym_session_get_size
= scheduler_pmd_sym_session_get_size
,
540 .sym_session_configure
= scheduler_pmd_sym_session_configure
,
541 .sym_session_clear
= scheduler_pmd_sym_session_clear
,
544 struct rte_cryptodev_ops
*rte_crypto_scheduler_pmd_ops
= &scheduler_pmd_ops
;