]> git.proxmox.com Git - ceph.git/blob - ceph/src/seastar/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
import 15.2.0 Octopus source
[ceph.git] / ceph / src / seastar / dpdk / drivers / event / dpaa2 / dpaa2_eventdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright 2017 NXP
4 *
5 */
6
7 #include <assert.h>
8 #include <stdio.h>
9 #include <stdbool.h>
10 #include <errno.h>
11 #include <stdint.h>
12 #include <string.h>
13 #include <sys/epoll.h>
14
15 #include <rte_atomic.h>
16 #include <rte_byteorder.h>
17 #include <rte_common.h>
18 #include <rte_debug.h>
19 #include <rte_dev.h>
20 #include <rte_eal.h>
21 #include <rte_fslmc.h>
22 #include <rte_lcore.h>
23 #include <rte_log.h>
24 #include <rte_malloc.h>
25 #include <rte_memcpy.h>
26 #include <rte_memory.h>
27 #include <rte_pci.h>
28 #include <rte_bus_vdev.h>
29 #include <rte_ethdev_driver.h>
30 #include <rte_cryptodev.h>
31 #include <rte_event_eth_rx_adapter.h>
32
33 #include <fslmc_vfio.h>
34 #include <dpaa2_hw_pvt.h>
35 #include <dpaa2_hw_mempool.h>
36 #include <dpaa2_hw_dpio.h>
37 #include <dpaa2_ethdev.h>
38 #ifdef RTE_LIBRTE_SECURITY
39 #include <dpaa2_sec_event.h>
40 #endif
41 #include "dpaa2_eventdev.h"
42 #include "dpaa2_eventdev_logs.h"
43 #include <portal/dpaa2_hw_pvt.h>
44 #include <mc/fsl_dpci.h>
45
46 /* Clarifications
47 * Evendev = SoC Instance
48 * Eventport = DPIO Instance
49 * Eventqueue = DPCON Instance
50 * 1 Eventdev can have N Eventqueue
51 * Soft Event Flow is DPCI Instance
52 */
53
54 /* Dynamic logging identified for mempool */
55 int dpaa2_logtype_event;
56
57 static uint16_t
58 dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[],
59 uint16_t nb_events)
60 {
61
62 struct dpaa2_port *dpaa2_portal = port;
63 struct dpaa2_dpio_dev *dpio_dev;
64 uint32_t queue_id = ev[0].queue_id;
65 struct dpaa2_eventq *evq_info;
66 uint32_t fqid;
67 struct qbman_swp *swp;
68 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
69 uint32_t loop, frames_to_send;
70 struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS];
71 uint16_t num_tx = 0;
72 int i, n, ret;
73 uint8_t channel_index;
74
75 if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
76 /* Affine current thread context to a qman portal */
77 ret = dpaa2_affine_qbman_swp();
78 if (ret < 0) {
79 DPAA2_EVENTDEV_ERR("Failure in affining portal");
80 return 0;
81 }
82 }
83 /* todo - dpaa2_portal shall have dpio_dev - no per thread variable */
84 dpio_dev = DPAA2_PER_LCORE_DPIO;
85 swp = DPAA2_PER_LCORE_PORTAL;
86
87 if (likely(dpaa2_portal->is_port_linked))
88 goto skip_linking;
89
90 /* Create mapping between portal and channel to receive packets */
91 for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) {
92 evq_info = &dpaa2_portal->evq_info[i];
93 if (!evq_info->event_port)
94 continue;
95
96 ret = dpio_add_static_dequeue_channel(dpio_dev->dpio,
97 CMD_PRI_LOW,
98 dpio_dev->token,
99 evq_info->dpcon->dpcon_id,
100 &channel_index);
101 if (ret < 0) {
102 DPAA2_EVENTDEV_ERR(
103 "Static dequeue config failed: err(%d)", ret);
104 goto err;
105 }
106
107 qbman_swp_push_set(swp, channel_index, 1);
108 evq_info->dpcon->channel_index = channel_index;
109 }
110 dpaa2_portal->is_port_linked = true;
111
112 skip_linking:
113 evq_info = &dpaa2_portal->evq_info[queue_id];
114
115 while (nb_events) {
116 frames_to_send = (nb_events > dpaa2_eqcr_size) ?
117 dpaa2_eqcr_size : nb_events;
118
119 for (loop = 0; loop < frames_to_send; loop++) {
120 const struct rte_event *event = &ev[num_tx + loop];
121
122 if (event->sched_type != RTE_SCHED_TYPE_ATOMIC)
123 fqid = evq_info->dpci->rx_queue[
124 DPAA2_EVENT_DPCI_PARALLEL_QUEUE].fqid;
125 else
126 fqid = evq_info->dpci->rx_queue[
127 DPAA2_EVENT_DPCI_ATOMIC_QUEUE].fqid;
128
129 /* Prepare enqueue descriptor */
130 qbman_eq_desc_clear(&eqdesc[loop]);
131 qbman_eq_desc_set_fq(&eqdesc[loop], fqid);
132 qbman_eq_desc_set_no_orp(&eqdesc[loop], 0);
133 qbman_eq_desc_set_response(&eqdesc[loop], 0, 0);
134
135 if (event->sched_type == RTE_SCHED_TYPE_ATOMIC
136 && event->mbuf->seqn) {
137 uint8_t dqrr_index = event->mbuf->seqn - 1;
138
139 qbman_eq_desc_set_dca(&eqdesc[loop], 1,
140 dqrr_index, 0);
141 DPAA2_PER_LCORE_DQRR_SIZE--;
142 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
143 }
144
145 memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
146
147 /*
148 * todo - need to align with hw context data
149 * to avoid copy
150 */
151 struct rte_event *ev_temp = rte_malloc(NULL,
152 sizeof(struct rte_event), 0);
153
154 if (!ev_temp) {
155 if (!loop)
156 return num_tx;
157 frames_to_send = loop;
158 DPAA2_EVENTDEV_ERR(
159 "Unable to allocate event object");
160 goto send_partial;
161 }
162 rte_memcpy(ev_temp, event, sizeof(struct rte_event));
163 DPAA2_SET_FD_ADDR((&fd_arr[loop]), (size_t)ev_temp);
164 DPAA2_SET_FD_LEN((&fd_arr[loop]),
165 sizeof(struct rte_event));
166 }
167 send_partial:
168 loop = 0;
169 while (loop < frames_to_send) {
170 loop += qbman_swp_enqueue_multiple_desc(swp,
171 &eqdesc[loop], &fd_arr[loop],
172 frames_to_send - loop);
173 }
174 num_tx += frames_to_send;
175 nb_events -= frames_to_send;
176 }
177
178 return num_tx;
179 err:
180 for (n = 0; n < i; n++) {
181 evq_info = &dpaa2_portal->evq_info[n];
182 if (!evq_info->event_port)
183 continue;
184 qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0);
185 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0,
186 dpio_dev->token,
187 evq_info->dpcon->dpcon_id);
188 }
189 return 0;
190
191 }
192
193 static uint16_t
194 dpaa2_eventdev_enqueue(void *port, const struct rte_event *ev)
195 {
196 return dpaa2_eventdev_enqueue_burst(port, ev, 1);
197 }
198
199 static void dpaa2_eventdev_dequeue_wait(uint64_t timeout_ticks)
200 {
201 struct epoll_event epoll_ev;
202
203 qbman_swp_interrupt_clear_status(DPAA2_PER_LCORE_PORTAL,
204 QBMAN_SWP_INTERRUPT_DQRI);
205
206 epoll_wait(DPAA2_PER_LCORE_DPIO->epoll_fd,
207 &epoll_ev, 1, timeout_ticks);
208 }
209
210 static void dpaa2_eventdev_process_parallel(struct qbman_swp *swp,
211 const struct qbman_fd *fd,
212 const struct qbman_result *dq,
213 struct dpaa2_queue *rxq,
214 struct rte_event *ev)
215 {
216 struct rte_event *ev_temp =
217 (struct rte_event *)(size_t)DPAA2_GET_FD_ADDR(fd);
218
219 RTE_SET_USED(rxq);
220
221 rte_memcpy(ev, ev_temp, sizeof(struct rte_event));
222 rte_free(ev_temp);
223
224 qbman_swp_dqrr_consume(swp, dq);
225 }
226
227 static void dpaa2_eventdev_process_atomic(struct qbman_swp *swp,
228 const struct qbman_fd *fd,
229 const struct qbman_result *dq,
230 struct dpaa2_queue *rxq,
231 struct rte_event *ev)
232 {
233 struct rte_event *ev_temp =
234 (struct rte_event *)(size_t)DPAA2_GET_FD_ADDR(fd);
235 uint8_t dqrr_index = qbman_get_dqrr_idx(dq);
236
237 RTE_SET_USED(swp);
238 RTE_SET_USED(rxq);
239
240 rte_memcpy(ev, ev_temp, sizeof(struct rte_event));
241 rte_free(ev_temp);
242 ev->mbuf->seqn = dqrr_index + 1;
243 DPAA2_PER_LCORE_DQRR_SIZE++;
244 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
245 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf;
246 }
247
248 static uint16_t
249 dpaa2_eventdev_dequeue_burst(void *port, struct rte_event ev[],
250 uint16_t nb_events, uint64_t timeout_ticks)
251 {
252 const struct qbman_result *dq;
253 struct dpaa2_dpio_dev *dpio_dev = NULL;
254 struct dpaa2_port *dpaa2_portal = port;
255 struct dpaa2_eventq *evq_info;
256 struct qbman_swp *swp;
257 const struct qbman_fd *fd;
258 struct dpaa2_queue *rxq;
259 int num_pkts = 0, ret, i = 0, n;
260 uint8_t channel_index;
261
262 if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
263 /* Affine current thread context to a qman portal */
264 ret = dpaa2_affine_qbman_swp();
265 if (ret < 0) {
266 DPAA2_EVENTDEV_ERR("Failure in affining portal");
267 return 0;
268 }
269 }
270
271 dpio_dev = DPAA2_PER_LCORE_DPIO;
272 swp = DPAA2_PER_LCORE_PORTAL;
273
274 if (likely(dpaa2_portal->is_port_linked))
275 goto skip_linking;
276
277 /* Create mapping between portal and channel to receive packets */
278 for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) {
279 evq_info = &dpaa2_portal->evq_info[i];
280 if (!evq_info->event_port)
281 continue;
282
283 ret = dpio_add_static_dequeue_channel(dpio_dev->dpio,
284 CMD_PRI_LOW,
285 dpio_dev->token,
286 evq_info->dpcon->dpcon_id,
287 &channel_index);
288 if (ret < 0) {
289 DPAA2_EVENTDEV_ERR(
290 "Static dequeue config failed: err(%d)", ret);
291 goto err;
292 }
293
294 qbman_swp_push_set(swp, channel_index, 1);
295 evq_info->dpcon->channel_index = channel_index;
296 }
297 dpaa2_portal->is_port_linked = true;
298
299 skip_linking:
300 /* Check if there are atomic contexts to be released */
301 while (DPAA2_PER_LCORE_DQRR_SIZE) {
302 if (DPAA2_PER_LCORE_DQRR_HELD & (1 << i)) {
303 qbman_swp_dqrr_idx_consume(swp, i);
304 DPAA2_PER_LCORE_DQRR_SIZE--;
305 DPAA2_PER_LCORE_DQRR_MBUF(i)->seqn =
306 DPAA2_INVALID_MBUF_SEQN;
307 }
308 i++;
309 }
310 DPAA2_PER_LCORE_DQRR_HELD = 0;
311
312 do {
313 dq = qbman_swp_dqrr_next(swp);
314 if (!dq) {
315 if (!num_pkts && timeout_ticks) {
316 dpaa2_eventdev_dequeue_wait(timeout_ticks);
317 timeout_ticks = 0;
318 continue;
319 }
320 return num_pkts;
321 }
322 qbman_swp_prefetch_dqrr_next(swp);
323
324 fd = qbman_result_DQ_fd(dq);
325 rxq = (struct dpaa2_queue *)(size_t)qbman_result_DQ_fqd_ctx(dq);
326 if (rxq) {
327 rxq->cb(swp, fd, dq, rxq, &ev[num_pkts]);
328 } else {
329 qbman_swp_dqrr_consume(swp, dq);
330 DPAA2_EVENTDEV_ERR("Null Return VQ received");
331 return 0;
332 }
333
334 num_pkts++;
335 } while (num_pkts < nb_events);
336
337 return num_pkts;
338 err:
339 for (n = 0; n < i; n++) {
340 evq_info = &dpaa2_portal->evq_info[n];
341 if (!evq_info->event_port)
342 continue;
343
344 qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0);
345 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0,
346 dpio_dev->token,
347 evq_info->dpcon->dpcon_id);
348 }
349 return 0;
350 }
351
352 static uint16_t
353 dpaa2_eventdev_dequeue(void *port, struct rte_event *ev,
354 uint64_t timeout_ticks)
355 {
356 return dpaa2_eventdev_dequeue_burst(port, ev, 1, timeout_ticks);
357 }
358
359 static void
360 dpaa2_eventdev_info_get(struct rte_eventdev *dev,
361 struct rte_event_dev_info *dev_info)
362 {
363 struct dpaa2_eventdev *priv = dev->data->dev_private;
364
365 EVENTDEV_INIT_FUNC_TRACE();
366
367 RTE_SET_USED(dev);
368
369 memset(dev_info, 0, sizeof(struct rte_event_dev_info));
370 dev_info->min_dequeue_timeout_ns =
371 DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT;
372 dev_info->max_dequeue_timeout_ns =
373 DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT;
374 dev_info->dequeue_timeout_ns =
375 DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS;
376 dev_info->max_event_queues = priv->max_event_queues;
377 dev_info->max_event_queue_flows =
378 DPAA2_EVENT_MAX_QUEUE_FLOWS;
379 dev_info->max_event_queue_priority_levels =
380 DPAA2_EVENT_MAX_QUEUE_PRIORITY_LEVELS;
381 dev_info->max_event_priority_levels =
382 DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS;
383 dev_info->max_event_ports = rte_fslmc_get_device_count(DPAA2_IO);
384 /* we only support dpio upto number of cores*/
385 if (dev_info->max_event_ports > rte_lcore_count())
386 dev_info->max_event_ports = rte_lcore_count();
387 dev_info->max_event_port_dequeue_depth =
388 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH;
389 dev_info->max_event_port_enqueue_depth =
390 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH;
391 dev_info->max_num_events = DPAA2_EVENT_MAX_NUM_EVENTS;
392 dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
393 RTE_EVENT_DEV_CAP_BURST_MODE|
394 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK |
395 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT |
396 RTE_EVENT_DEV_CAP_NONSEQ_MODE;
397
398 }
399
400 static int
401 dpaa2_eventdev_configure(const struct rte_eventdev *dev)
402 {
403 struct dpaa2_eventdev *priv = dev->data->dev_private;
404 struct rte_event_dev_config *conf = &dev->data->dev_conf;
405
406 EVENTDEV_INIT_FUNC_TRACE();
407
408 priv->nb_event_queues = conf->nb_event_queues;
409 priv->nb_event_ports = conf->nb_event_ports;
410 priv->nb_event_queue_flows = conf->nb_event_queue_flows;
411 priv->nb_event_port_dequeue_depth = conf->nb_event_port_dequeue_depth;
412 priv->nb_event_port_enqueue_depth = conf->nb_event_port_enqueue_depth;
413 priv->event_dev_cfg = conf->event_dev_cfg;
414
415 /* Check dequeue timeout method is per dequeue or global */
416 if (priv->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) {
417 /*
418 * Use timeout value as given in dequeue operation.
419 * So invalidating this timeout value.
420 */
421 priv->dequeue_timeout_ns = 0;
422
423 } else if (conf->dequeue_timeout_ns == 0) {
424 priv->dequeue_timeout_ns = DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS;
425 } else {
426 priv->dequeue_timeout_ns = conf->dequeue_timeout_ns;
427 }
428
429 DPAA2_EVENTDEV_DEBUG("Configured eventdev devid=%d",
430 dev->data->dev_id);
431 return 0;
432 }
433
434 static int
435 dpaa2_eventdev_start(struct rte_eventdev *dev)
436 {
437 EVENTDEV_INIT_FUNC_TRACE();
438
439 RTE_SET_USED(dev);
440
441 return 0;
442 }
443
444 static void
445 dpaa2_eventdev_stop(struct rte_eventdev *dev)
446 {
447 EVENTDEV_INIT_FUNC_TRACE();
448
449 RTE_SET_USED(dev);
450 }
451
452 static int
453 dpaa2_eventdev_close(struct rte_eventdev *dev)
454 {
455 EVENTDEV_INIT_FUNC_TRACE();
456
457 RTE_SET_USED(dev);
458
459 return 0;
460 }
461
462 static void
463 dpaa2_eventdev_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
464 struct rte_event_queue_conf *queue_conf)
465 {
466 EVENTDEV_INIT_FUNC_TRACE();
467
468 RTE_SET_USED(dev);
469 RTE_SET_USED(queue_id);
470 RTE_SET_USED(queue_conf);
471
472 queue_conf->nb_atomic_flows = DPAA2_EVENT_QUEUE_ATOMIC_FLOWS;
473 queue_conf->schedule_type = RTE_SCHED_TYPE_ATOMIC |
474 RTE_SCHED_TYPE_PARALLEL;
475 queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
476 }
477
478 static int
479 dpaa2_eventdev_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
480 const struct rte_event_queue_conf *queue_conf)
481 {
482 struct dpaa2_eventdev *priv = dev->data->dev_private;
483 struct dpaa2_eventq *evq_info = &priv->evq_info[queue_id];
484
485 EVENTDEV_INIT_FUNC_TRACE();
486
487 switch (queue_conf->schedule_type) {
488 case RTE_SCHED_TYPE_PARALLEL:
489 case RTE_SCHED_TYPE_ATOMIC:
490 break;
491 case RTE_SCHED_TYPE_ORDERED:
492 DPAA2_EVENTDEV_ERR("Schedule type is not supported.");
493 return -1;
494 }
495 evq_info->event_queue_cfg = queue_conf->event_queue_cfg;
496 evq_info->event_queue_id = queue_id;
497
498 return 0;
499 }
500
501 static void
502 dpaa2_eventdev_queue_release(struct rte_eventdev *dev, uint8_t queue_id)
503 {
504 EVENTDEV_INIT_FUNC_TRACE();
505
506 RTE_SET_USED(dev);
507 RTE_SET_USED(queue_id);
508 }
509
510 static void
511 dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
512 struct rte_event_port_conf *port_conf)
513 {
514 EVENTDEV_INIT_FUNC_TRACE();
515
516 RTE_SET_USED(dev);
517 RTE_SET_USED(port_id);
518
519 port_conf->new_event_threshold =
520 DPAA2_EVENT_MAX_NUM_EVENTS;
521 port_conf->dequeue_depth =
522 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH;
523 port_conf->enqueue_depth =
524 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH;
525 port_conf->disable_implicit_release = 0;
526 }
527
528 static int
529 dpaa2_eventdev_port_setup(struct rte_eventdev *dev, uint8_t port_id,
530 const struct rte_event_port_conf *port_conf)
531 {
532 char event_port_name[32];
533 struct dpaa2_port *portal;
534
535 EVENTDEV_INIT_FUNC_TRACE();
536
537 RTE_SET_USED(port_conf);
538
539 sprintf(event_port_name, "event-port-%d", port_id);
540 portal = rte_malloc(event_port_name, sizeof(struct dpaa2_port), 0);
541 if (!portal) {
542 DPAA2_EVENTDEV_ERR("Memory allocation failure");
543 return -ENOMEM;
544 }
545
546 memset(portal, 0, sizeof(struct dpaa2_port));
547 dev->data->ports[port_id] = portal;
548 return 0;
549 }
550
551 static void
552 dpaa2_eventdev_port_release(void *port)
553 {
554 struct dpaa2_port *portal = port;
555
556 EVENTDEV_INIT_FUNC_TRACE();
557
558 /* TODO: Cleanup is required when ports are in linked state. */
559 if (portal->is_port_linked)
560 DPAA2_EVENTDEV_WARN("Event port must be unlinked before release");
561
562 if (portal)
563 rte_free(portal);
564
565 portal = NULL;
566 }
567
568 static int
569 dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port,
570 const uint8_t queues[], const uint8_t priorities[],
571 uint16_t nb_links)
572 {
573 struct dpaa2_eventdev *priv = dev->data->dev_private;
574 struct dpaa2_port *dpaa2_portal = port;
575 struct dpaa2_eventq *evq_info;
576 uint16_t i;
577
578 EVENTDEV_INIT_FUNC_TRACE();
579
580 RTE_SET_USED(priorities);
581
582 for (i = 0; i < nb_links; i++) {
583 evq_info = &priv->evq_info[queues[i]];
584 memcpy(&dpaa2_portal->evq_info[queues[i]], evq_info,
585 sizeof(struct dpaa2_eventq));
586 dpaa2_portal->evq_info[queues[i]].event_port = port;
587 dpaa2_portal->num_linked_evq++;
588 }
589
590 return (int)nb_links;
591 }
592
593 static int
594 dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port,
595 uint8_t queues[], uint16_t nb_unlinks)
596 {
597 struct dpaa2_port *dpaa2_portal = port;
598 int i;
599 struct dpaa2_dpio_dev *dpio_dev = NULL;
600 struct dpaa2_eventq *evq_info;
601 struct qbman_swp *swp;
602
603 EVENTDEV_INIT_FUNC_TRACE();
604
605 RTE_SET_USED(dev);
606 RTE_SET_USED(queues);
607
608 for (i = 0; i < nb_unlinks; i++) {
609 evq_info = &dpaa2_portal->evq_info[queues[i]];
610
611 if (DPAA2_PER_LCORE_DPIO && evq_info->dpcon) {
612 /* todo dpaa2_portal shall have dpio_dev-no per lcore*/
613 dpio_dev = DPAA2_PER_LCORE_DPIO;
614 swp = DPAA2_PER_LCORE_PORTAL;
615
616 qbman_swp_push_set(swp,
617 evq_info->dpcon->channel_index, 0);
618 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0,
619 dpio_dev->token,
620 evq_info->dpcon->dpcon_id);
621 }
622 memset(evq_info, 0, sizeof(struct dpaa2_eventq));
623 if (dpaa2_portal->num_linked_evq)
624 dpaa2_portal->num_linked_evq--;
625 }
626
627 if (!dpaa2_portal->num_linked_evq)
628 dpaa2_portal->is_port_linked = false;
629
630 return (int)nb_unlinks;
631 }
632
633
634 static int
635 dpaa2_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
636 uint64_t *timeout_ticks)
637 {
638 uint32_t scale = 1000*1000;
639
640 EVENTDEV_INIT_FUNC_TRACE();
641
642 RTE_SET_USED(dev);
643 *timeout_ticks = ns * scale;
644
645 return 0;
646 }
647
648 static void
649 dpaa2_eventdev_dump(struct rte_eventdev *dev, FILE *f)
650 {
651 EVENTDEV_INIT_FUNC_TRACE();
652
653 RTE_SET_USED(dev);
654 RTE_SET_USED(f);
655 }
656
657 static int
658 dpaa2_eventdev_eth_caps_get(const struct rte_eventdev *dev,
659 const struct rte_eth_dev *eth_dev,
660 uint32_t *caps)
661 {
662 const char *ethdev_driver = eth_dev->device->driver->name;
663
664 EVENTDEV_INIT_FUNC_TRACE();
665
666 RTE_SET_USED(dev);
667
668 if (!strcmp(ethdev_driver, "net_dpaa2"))
669 *caps = RTE_EVENT_ETH_RX_ADAPTER_DPAA2_CAP;
670 else
671 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP;
672
673 return 0;
674 }
675
676 static int
677 dpaa2_eventdev_eth_queue_add_all(const struct rte_eventdev *dev,
678 const struct rte_eth_dev *eth_dev,
679 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
680 {
681 struct dpaa2_eventdev *priv = dev->data->dev_private;
682 uint8_t ev_qid = queue_conf->ev.queue_id;
683 uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id;
684 int i, ret;
685
686 EVENTDEV_INIT_FUNC_TRACE();
687
688 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
689 ret = dpaa2_eth_eventq_attach(eth_dev, i,
690 dpcon_id, queue_conf);
691 if (ret) {
692 DPAA2_EVENTDEV_ERR(
693 "Event queue attach failed: err(%d)", ret);
694 goto fail;
695 }
696 }
697 return 0;
698 fail:
699 for (i = (i - 1); i >= 0 ; i--)
700 dpaa2_eth_eventq_detach(eth_dev, i);
701
702 return ret;
703 }
704
705 static int
706 dpaa2_eventdev_eth_queue_add(const struct rte_eventdev *dev,
707 const struct rte_eth_dev *eth_dev,
708 int32_t rx_queue_id,
709 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
710 {
711 struct dpaa2_eventdev *priv = dev->data->dev_private;
712 uint8_t ev_qid = queue_conf->ev.queue_id;
713 uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id;
714 int ret;
715
716 EVENTDEV_INIT_FUNC_TRACE();
717
718 if (rx_queue_id == -1)
719 return dpaa2_eventdev_eth_queue_add_all(dev,
720 eth_dev, queue_conf);
721
722 ret = dpaa2_eth_eventq_attach(eth_dev, rx_queue_id,
723 dpcon_id, queue_conf);
724 if (ret) {
725 DPAA2_EVENTDEV_ERR(
726 "Event queue attach failed: err(%d)", ret);
727 return ret;
728 }
729 return 0;
730 }
731
732 static int
733 dpaa2_eventdev_eth_queue_del_all(const struct rte_eventdev *dev,
734 const struct rte_eth_dev *eth_dev)
735 {
736 int i, ret;
737
738 EVENTDEV_INIT_FUNC_TRACE();
739
740 RTE_SET_USED(dev);
741
742 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
743 ret = dpaa2_eth_eventq_detach(eth_dev, i);
744 if (ret) {
745 DPAA2_EVENTDEV_ERR(
746 "Event queue detach failed: err(%d)", ret);
747 return ret;
748 }
749 }
750
751 return 0;
752 }
753
754 static int
755 dpaa2_eventdev_eth_queue_del(const struct rte_eventdev *dev,
756 const struct rte_eth_dev *eth_dev,
757 int32_t rx_queue_id)
758 {
759 int ret;
760
761 EVENTDEV_INIT_FUNC_TRACE();
762
763 if (rx_queue_id == -1)
764 return dpaa2_eventdev_eth_queue_del_all(dev, eth_dev);
765
766 ret = dpaa2_eth_eventq_detach(eth_dev, rx_queue_id);
767 if (ret) {
768 DPAA2_EVENTDEV_ERR(
769 "Event queue detach failed: err(%d)", ret);
770 return ret;
771 }
772
773 return 0;
774 }
775
776 static int
777 dpaa2_eventdev_eth_start(const struct rte_eventdev *dev,
778 const struct rte_eth_dev *eth_dev)
779 {
780 EVENTDEV_INIT_FUNC_TRACE();
781
782 RTE_SET_USED(dev);
783 RTE_SET_USED(eth_dev);
784
785 return 0;
786 }
787
788 static int
789 dpaa2_eventdev_eth_stop(const struct rte_eventdev *dev,
790 const struct rte_eth_dev *eth_dev)
791 {
792 EVENTDEV_INIT_FUNC_TRACE();
793
794 RTE_SET_USED(dev);
795 RTE_SET_USED(eth_dev);
796
797 return 0;
798 }
799
800 #ifdef RTE_LIBRTE_SECURITY
801 static int
802 dpaa2_eventdev_crypto_caps_get(const struct rte_eventdev *dev,
803 const struct rte_cryptodev *cdev,
804 uint32_t *caps)
805 {
806 const char *name = cdev->data->name;
807
808 EVENTDEV_INIT_FUNC_TRACE();
809
810 RTE_SET_USED(dev);
811
812 if (!strncmp(name, "dpsec-", 6))
813 *caps = RTE_EVENT_CRYPTO_ADAPTER_DPAA2_CAP;
814 else
815 return -1;
816
817 return 0;
818 }
819
820 static int
821 dpaa2_eventdev_crypto_queue_add_all(const struct rte_eventdev *dev,
822 const struct rte_cryptodev *cryptodev,
823 const struct rte_event *ev)
824 {
825 struct dpaa2_eventdev *priv = dev->data->dev_private;
826 uint8_t ev_qid = ev->queue_id;
827 uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id;
828 int i, ret;
829
830 EVENTDEV_INIT_FUNC_TRACE();
831
832 for (i = 0; i < cryptodev->data->nb_queue_pairs; i++) {
833 ret = dpaa2_sec_eventq_attach(cryptodev, i,
834 dpcon_id, ev);
835 if (ret) {
836 DPAA2_EVENTDEV_ERR("dpaa2_sec_eventq_attach failed: ret %d\n",
837 ret);
838 goto fail;
839 }
840 }
841 return 0;
842 fail:
843 for (i = (i - 1); i >= 0 ; i--)
844 dpaa2_sec_eventq_detach(cryptodev, i);
845
846 return ret;
847 }
848
849 static int
850 dpaa2_eventdev_crypto_queue_add(const struct rte_eventdev *dev,
851 const struct rte_cryptodev *cryptodev,
852 int32_t rx_queue_id,
853 const struct rte_event *ev)
854 {
855 struct dpaa2_eventdev *priv = dev->data->dev_private;
856 uint8_t ev_qid = ev->queue_id;
857 uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id;
858 int ret;
859
860 EVENTDEV_INIT_FUNC_TRACE();
861
862 if (rx_queue_id == -1)
863 return dpaa2_eventdev_crypto_queue_add_all(dev,
864 cryptodev, ev);
865
866 ret = dpaa2_sec_eventq_attach(cryptodev, rx_queue_id,
867 dpcon_id, ev);
868 if (ret) {
869 DPAA2_EVENTDEV_ERR(
870 "dpaa2_sec_eventq_attach failed: ret: %d\n", ret);
871 return ret;
872 }
873 return 0;
874 }
875
876 static int
877 dpaa2_eventdev_crypto_queue_del_all(const struct rte_eventdev *dev,
878 const struct rte_cryptodev *cdev)
879 {
880 int i, ret;
881
882 EVENTDEV_INIT_FUNC_TRACE();
883
884 RTE_SET_USED(dev);
885
886 for (i = 0; i < cdev->data->nb_queue_pairs; i++) {
887 ret = dpaa2_sec_eventq_detach(cdev, i);
888 if (ret) {
889 DPAA2_EVENTDEV_ERR(
890 "dpaa2_sec_eventq_detach failed:ret %d\n", ret);
891 return ret;
892 }
893 }
894
895 return 0;
896 }
897
898 static int
899 dpaa2_eventdev_crypto_queue_del(const struct rte_eventdev *dev,
900 const struct rte_cryptodev *cryptodev,
901 int32_t rx_queue_id)
902 {
903 int ret;
904
905 EVENTDEV_INIT_FUNC_TRACE();
906
907 if (rx_queue_id == -1)
908 return dpaa2_eventdev_crypto_queue_del_all(dev, cryptodev);
909
910 ret = dpaa2_sec_eventq_detach(cryptodev, rx_queue_id);
911 if (ret) {
912 DPAA2_EVENTDEV_ERR(
913 "dpaa2_sec_eventq_detach failed: ret: %d\n", ret);
914 return ret;
915 }
916
917 return 0;
918 }
919
920 static int
921 dpaa2_eventdev_crypto_start(const struct rte_eventdev *dev,
922 const struct rte_cryptodev *cryptodev)
923 {
924 EVENTDEV_INIT_FUNC_TRACE();
925
926 RTE_SET_USED(dev);
927 RTE_SET_USED(cryptodev);
928
929 return 0;
930 }
931
932 static int
933 dpaa2_eventdev_crypto_stop(const struct rte_eventdev *dev,
934 const struct rte_cryptodev *cryptodev)
935 {
936 EVENTDEV_INIT_FUNC_TRACE();
937
938 RTE_SET_USED(dev);
939 RTE_SET_USED(cryptodev);
940
941 return 0;
942 }
943 #endif
944
945 static struct rte_eventdev_ops dpaa2_eventdev_ops = {
946 .dev_infos_get = dpaa2_eventdev_info_get,
947 .dev_configure = dpaa2_eventdev_configure,
948 .dev_start = dpaa2_eventdev_start,
949 .dev_stop = dpaa2_eventdev_stop,
950 .dev_close = dpaa2_eventdev_close,
951 .queue_def_conf = dpaa2_eventdev_queue_def_conf,
952 .queue_setup = dpaa2_eventdev_queue_setup,
953 .queue_release = dpaa2_eventdev_queue_release,
954 .port_def_conf = dpaa2_eventdev_port_def_conf,
955 .port_setup = dpaa2_eventdev_port_setup,
956 .port_release = dpaa2_eventdev_port_release,
957 .port_link = dpaa2_eventdev_port_link,
958 .port_unlink = dpaa2_eventdev_port_unlink,
959 .timeout_ticks = dpaa2_eventdev_timeout_ticks,
960 .dump = dpaa2_eventdev_dump,
961 .eth_rx_adapter_caps_get = dpaa2_eventdev_eth_caps_get,
962 .eth_rx_adapter_queue_add = dpaa2_eventdev_eth_queue_add,
963 .eth_rx_adapter_queue_del = dpaa2_eventdev_eth_queue_del,
964 .eth_rx_adapter_start = dpaa2_eventdev_eth_start,
965 .eth_rx_adapter_stop = dpaa2_eventdev_eth_stop,
966 #ifdef RTE_LIBRTE_SECURITY
967 .crypto_adapter_caps_get = dpaa2_eventdev_crypto_caps_get,
968 .crypto_adapter_queue_pair_add = dpaa2_eventdev_crypto_queue_add,
969 .crypto_adapter_queue_pair_del = dpaa2_eventdev_crypto_queue_del,
970 .crypto_adapter_start = dpaa2_eventdev_crypto_start,
971 .crypto_adapter_stop = dpaa2_eventdev_crypto_stop,
972 #endif
973 };
974
975 static int
976 dpaa2_eventdev_setup_dpci(struct dpaa2_dpci_dev *dpci_dev,
977 struct dpaa2_dpcon_dev *dpcon_dev)
978 {
979 struct dpci_rx_queue_cfg rx_queue_cfg;
980 int ret, i;
981
982 /*Do settings to get the frame on a DPCON object*/
983 rx_queue_cfg.options = DPCI_QUEUE_OPT_DEST |
984 DPCI_QUEUE_OPT_USER_CTX;
985 rx_queue_cfg.dest_cfg.dest_type = DPCI_DEST_DPCON;
986 rx_queue_cfg.dest_cfg.dest_id = dpcon_dev->dpcon_id;
987 rx_queue_cfg.dest_cfg.priority = DPAA2_EVENT_DEFAULT_DPCI_PRIO;
988
989 dpci_dev->rx_queue[DPAA2_EVENT_DPCI_PARALLEL_QUEUE].cb =
990 dpaa2_eventdev_process_parallel;
991 dpci_dev->rx_queue[DPAA2_EVENT_DPCI_ATOMIC_QUEUE].cb =
992 dpaa2_eventdev_process_atomic;
993
994 for (i = 0 ; i < DPAA2_EVENT_DPCI_MAX_QUEUES; i++) {
995 rx_queue_cfg.user_ctx = (size_t)(&dpci_dev->rx_queue[i]);
996 ret = dpci_set_rx_queue(&dpci_dev->dpci,
997 CMD_PRI_LOW,
998 dpci_dev->token, i,
999 &rx_queue_cfg);
1000 if (ret) {
1001 DPAA2_EVENTDEV_ERR(
1002 "DPCI Rx queue setup failed: err(%d)",
1003 ret);
1004 return ret;
1005 }
1006 }
1007 return 0;
1008 }
1009
1010 static int
1011 dpaa2_eventdev_create(const char *name)
1012 {
1013 struct rte_eventdev *eventdev;
1014 struct dpaa2_eventdev *priv;
1015 struct dpaa2_dpcon_dev *dpcon_dev = NULL;
1016 struct dpaa2_dpci_dev *dpci_dev = NULL;
1017 int ret;
1018
1019 eventdev = rte_event_pmd_vdev_init(name,
1020 sizeof(struct dpaa2_eventdev),
1021 rte_socket_id());
1022 if (eventdev == NULL) {
1023 DPAA2_EVENTDEV_ERR("Failed to create Event device %s", name);
1024 goto fail;
1025 }
1026
1027 eventdev->dev_ops = &dpaa2_eventdev_ops;
1028 eventdev->enqueue = dpaa2_eventdev_enqueue;
1029 eventdev->enqueue_burst = dpaa2_eventdev_enqueue_burst;
1030 eventdev->enqueue_new_burst = dpaa2_eventdev_enqueue_burst;
1031 eventdev->enqueue_forward_burst = dpaa2_eventdev_enqueue_burst;
1032 eventdev->dequeue = dpaa2_eventdev_dequeue;
1033 eventdev->dequeue_burst = dpaa2_eventdev_dequeue_burst;
1034
1035 /* For secondary processes, the primary has done all the work */
1036 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1037 return 0;
1038
1039 priv = eventdev->data->dev_private;
1040 priv->max_event_queues = 0;
1041
1042 do {
1043 dpcon_dev = rte_dpaa2_alloc_dpcon_dev();
1044 if (!dpcon_dev)
1045 break;
1046 priv->evq_info[priv->max_event_queues].dpcon = dpcon_dev;
1047
1048 dpci_dev = rte_dpaa2_alloc_dpci_dev();
1049 if (!dpci_dev) {
1050 rte_dpaa2_free_dpcon_dev(dpcon_dev);
1051 break;
1052 }
1053 priv->evq_info[priv->max_event_queues].dpci = dpci_dev;
1054
1055 ret = dpaa2_eventdev_setup_dpci(dpci_dev, dpcon_dev);
1056 if (ret) {
1057 DPAA2_EVENTDEV_ERR(
1058 "DPCI setup failed: err(%d)", ret);
1059 return ret;
1060 }
1061 priv->max_event_queues++;
1062 } while (dpcon_dev && dpci_dev);
1063
1064 RTE_LOG(INFO, PMD, "%s eventdev created\n", name);
1065
1066 return 0;
1067 fail:
1068 return -EFAULT;
1069 }
1070
1071 static int
1072 dpaa2_eventdev_probe(struct rte_vdev_device *vdev)
1073 {
1074 const char *name;
1075
1076 name = rte_vdev_device_name(vdev);
1077 DPAA2_EVENTDEV_INFO("Initializing %s", name);
1078 return dpaa2_eventdev_create(name);
1079 }
1080
1081 static int
1082 dpaa2_eventdev_remove(struct rte_vdev_device *vdev)
1083 {
1084 const char *name;
1085
1086 name = rte_vdev_device_name(vdev);
1087 DPAA2_EVENTDEV_INFO("Closing %s", name);
1088
1089 return rte_event_pmd_vdev_uninit(name);
1090 }
1091
1092 static struct rte_vdev_driver vdev_eventdev_dpaa2_pmd = {
1093 .probe = dpaa2_eventdev_probe,
1094 .remove = dpaa2_eventdev_remove
1095 };
1096
1097 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DPAA2_PMD, vdev_eventdev_dpaa2_pmd);
1098
1099 RTE_INIT(dpaa2_eventdev_init_log)
1100 {
1101 dpaa2_logtype_event = rte_log_register("pmd.event.dpaa2");
1102 if (dpaa2_logtype_event >= 0)
1103 rte_log_set_level(dpaa2_logtype_event, RTE_LOG_NOTICE);
1104 }
Ceph