]> git.proxmox.com Git - ceph.git/blame - ceph/src/spdk/dpdk/drivers/event/sw/sw_evdev.c
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update source to Ceph Pacific 16.2.2
[ceph.git] / ceph / src / spdk / dpdk / drivers / event / sw / sw_evdev.c
CommitLineData
11fdf7f2
TL		 1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2017 Intel Corporation
3 */
4
5#include <inttypes.h>
6#include <string.h>
7
8#include <rte_bus_vdev.h>
9#include <rte_kvargs.h>
10#include <rte_ring.h>
11#include <rte_errno.h>
12#include <rte_event_ring.h>
13#include <rte_service_component.h>
14
15#include "sw_evdev.h"
16#include "iq_chunk.h"
17
18#define EVENTDEV_NAME_SW_PMD event_sw
19#define NUMA_NODE_ARG "numa_node"
20#define SCHED_QUANTA_ARG "sched_quanta"
21#define CREDIT_QUANTA_ARG "credit_quanta"
22
23static void
24sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info);
25
26static int
27sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[],
28 const uint8_t priorities[], uint16_t num)
29{
30 struct sw_port *p = port;
31 struct sw_evdev *sw = sw_pmd_priv(dev);
32 int i;
33
34 RTE_SET_USED(priorities);
35 for (i = 0; i < num; i++) {
36 struct sw_qid *q = &sw->qids[queues[i]];
37 unsigned int j;
38
39 /* check for qid map overflow */
40 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) {
f67539c2 41 rte_errno = EDQUOT;
11fdf7f2
TL		 42 break;
43 }
44
45 if (p->is_directed && p->num_qids_mapped > 0) {
f67539c2 46 rte_errno = EDQUOT;
11fdf7f2
TL		 47 break;
48 }
49
50 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
51 if (q->cq_map[j] == p->id)
52 break;
53 }
54
55 /* check if port is already linked */
56 if (j < q->cq_num_mapped_cqs)
57 continue;
58
59 if (q->type == SW_SCHED_TYPE_DIRECT) {
60 /* check directed qids only map to one port */
61 if (p->num_qids_mapped > 0) {
f67539c2 62 rte_errno = EDQUOT;
11fdf7f2
TL		 63 break;
64 }
65 /* check port only takes a directed flow */
66 if (num > 1) {
f67539c2 67 rte_errno = EDQUOT;
11fdf7f2
TL		 68 break;
69 }
70
71 p->is_directed = 1;
72 p->num_qids_mapped = 1;
73 } else if (q->type == RTE_SCHED_TYPE_ORDERED) {
74 p->num_ordered_qids++;
75 p->num_qids_mapped++;
76 } else if (q->type == RTE_SCHED_TYPE_ATOMIC ||
77 q->type == RTE_SCHED_TYPE_PARALLEL) {
78 p->num_qids_mapped++;
79 }
80
81 q->cq_map[q->cq_num_mapped_cqs] = p->id;
82 rte_smp_wmb();
83 q->cq_num_mapped_cqs++;
84 }
85 return i;
86}
87
88static int
89sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
90 uint16_t nb_unlinks)
91{
92 struct sw_port *p = port;
93 struct sw_evdev *sw = sw_pmd_priv(dev);
94 unsigned int i, j;
95
96 int unlinked = 0;
97 for (i = 0; i < nb_unlinks; i++) {
98 struct sw_qid *q = &sw->qids[queues[i]];
99 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
100 if (q->cq_map[j] == p->id) {
101 q->cq_map[j] =
102 q->cq_map[q->cq_num_mapped_cqs - 1];
103 rte_smp_wmb();
104 q->cq_num_mapped_cqs--;
105 unlinked++;
106
107 p->num_qids_mapped--;
108
109 if (q->type == RTE_SCHED_TYPE_ORDERED)
110 p->num_ordered_qids--;
111
112 continue;
113 }
114 }
115 }
9f95a23c
TL		 116
117 p->unlinks_in_progress += unlinked;
118 rte_smp_mb();
119
11fdf7f2
TL		 120 return unlinked;
121}
122
9f95a23c
TL		 123static int
124sw_port_unlinks_in_progress(struct rte_eventdev *dev, void *port)
125{
126 RTE_SET_USED(dev);
127 struct sw_port *p = port;
128 return p->unlinks_in_progress;
129}
130
11fdf7f2
TL		 131static int
132sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
133 const struct rte_event_port_conf *conf)
134{
135 struct sw_evdev *sw = sw_pmd_priv(dev);
136 struct sw_port *p = &sw->ports[port_id];
137 char buf[RTE_RING_NAMESIZE];
138 unsigned int i;
139
140 struct rte_event_dev_info info;
141 sw_info_get(dev, &info);
142
143 /* detect re-configuring and return credits to instance if needed */
144 if (p->initialized) {
145 /* taking credits from pool is done one quanta at a time, and
146 * credits may be spend (counted in p->inflights) or still
147 * available in the port (p->inflight_credits). We must return
148 * the sum to no leak credits
149 */
150 int possible_inflights = p->inflight_credits + p->inflights;
151 rte_atomic32_sub(&sw->inflights, possible_inflights);
152 }
153
154 *p = (struct sw_port){0}; /* zero entire structure */
155 p->id = port_id;
156 p->sw = sw;
157
158 /* check to see if rings exists - port_setup() can be called multiple
159 * times legally (assuming device is stopped). If ring exists, free it
160 * to so it gets re-created with the correct size
161 */
162 snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id,
163 port_id, "rx_worker_ring");
164 struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf);
165 if (existing_ring)
166 rte_event_ring_free(existing_ring);
167
168 p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH,
169 dev->data->socket_id,
170 RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
171 if (p->rx_worker_ring == NULL) {
172 SW_LOG_ERR("Error creating RX worker ring for port %d\n",
173 port_id);
174 return -1;
175 }
176
177 p->inflight_max = conf->new_event_threshold;
178 p->implicit_release = !conf->disable_implicit_release;
179
180 /* check if ring exists, same as rx_worker above */
181 snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
182 port_id, "cq_worker_ring");
183 existing_ring = rte_event_ring_lookup(buf);
184 if (existing_ring)
185 rte_event_ring_free(existing_ring);
186
187 p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth,
188 dev->data->socket_id,
189 RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
190 if (p->cq_worker_ring == NULL) {
191 rte_event_ring_free(p->rx_worker_ring);
192 SW_LOG_ERR("Error creating CQ worker ring for port %d\n",
193 port_id);
194 return -1;
195 }
196 sw->cq_ring_space[port_id] = conf->dequeue_depth;
197
198 /* set hist list contents to empty */
199 for (i = 0; i < SW_PORT_HIST_LIST; i++) {
200 p->hist_list[i].fid = -1;
201 p->hist_list[i].qid = -1;
202 }
203 dev->data->ports[port_id] = p;
204
205 rte_smp_wmb();
206 p->initialized = 1;
207 return 0;
208}
209
210static void
211sw_port_release(void *port)
212{
213 struct sw_port *p = (void *)port;
214 if (p == NULL)
215 return;
216
217 rte_event_ring_free(p->rx_worker_ring);
218 rte_event_ring_free(p->cq_worker_ring);
219 memset(p, 0, sizeof(*p));
220}
221
222static int32_t
223qid_init(struct sw_evdev *sw, unsigned int idx, int type,
224 const struct rte_event_queue_conf *queue_conf)
225{
226 unsigned int i;
227 int dev_id = sw->data->dev_id;
228 int socket_id = sw->data->socket_id;
229 char buf[IQ_ROB_NAMESIZE];
230 struct sw_qid *qid = &sw->qids[idx];
231
232 /* Initialize the FID structures to no pinning (-1), and zero packets */
233 const struct sw_fid_t fid = {.cq = -1, .pcount = 0};
234 for (i = 0; i < RTE_DIM(qid->fids); i++)
235 qid->fids[i] = fid;
236
237 qid->id = idx;
238 qid->type = type;
239 qid->priority = queue_conf->priority;
240
241 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
242 char ring_name[RTE_RING_NAMESIZE];
243 uint32_t window_size;
244
245 /* rte_ring and window_size_mask require require window_size to
246 * be a power-of-2.
247 */
248 window_size = rte_align32pow2(
249 queue_conf->nb_atomic_order_sequences);
250
251 qid->window_size = window_size - 1;
252
253 if (!window_size) {
254 SW_LOG_DBG(
255 "invalid reorder_window_size for ordered queue\n"
256 );
257 goto cleanup;
258 }
259
260 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i);
261 qid->reorder_buffer = rte_zmalloc_socket(buf,
262 window_size * sizeof(qid->reorder_buffer[0]),
263 0, socket_id);
264 if (!qid->reorder_buffer) {
265 SW_LOG_DBG("reorder_buffer malloc failed\n");
266 goto cleanup;
267 }
268
269 memset(&qid->reorder_buffer[0],
270 0,
271 window_size * sizeof(qid->reorder_buffer[0]));
272
273 snprintf(ring_name, sizeof(ring_name), "sw%d_q%d_freelist",
274 dev_id, idx);
275
276 /* lookup the ring, and if it already exists, free it */
277 struct rte_ring *cleanup = rte_ring_lookup(ring_name);
278 if (cleanup)
279 rte_ring_free(cleanup);
280
281 qid->reorder_buffer_freelist = rte_ring_create(ring_name,
282 window_size,
283 socket_id,
284 RING_F_SP_ENQ | RING_F_SC_DEQ);
285 if (!qid->reorder_buffer_freelist) {
286 SW_LOG_DBG("freelist ring create failed");
287 goto cleanup;
288 }
289
290 /* Populate the freelist with reorder buffer entries. Enqueue
291 * 'window_size - 1' entries because the rte_ring holds only
292 * that many.
293 */
294 for (i = 0; i < window_size - 1; i++) {
295 if (rte_ring_sp_enqueue(qid->reorder_buffer_freelist,
296 &qid->reorder_buffer[i]) < 0)
297 goto cleanup;
298 }
299
300 qid->reorder_buffer_index = 0;
301 qid->cq_next_tx = 0;
302 }
303
304 qid->initialized = 1;
305
306 return 0;
307
308cleanup:
309 if (qid->reorder_buffer) {
310 rte_free(qid->reorder_buffer);
311 qid->reorder_buffer = NULL;
312 }
313
314 if (qid->reorder_buffer_freelist) {
315 rte_ring_free(qid->reorder_buffer_freelist);
316 qid->reorder_buffer_freelist = NULL;
317 }
318
319 return -EINVAL;
320}
321
322static void
323sw_queue_release(struct rte_eventdev *dev, uint8_t id)
324{
325 struct sw_evdev *sw = sw_pmd_priv(dev);
326 struct sw_qid *qid = &sw->qids[id];
327
328 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
329 rte_free(qid->reorder_buffer);
330 rte_ring_free(qid->reorder_buffer_freelist);
331 }
332 memset(qid, 0, sizeof(*qid));
333}
334
335static int
336sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
337 const struct rte_event_queue_conf *conf)
338{
339 int type;
340
341 type = conf->schedule_type;
342
343 if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) {
344 type = SW_SCHED_TYPE_DIRECT;
345 } else if (RTE_EVENT_QUEUE_CFG_ALL_TYPES
346 & conf->event_queue_cfg) {
347 SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n");
348 return -ENOTSUP;
349 }
350
351 struct sw_evdev *sw = sw_pmd_priv(dev);
352
353 if (sw->qids[queue_id].initialized)
354 sw_queue_release(dev, queue_id);
355
356 return qid_init(sw, queue_id, type, conf);
357}
358
359static void
360sw_init_qid_iqs(struct sw_evdev *sw)
361{
362 int i, j;
363
364 /* Initialize the IQ memory of all configured qids */
365 for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) {
366 struct sw_qid *qid = &sw->qids[i];
367
368 if (!qid->initialized)
369 continue;
370
371 for (j = 0; j < SW_IQS_MAX; j++)
372 iq_init(sw, &qid->iq[j]);
373 }
374}
375
376static int
377sw_qids_empty(struct sw_evdev *sw)
378{
379 unsigned int i, j;
380
381 for (i = 0; i < sw->qid_count; i++) {
382 for (j = 0; j < SW_IQS_MAX; j++) {
383 if (iq_count(&sw->qids[i].iq[j]))
384 return 0;
385 }
386 }
387
388 return 1;
389}
390
391static int
392sw_ports_empty(struct sw_evdev *sw)
393{
394 unsigned int i;
395
396 for (i = 0; i < sw->port_count; i++) {
397 if ((rte_event_ring_count(sw->ports[i].rx_worker_ring)) ||
398 rte_event_ring_count(sw->ports[i].cq_worker_ring))
399 return 0;
400 }
401
402 return 1;
403}
404
405static void
406sw_drain_ports(struct rte_eventdev *dev)
407{
408 struct sw_evdev *sw = sw_pmd_priv(dev);
409 eventdev_stop_flush_t flush;
410 unsigned int i;
411 uint8_t dev_id;
412 void *arg;
413
414 flush = dev->dev_ops->dev_stop_flush;
415 dev_id = dev->data->dev_id;
416 arg = dev->data->dev_stop_flush_arg;
417
418 for (i = 0; i < sw->port_count; i++) {
419 struct rte_event ev;
420
421 while (rte_event_dequeue_burst(dev_id, i, &ev, 1, 0)) {
422 if (flush)
423 flush(dev_id, ev, arg);
424
425 ev.op = RTE_EVENT_OP_RELEASE;
426 rte_event_enqueue_burst(dev_id, i, &ev, 1);
427 }
428 }
429}
430
431static void
432sw_drain_queue(struct rte_eventdev *dev, struct sw_iq *iq)
433{
434 struct sw_evdev *sw = sw_pmd_priv(dev);
435 eventdev_stop_flush_t flush;
436 uint8_t dev_id;
437 void *arg;
438
439 flush = dev->dev_ops->dev_stop_flush;
440 dev_id = dev->data->dev_id;
441 arg = dev->data->dev_stop_flush_arg;
442
443 while (iq_count(iq) > 0) {
444 struct rte_event ev;
445
446 iq_dequeue_burst(sw, iq, &ev, 1);
447
448 if (flush)
449 flush(dev_id, ev, arg);
450 }
451}
452
453static void
454sw_drain_queues(struct rte_eventdev *dev)
455{
456 struct sw_evdev *sw = sw_pmd_priv(dev);
457 unsigned int i, j;
458
459 for (i = 0; i < sw->qid_count; i++) {
460 for (j = 0; j < SW_IQS_MAX; j++)
461 sw_drain_queue(dev, &sw->qids[i].iq[j]);
462 }
463}
464
465static void
466sw_clean_qid_iqs(struct rte_eventdev *dev)
467{
468 struct sw_evdev *sw = sw_pmd_priv(dev);
469 int i, j;
470
471 /* Release the IQ memory of all configured qids */
472 for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) {
473 struct sw_qid *qid = &sw->qids[i];
474
475 for (j = 0; j < SW_IQS_MAX; j++) {
476 if (!qid->iq[j].head)
477 continue;
478 iq_free_chunk_list(sw, qid->iq[j].head);
479 qid->iq[j].head = NULL;
480 }
481 }
482}
483
484static void
485sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
486 struct rte_event_queue_conf *conf)
487{
488 RTE_SET_USED(dev);
489 RTE_SET_USED(queue_id);
490
491 static const struct rte_event_queue_conf default_conf = {
492 .nb_atomic_flows = 4096,
493 .nb_atomic_order_sequences = 1,
494 .schedule_type = RTE_SCHED_TYPE_ATOMIC,
495 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
496 };
497
498 *conf = default_conf;
499}
500
501static void
502sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
503 struct rte_event_port_conf *port_conf)
504{
505 RTE_SET_USED(dev);
506 RTE_SET_USED(port_id);
507
508 port_conf->new_event_threshold = 1024;
509 port_conf->dequeue_depth = 16;
510 port_conf->enqueue_depth = 16;
511 port_conf->disable_implicit_release = 0;
512}
513
514static int
515sw_dev_configure(const struct rte_eventdev *dev)
516{
517 struct sw_evdev *sw = sw_pmd_priv(dev);
518 const struct rte_eventdev_data *data = dev->data;
519 const struct rte_event_dev_config *conf = &data->dev_conf;
520 int num_chunks, i;
521
522 sw->qid_count = conf->nb_event_queues;
523 sw->port_count = conf->nb_event_ports;
524 sw->nb_events_limit = conf->nb_events_limit;
525 rte_atomic32_set(&sw->inflights, 0);
526
527 /* Number of chunks sized for worst-case spread of events across IQs */
528 num_chunks = ((SW_INFLIGHT_EVENTS_TOTAL/SW_EVS_PER_Q_CHUNK)+1) +
529 sw->qid_count*SW_IQS_MAX*2;
530
531 /* If this is a reconfiguration, free the previous IQ allocation. All
532 * IQ chunk references were cleaned out of the QIDs in sw_stop(), and
533 * will be reinitialized in sw_start().
534 */
535 if (sw->chunks)
536 rte_free(sw->chunks);
537
538 sw->chunks = rte_malloc_socket(NULL,
539 sizeof(struct sw_queue_chunk) *
540 num_chunks,
541 0,
542 sw->data->socket_id);
543 if (!sw->chunks)
544 return -ENOMEM;
545
546 sw->chunk_list_head = NULL;
547 for (i = 0; i < num_chunks; i++)
548 iq_free_chunk(sw, &sw->chunks[i]);
549
550 if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
551 return -ENOTSUP;
552
553 return 0;
554}
555
556struct rte_eth_dev;
557
558static int
559sw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev,
560 const struct rte_eth_dev *eth_dev,
561 uint32_t *caps)
562{
563 RTE_SET_USED(dev);
564 RTE_SET_USED(eth_dev);
565 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP;
566 return 0;
567}
568
569static int
570sw_timer_adapter_caps_get(const struct rte_eventdev *dev,
571 uint64_t flags,
572 uint32_t *caps,
573 const struct rte_event_timer_adapter_ops **ops)
574{
575 RTE_SET_USED(dev);
576 RTE_SET_USED(flags);
577 *caps = 0;
578
579 /* Use default SW ops */
580 *ops = NULL;
581
582 return 0;
583}
584
585static int
586sw_crypto_adapter_caps_get(const struct rte_eventdev *dev,
587 const struct rte_cryptodev *cdev,
588 uint32_t *caps)
589{
590 RTE_SET_USED(dev);
591 RTE_SET_USED(cdev);
592 *caps = RTE_EVENT_CRYPTO_ADAPTER_SW_CAP;
593 return 0;
594}
595
596static void
597sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info)
598{
599 RTE_SET_USED(dev);
600
601 static const struct rte_event_dev_info evdev_sw_info = {
602 .driver_name = SW_PMD_NAME,
603 .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
604 .max_event_queue_flows = SW_QID_NUM_FIDS,
605 .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX,
606 .max_event_priority_levels = SW_IQS_MAX,
607 .max_event_ports = SW_PORTS_MAX,
608 .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH,
609 .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH,
610 .max_num_events = SW_INFLIGHT_EVENTS_TOTAL,
611 .event_dev_cap = (
612 RTE_EVENT_DEV_CAP_QUEUE_QOS |
613 RTE_EVENT_DEV_CAP_BURST_MODE |
614 RTE_EVENT_DEV_CAP_EVENT_QOS |
615 RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE|
616 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK |
617 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT |
618 RTE_EVENT_DEV_CAP_NONSEQ_MODE),
619 };
620
621 *info = evdev_sw_info;
622}
623
624static void
625sw_dump(struct rte_eventdev *dev, FILE *f)
626{
627 const struct sw_evdev *sw = sw_pmd_priv(dev);
628
629 static const char * const q_type_strings[] = {
630 "Ordered", "Atomic", "Parallel", "Directed"
631 };
632 uint32_t i;
633 fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name",
634 sw->port_count, sw->qid_count);
635
636 fprintf(f, "\trx %"PRIu64"\n\tdrop %"PRIu64"\n\ttx %"PRIu64"\n",
637 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts);
638 fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called);
639 fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called);
640 fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues);
641 fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues);
642 uint32_t inflights = rte_atomic32_read(&sw->inflights);
643 uint32_t credits = sw->nb_events_limit - inflights;
644 fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits);
645
646#define COL_RED "\x1b[31m"
647#define COL_RESET "\x1b[0m"
648
649 for (i = 0; i < sw->port_count; i++) {
650 int max, j;
651 const struct sw_port *p = &sw->ports[i];
652 if (!p->initialized) {
653 fprintf(f, " %sPort %d not initialized.%s\n",
654 COL_RED, i, COL_RESET);
655 continue;
656 }
657 fprintf(f, " Port %d %s\n", i,
658 p->is_directed ? " (SingleCons)" : "");
659 fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64
660 "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts,
661 sw->ports[i].stats.rx_dropped,
662 sw->ports[i].stats.tx_pkts,
663 (p->inflights == p->inflight_max) ?
664 COL_RED : COL_RESET,
665 sw->ports[i].inflights, COL_RESET);
666
667 fprintf(f, "\tMax New: %u"
668 "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n",
669 sw->ports[i].inflight_max,
670 sw->ports[i].avg_pkt_ticks,
671 sw->ports[i].inflight_credits);
672 fprintf(f, "\tReceive burst distribution:\n");
673 float zp_percent = p->zero_polls * 100.0 / p->total_polls;
674 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ",
675 zp_percent);
676 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;)
677 if (p->poll_buckets[max] != 0)
678 break;
679 for (j = 0; j <= max; j++) {
680 if (p->poll_buckets[j] != 0) {
681 float poll_pc = p->poll_buckets[j] * 100.0 /
682 p->total_polls;
683 fprintf(f, "%u-%u:%.02f%% ",
684 ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1),
685 ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT),
686 poll_pc);
687 }
688 }
689 fprintf(f, "\n");
690
691 if (p->rx_worker_ring) {
692 uint64_t used = rte_event_ring_count(p->rx_worker_ring);
693 uint64_t space = rte_event_ring_free_count(
694 p->rx_worker_ring);
695 const char *col = (space == 0) ? COL_RED : COL_RESET;
696 fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4"
697 PRIu64 COL_RESET"\n", col, used, space);
698 } else
699 fprintf(f, "\trx ring not initialized.\n");
700
701 if (p->cq_worker_ring) {
702 uint64_t used = rte_event_ring_count(p->cq_worker_ring);
703 uint64_t space = rte_event_ring_free_count(
704 p->cq_worker_ring);
705 const char *col = (space == 0) ? COL_RED : COL_RESET;
706 fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4"
707 PRIu64 COL_RESET"\n", col, used, space);
708 } else
709 fprintf(f, "\tcq ring not initialized.\n");
710 }
711
712 for (i = 0; i < sw->qid_count; i++) {
713 const struct sw_qid *qid = &sw->qids[i];
714 if (!qid->initialized) {
715 fprintf(f, " %sQueue %d not initialized.%s\n",
716 COL_RED, i, COL_RESET);
717 continue;
718 }
719 int affinities_per_port[SW_PORTS_MAX] = {0};
720 uint32_t inflights = 0;
721
722 fprintf(f, " Queue %d (%s)\n", i, q_type_strings[qid->type]);
723 fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64"\n",
724 qid->stats.rx_pkts, qid->stats.rx_dropped,
725 qid->stats.tx_pkts);
726 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
727 struct rte_ring *rob_buf_free =
728 qid->reorder_buffer_freelist;
729 if (rob_buf_free)
730 fprintf(f, "\tReorder entries in use: %u\n",
731 rte_ring_free_count(rob_buf_free));
732 else
733 fprintf(f,
734 "\tReorder buffer not initialized\n");
735 }
736
737 uint32_t flow;
738 for (flow = 0; flow < RTE_DIM(qid->fids); flow++)
739 if (qid->fids[flow].cq != -1) {
740 affinities_per_port[qid->fids[flow].cq]++;
741 inflights += qid->fids[flow].pcount;
742 }
743
744 uint32_t port;
745 fprintf(f, "\tPer Port Stats:\n");
746 for (port = 0; port < sw->port_count; port++) {
747 fprintf(f, "\t Port %d: Pkts: %"PRIu64, port,
748 qid->to_port[port]);
749 fprintf(f, "\tFlows: %d\n", affinities_per_port[port]);
750 }
751
752 uint32_t iq;
753 uint32_t iq_printed = 0;
754 for (iq = 0; iq < SW_IQS_MAX; iq++) {
755 if (!qid->iq[iq].head) {
756 fprintf(f, "\tiq %d is not initialized.\n", iq);
757 iq_printed = 1;
758 continue;
759 }
760 uint32_t used = iq_count(&qid->iq[iq]);
761 const char *col = COL_RESET;
762 if (used > 0) {
763 fprintf(f, "\t%siq %d: Used %d"
764 COL_RESET"\n", col, iq, used);
765 iq_printed = 1;
766 }
767 }
768 if (iq_printed == 0)
769 fprintf(f, "\t-- iqs empty --\n");
770 }
771}
772
773static int
774sw_start(struct rte_eventdev *dev)
775{
776 unsigned int i, j;
777 struct sw_evdev *sw = sw_pmd_priv(dev);
778
779 rte_service_component_runstate_set(sw->service_id, 1);
780
781 /* check a service core is mapped to this service */
782 if (!rte_service_runstate_get(sw->service_id)) {
783 SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
784 sw->service_name);
785 return -ENOENT;
786 }
787
788 /* check all ports are set up */
789 for (i = 0; i < sw->port_count; i++)
790 if (sw->ports[i].rx_worker_ring == NULL) {
791 SW_LOG_ERR("Port %d not configured\n", i);
792 return -ESTALE;
793 }
794
795 /* check all queues are configured and mapped to ports*/
796 for (i = 0; i < sw->qid_count; i++)
797 if (!sw->qids[i].initialized ||
798 sw->qids[i].cq_num_mapped_cqs == 0) {
799 SW_LOG_ERR("Queue %d not configured\n", i);
800 return -ENOLINK;
801 }
802
803 /* build up our prioritized array of qids */
804 /* We don't use qsort here, as if all/multiple entries have the same
805 * priority, the result is non-deterministic. From "man 3 qsort":
806 * "If two members compare as equal, their order in the sorted
807 * array is undefined."
808 */
809 uint32_t qidx = 0;
810 for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) {
811 for (i = 0; i < sw->qid_count; i++) {
812 if (sw->qids[i].priority == j) {
813 sw->qids_prioritized[qidx] = &sw->qids[i];
814 qidx++;
815 }
816 }
817 }
818
819 sw_init_qid_iqs(sw);
820
821 if (sw_xstats_init(sw) < 0)
822 return -EINVAL;
823
824 rte_smp_wmb();
825 sw->started = 1;
826
827 return 0;
828}
829
830static void
831sw_stop(struct rte_eventdev *dev)
832{
833 struct sw_evdev *sw = sw_pmd_priv(dev);
834 int32_t runstate;
835
836 /* Stop the scheduler if it's running */
837 runstate = rte_service_runstate_get(sw->service_id);
838 if (runstate == 1)
839 rte_service_runstate_set(sw->service_id, 0);
840
841 while (rte_service_may_be_active(sw->service_id))
842 rte_pause();
843
844 /* Flush all events out of the device */
845 while (!(sw_qids_empty(sw) && sw_ports_empty(sw))) {
846 sw_event_schedule(dev);
847 sw_drain_ports(dev);
848 sw_drain_queues(dev);
849 }
850
851 sw_clean_qid_iqs(dev);
852 sw_xstats_uninit(sw);
853 sw->started = 0;
854 rte_smp_wmb();
855
856 if (runstate == 1)
857 rte_service_runstate_set(sw->service_id, 1);
858}
859
860static int
861sw_close(struct rte_eventdev *dev)
862{
863 struct sw_evdev *sw = sw_pmd_priv(dev);
864 uint32_t i;
865
866 for (i = 0; i < sw->qid_count; i++)
867 sw_queue_release(dev, i);
868 sw->qid_count = 0;
869
870 for (i = 0; i < sw->port_count; i++)
871 sw_port_release(&sw->ports[i]);
872 sw->port_count = 0;
873
874 memset(&sw->stats, 0, sizeof(sw->stats));
875 sw->sched_called = 0;
876 sw->sched_no_iq_enqueues = 0;
877 sw->sched_no_cq_enqueues = 0;
878 sw->sched_cq_qid_called = 0;
879
880 return 0;
881}
882
883static int
884assign_numa_node(const char *key __rte_unused, const char *value, void *opaque)
885{
886 int *socket_id = opaque;
887 *socket_id = atoi(value);
888 if (*socket_id >= RTE_MAX_NUMA_NODES)
889 return -1;
890 return 0;
891}
892
893static int
894set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque)
895{
896 int *quanta = opaque;
897 *quanta = atoi(value);
898 if (*quanta < 0 || *quanta >= 4096)
899 return -1;
900 return 0;
901}
902
903static int
904set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
905{
906 int *credit = opaque;
907 *credit = atoi(value);
908 if (*credit < 0 || *credit >= 128)
909 return -1;
910 return 0;
911}
912
913
914static int32_t sw_sched_service_func(void *args)
915{
916 struct rte_eventdev *dev = args;
917 sw_event_schedule(dev);
918 return 0;
919}
920
921static int
922sw_probe(struct rte_vdev_device *vdev)
923{
924 static struct rte_eventdev_ops evdev_sw_ops = {
925 .dev_configure = sw_dev_configure,
926 .dev_infos_get = sw_info_get,
927 .dev_close = sw_close,
928 .dev_start = sw_start,
929 .dev_stop = sw_stop,
930 .dump = sw_dump,
931
932 .queue_def_conf = sw_queue_def_conf,
933 .queue_setup = sw_queue_setup,
934 .queue_release = sw_queue_release,
935 .port_def_conf = sw_port_def_conf,
936 .port_setup = sw_port_setup,
937 .port_release = sw_port_release,
938 .port_link = sw_port_link,
939 .port_unlink = sw_port_unlink,
9f95a23c 940 .port_unlinks_in_progress = sw_port_unlinks_in_progress,
11fdf7f2
TL		 941
942 .eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get,
943
944 .timer_adapter_caps_get = sw_timer_adapter_caps_get,
945
946 .crypto_adapter_caps_get = sw_crypto_adapter_caps_get,
947
948 .xstats_get = sw_xstats_get,
949 .xstats_get_names = sw_xstats_get_names,
950 .xstats_get_by_name = sw_xstats_get_by_name,
951 .xstats_reset = sw_xstats_reset,
952
953 .dev_selftest = test_sw_eventdev,
954 };
955
956 static const char *const args[] = {
957 NUMA_NODE_ARG,
958 SCHED_QUANTA_ARG,
959 CREDIT_QUANTA_ARG,
960 NULL
961 };
962 const char *name;
963 const char *params;
964 struct rte_eventdev *dev;
965 struct sw_evdev *sw;
966 int socket_id = rte_socket_id();
967 int sched_quanta = SW_DEFAULT_SCHED_QUANTA;
968 int credit_quanta = SW_DEFAULT_CREDIT_QUANTA;
969
970 name = rte_vdev_device_name(vdev);
971 params = rte_vdev_device_args(vdev);
972 if (params != NULL && params[0] != '\0') {
973 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
974
975 if (!kvlist) {
976 SW_LOG_INFO(
977 "Ignoring unsupported parameters when creating device '%s'\n",
978 name);
979 } else {
980 int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
981 assign_numa_node, &socket_id);
982 if (ret != 0) {
983 SW_LOG_ERR(
984 "%s: Error parsing numa node parameter",
985 name);
986 rte_kvargs_free(kvlist);
987 return ret;
988 }
989
990 ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG,
991 set_sched_quanta, &sched_quanta);
992 if (ret != 0) {
993 SW_LOG_ERR(
994 "%s: Error parsing sched quanta parameter",
995 name);
996 rte_kvargs_free(kvlist);
997 return ret;
998 }
999
1000 ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG,
1001 set_credit_quanta, &credit_quanta);
1002 if (ret != 0) {
1003 SW_LOG_ERR(
1004 "%s: Error parsing credit quanta parameter",
1005 name);
1006 rte_kvargs_free(kvlist);
1007 return ret;
1008 }
1009
1010 rte_kvargs_free(kvlist);
1011 }
1012 }
1013
1014 SW_LOG_INFO(
1015 "Creating eventdev sw device %s, numa_node=%d, sched_quanta=%d, credit_quanta=%d\n",
1016 name, socket_id, sched_quanta, credit_quanta);
1017
1018 dev = rte_event_pmd_vdev_init(name,
1019 sizeof(struct sw_evdev), socket_id);
1020 if (dev == NULL) {
1021 SW_LOG_ERR("eventdev vdev init() failed");
1022 return -EFAULT;
1023 }
1024 dev->dev_ops = &evdev_sw_ops;
1025 dev->enqueue = sw_event_enqueue;
1026 dev->enqueue_burst = sw_event_enqueue_burst;
1027 dev->enqueue_new_burst = sw_event_enqueue_burst;
1028 dev->enqueue_forward_burst = sw_event_enqueue_burst;
1029 dev->dequeue = sw_event_dequeue;
1030 dev->dequeue_burst = sw_event_dequeue_burst;
1031
1032 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1033 return 0;
1034
1035 sw = dev->data->dev_private;
1036 sw->data = dev->data;
1037
1038 /* copy values passed from vdev command line to instance */
1039 sw->credit_update_quanta = credit_quanta;
1040 sw->sched_quanta = sched_quanta;
1041
1042 /* register service with EAL */
1043 struct rte_service_spec service;
1044 memset(&service, 0, sizeof(struct rte_service_spec));
1045 snprintf(service.name, sizeof(service.name), "%s_service", name);
1046 snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
1047 name);
1048 service.socket_id = socket_id;
1049 service.callback = sw_sched_service_func;
1050 service.callback_userdata = (void *)dev;
1051
1052 int32_t ret = rte_service_component_register(&service, &sw->service_id);
1053 if (ret) {
1054 SW_LOG_ERR("service register() failed");
1055 return -ENOEXEC;
1056 }
1057
1058 dev->data->service_inited = 1;
1059 dev->data->service_id = sw->service_id;
1060
1061 return 0;
1062}
1063
1064static int
1065sw_remove(struct rte_vdev_device *vdev)
1066{
1067 const char *name;
1068
1069 name = rte_vdev_device_name(vdev);
1070 if (name == NULL)
1071 return -EINVAL;
1072
1073 SW_LOG_INFO("Closing eventdev sw device %s\n", name);
1074
1075 return rte_event_pmd_vdev_uninit(name);
1076}
1077
1078static struct rte_vdev_driver evdev_sw_pmd_drv = {
1079 .probe = sw_probe,
1080 .remove = sw_remove
1081};
1082
1083RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv);
1084RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> "
1085 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>");
1086
1087/* declared extern in header, for access from other .c files */
1088int eventdev_sw_log_level;
1089
1090RTE_INIT(evdev_sw_init_log)
1091{
1092 eventdev_sw_log_level = rte_log_register("pmd.event.sw");
1093 if (eventdev_sw_log_level >= 0)
1094 rte_log_set_level(eventdev_sw_log_level, RTE_LOG_NOTICE);
1095}
Ceph