ceph/src/spdk/dpdk/examples/ip_pipeline/thread.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <stdlib.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_cycles.h>
   9 #include <rte_lcore.h>
  10 #include <rte_ring.h>
  11
  12 #include <rte_table_acl.h>
  13 #include <rte_table_array.h>
  14 #include <rte_table_hash.h>
  15 #include <rte_table_lpm.h>
  16 #include <rte_table_lpm_ipv6.h>
  17
  18 #include "common.h"
  19 #include "thread.h"
  20 #include "pipeline.h"
  21
  22 #ifndef THREAD_PIPELINES_MAX
  23 #define THREAD_PIPELINES_MAX                               256
  24 #endif
  25
  26 #ifndef THREAD_MSGQ_SIZE
  27 #define THREAD_MSGQ_SIZE                                   64
  28 #endif
  29
  30 #ifndef THREAD_TIMER_PERIOD_MS
  31 #define THREAD_TIMER_PERIOD_MS                             100
  32 #endif
  33
  34 /**
  35  * Master thead: data plane thread context
  36  */
  37 struct thread {
  38         struct rte_ring *msgq_req;
  39         struct rte_ring *msgq_rsp;
  40
  41         uint32_t enabled;
  42 };
  43
  44 static struct thread thread[RTE_MAX_LCORE];
  45
  46 /**
  47  * Data plane threads: context
  48  */
  49 struct table_data {
  50         struct rte_table_action *a;
  51 };
  52
  53 struct pipeline_data {
  54         struct rte_pipeline *p;
  55         struct table_data table_data[RTE_PIPELINE_TABLE_MAX];
  56         uint32_t n_tables;
  57
  58         struct rte_ring *msgq_req;
  59         struct rte_ring *msgq_rsp;
  60         uint64_t timer_period; /* Measured in CPU cycles. */
  61         uint64_t time_next;
  62
  63         uint8_t buffer[TABLE_RULE_ACTION_SIZE_MAX];
  64 };
  65
  66 struct thread_data {
  67         struct rte_pipeline *p[THREAD_PIPELINES_MAX];
  68         uint32_t n_pipelines;
  69
  70         struct pipeline_data pipeline_data[THREAD_PIPELINES_MAX];
  71         struct rte_ring *msgq_req;
  72         struct rte_ring *msgq_rsp;
  73         uint64_t timer_period; /* Measured in CPU cycles. */
  74         uint64_t time_next;
  75         uint64_t time_next_min;
  76 } __rte_cache_aligned;
  77
  78 static struct thread_data thread_data[RTE_MAX_LCORE];
  79
  80 /**
  81  * Master thread: data plane thread init
  82  */
  83 static void
  84 thread_free(void)
  85 {
  86         uint32_t i;
  87
  88         for (i = 0; i < RTE_MAX_LCORE; i++) {
  89                 struct thread *t = &thread[i];
  90
  91                 if (!rte_lcore_is_enabled(i))
  92                         continue;
  93
  94                 /* MSGQs */
  95                 if (t->msgq_req)
  96                         rte_ring_free(t->msgq_req);
  97
  98                 if (t->msgq_rsp)
  99                         rte_ring_free(t->msgq_rsp);
 100         }
 101 }
 102
 103 int
 104 thread_init(void)
 105 {
 106         uint32_t i;
 107
 108         RTE_LCORE_FOREACH_SLAVE(i) {
 109                 char name[NAME_MAX];
 110                 struct rte_ring *msgq_req, *msgq_rsp;
 111                 struct thread *t = &thread[i];
 112                 struct thread_data *t_data = &thread_data[i];
 113                 uint32_t cpu_id = rte_lcore_to_socket_id(i);
 114
 115                 /* MSGQs */
 116                 snprintf(name, sizeof(name), "THREAD-%04x-MSGQ-REQ", i);
 117
 118                 msgq_req = rte_ring_create(name,
 119                         THREAD_MSGQ_SIZE,
 120                         cpu_id,
 121                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 122
 123                 if (msgq_req == NULL) {
 124                         thread_free();
 125                         return -1;
 126                 }
 127
 128                 snprintf(name, sizeof(name), "THREAD-%04x-MSGQ-RSP", i);
 129
 130                 msgq_rsp = rte_ring_create(name,
 131                         THREAD_MSGQ_SIZE,
 132                         cpu_id,
 133                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 134
 135                 if (msgq_rsp == NULL) {
 136                         thread_free();
 137                         return -1;
 138                 }
 139
 140                 /* Master thread records */
 141                 t->msgq_req = msgq_req;
 142                 t->msgq_rsp = msgq_rsp;
 143                 t->enabled = 1;
 144
 145                 /* Data plane thread records */
 146                 t_data->n_pipelines = 0;
 147                 t_data->msgq_req = msgq_req;
 148                 t_data->msgq_rsp = msgq_rsp;
 149                 t_data->timer_period =
 150                         (rte_get_tsc_hz() * THREAD_TIMER_PERIOD_MS) / 1000;
 151                 t_data->time_next = rte_get_tsc_cycles() + t_data->timer_period;
 152                 t_data->time_next_min = t_data->time_next;
 153         }
 154
 155         return 0;
 156 }
 157
 158 static inline int
 159 thread_is_running(uint32_t thread_id)
 160 {
 161         enum rte_lcore_state_t thread_state;
 162
 163         thread_state = rte_eal_get_lcore_state(thread_id);
 164         return (thread_state == RUNNING) ? 1 : 0;
 165 }
 166
 167 /**
 168  * Pipeline is running when:
 169  *    (A) Pipeline is mapped to a data plane thread AND
 170  *    (B) Its data plane thread is in RUNNING state.
 171  */
 172 static inline int
 173 pipeline_is_running(struct pipeline *p)
 174 {
 175         if (p->enabled == 0)
 176                 return 0;
 177
 178         return thread_is_running(p->thread_id);
 179 }
 180
 181 /**
 182  * Master thread & data plane threads: message passing
 183  */
 184 enum thread_req_type {
 185         THREAD_REQ_PIPELINE_ENABLE = 0,
 186         THREAD_REQ_PIPELINE_DISABLE,
 187         THREAD_REQ_MAX
 188 };
 189
 190 struct thread_msg_req {
 191         enum thread_req_type type;
 192
 193         union {
 194                 struct {
 195                         struct rte_pipeline *p;
 196                         struct {
 197                                 struct rte_table_action *a;
 198                         } table[RTE_PIPELINE_TABLE_MAX];
 199                         struct rte_ring *msgq_req;
 200                         struct rte_ring *msgq_rsp;
 201                         uint32_t timer_period_ms;
 202                         uint32_t n_tables;
 203                 } pipeline_enable;
 204
 205                 struct {
 206                         struct rte_pipeline *p;
 207                 } pipeline_disable;
 208         };
 209 };
 210
 211 struct thread_msg_rsp {
 212         int status;
 213 };
 214
 215 /**
 216  * Master thread
 217  */
 218 static struct thread_msg_req *
 219 thread_msg_alloc(void)
 220 {
 221         size_t size = RTE_MAX(sizeof(struct thread_msg_req),
 222                 sizeof(struct thread_msg_rsp));
 223
 224         return calloc(1, size);
 225 }
 226
 227 static void
 228 thread_msg_free(struct thread_msg_rsp *rsp)
 229 {
 230         free(rsp);
 231 }
 232
 233 static struct thread_msg_rsp *
 234 thread_msg_send_recv(uint32_t thread_id,
 235         struct thread_msg_req *req)
 236 {
 237         struct thread *t = &thread[thread_id];
 238         struct rte_ring *msgq_req = t->msgq_req;
 239         struct rte_ring *msgq_rsp = t->msgq_rsp;
 240         struct thread_msg_rsp *rsp;
 241         int status;
 242
 243         /* send */
 244         do {
 245                 status = rte_ring_sp_enqueue(msgq_req, req);
 246         } while (status == -ENOBUFS);
 247
 248         /* recv */
 249         do {
 250                 status = rte_ring_sc_dequeue(msgq_rsp, (void **) &rsp);
 251         } while (status != 0);
 252
 253         return rsp;
 254 }
 255
 256 int
 257 thread_pipeline_enable(uint32_t thread_id,
 258         const char *pipeline_name)
 259 {
 260         struct pipeline *p = pipeline_find(pipeline_name);
 261         struct thread *t;
 262         struct thread_msg_req *req;
 263         struct thread_msg_rsp *rsp;
 264         uint32_t i;
 265         int status;
 266
 267         /* Check input params */
 268         if ((thread_id >= RTE_MAX_LCORE) ||
 269                 (p == NULL) ||
 270                 (p->n_ports_in == 0) ||
 271                 (p->n_ports_out == 0) ||
 272                 (p->n_tables == 0))
 273                 return -1;
 274
 275         t = &thread[thread_id];
 276         if ((t->enabled == 0) ||
 277                 p->enabled)
 278                 return -1;
 279
 280         if (!thread_is_running(thread_id)) {
 281                 struct thread_data *td = &thread_data[thread_id];
 282                 struct pipeline_data *tdp = &td->pipeline_data[td->n_pipelines];
 283
 284                 if (td->n_pipelines >= THREAD_PIPELINES_MAX)
 285                         return -1;
 286
 287                 /* Data plane thread */
 288                 td->p[td->n_pipelines] = p->p;
 289
 290                 tdp->p = p->p;
 291                 for (i = 0; i < p->n_tables; i++)
 292                         tdp->table_data[i].a = p->table[i].a;
 293
 294                 tdp->n_tables = p->n_tables;
 295
 296                 tdp->msgq_req = p->msgq_req;
 297                 tdp->msgq_rsp = p->msgq_rsp;
 298                 tdp->timer_period = (rte_get_tsc_hz() * p->timer_period_ms) / 1000;
 299                 tdp->time_next = rte_get_tsc_cycles() + tdp->timer_period;
 300
 301                 td->n_pipelines++;
 302
 303                 /* Pipeline */
 304                 p->thread_id = thread_id;
 305                 p->enabled = 1;
 306
 307                 return 0;
 308         }
 309
 310         /* Allocate request */
 311         req = thread_msg_alloc();
 312         if (req == NULL)
 313                 return -1;
 314
 315         /* Write request */
 316         req->type = THREAD_REQ_PIPELINE_ENABLE;
 317         req->pipeline_enable.p = p->p;
 318         for (i = 0; i < p->n_tables; i++)
 319                 req->pipeline_enable.table[i].a =
 320                         p->table[i].a;
 321         req->pipeline_enable.msgq_req = p->msgq_req;
 322         req->pipeline_enable.msgq_rsp = p->msgq_rsp;
 323         req->pipeline_enable.timer_period_ms = p->timer_period_ms;
 324         req->pipeline_enable.n_tables = p->n_tables;
 325
 326         /* Send request and wait for response */
 327         rsp = thread_msg_send_recv(thread_id, req);
 328         if (rsp == NULL)
 329                 return -1;
 330
 331         /* Read response */
 332         status = rsp->status;
 333
 334         /* Free response */
 335         thread_msg_free(rsp);
 336
 337         /* Request completion */
 338         if (status)
 339                 return status;
 340
 341         p->thread_id = thread_id;
 342         p->enabled = 1;
 343
 344         return 0;
 345 }
 346
 347 int
 348 thread_pipeline_disable(uint32_t thread_id,
 349         const char *pipeline_name)
 350 {
 351         struct pipeline *p = pipeline_find(pipeline_name);
 352         struct thread *t;
 353         struct thread_msg_req *req;
 354         struct thread_msg_rsp *rsp;
 355         int status;
 356
 357         /* Check input params */
 358         if ((thread_id >= RTE_MAX_LCORE) ||
 359                 (p == NULL))
 360                 return -1;
 361
 362         t = &thread[thread_id];
 363         if (t->enabled == 0)
 364                 return -1;
 365
 366         if (p->enabled == 0)
 367                 return 0;
 368
 369         if (p->thread_id != thread_id)
 370                 return -1;
 371
 372         if (!thread_is_running(thread_id)) {
 373                 struct thread_data *td = &thread_data[thread_id];
 374                 uint32_t i;
 375
 376                 for (i = 0; i < td->n_pipelines; i++) {
 377                         struct pipeline_data *tdp = &td->pipeline_data[i];
 378
 379                         if (tdp->p != p->p)
 380                                 continue;
 381
 382                         /* Data plane thread */
 383                         if (i < td->n_pipelines - 1) {
 384                                 struct rte_pipeline *pipeline_last =
 385                                         td->p[td->n_pipelines - 1];
 386                                 struct pipeline_data *tdp_last =
 387                                         &td->pipeline_data[td->n_pipelines - 1];
 388
 389                                 td->p[i] = pipeline_last;
 390                                 memcpy(tdp, tdp_last, sizeof(*tdp));
 391                         }
 392
 393                         td->n_pipelines--;
 394
 395                         /* Pipeline */
 396                         p->enabled = 0;
 397
 398                         break;
 399                 }
 400
 401                 return 0;
 402         }
 403
 404         /* Allocate request */
 405         req = thread_msg_alloc();
 406         if (req == NULL)
 407                 return -1;
 408
 409         /* Write request */
 410         req->type = THREAD_REQ_PIPELINE_DISABLE;
 411         req->pipeline_disable.p = p->p;
 412
 413         /* Send request and wait for response */
 414         rsp = thread_msg_send_recv(thread_id, req);
 415         if (rsp == NULL)
 416                 return -1;
 417
 418         /* Read response */
 419         status = rsp->status;
 420
 421         /* Free response */
 422         thread_msg_free(rsp);
 423
 424         /* Request completion */
 425         if (status)
 426                 return status;
 427
 428         p->enabled = 0;
 429
 430         return 0;
 431 }
 432
 433 /**
 434  * Data plane threads: message handling
 435  */
 436 static inline struct thread_msg_req *
 437 thread_msg_recv(struct rte_ring *msgq_req)
 438 {
 439         struct thread_msg_req *req;
 440
 441         int status = rte_ring_sc_dequeue(msgq_req, (void **) &req);
 442
 443         if (status != 0)
 444                 return NULL;
 445
 446         return req;
 447 }
 448
 449 static inline void
 450 thread_msg_send(struct rte_ring *msgq_rsp,
 451         struct thread_msg_rsp *rsp)
 452 {
 453         int status;
 454
 455         do {
 456                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
 457         } while (status == -ENOBUFS);
 458 }
 459
 460 static struct thread_msg_rsp *
 461 thread_msg_handle_pipeline_enable(struct thread_data *t,
 462         struct thread_msg_req *req)
 463 {
 464         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *) req;
 465         struct pipeline_data *p = &t->pipeline_data[t->n_pipelines];
 466         uint32_t i;
 467
 468         /* Request */
 469         if (t->n_pipelines >= THREAD_PIPELINES_MAX) {
 470                 rsp->status = -1;
 471                 return rsp;
 472         }
 473
 474         t->p[t->n_pipelines] = req->pipeline_enable.p;
 475
 476         p->p = req->pipeline_enable.p;
 477         for (i = 0; i < req->pipeline_enable.n_tables; i++)
 478                 p->table_data[i].a =
 479                         req->pipeline_enable.table[i].a;
 480
 481         p->n_tables = req->pipeline_enable.n_tables;
 482
 483         p->msgq_req = req->pipeline_enable.msgq_req;
 484         p->msgq_rsp = req->pipeline_enable.msgq_rsp;
 485         p->timer_period =
 486                 (rte_get_tsc_hz() * req->pipeline_enable.timer_period_ms) / 1000;
 487         p->time_next = rte_get_tsc_cycles() + p->timer_period;
 488
 489         t->n_pipelines++;
 490
 491         /* Response */
 492         rsp->status = 0;
 493         return rsp;
 494 }
 495
 496 static struct thread_msg_rsp *
 497 thread_msg_handle_pipeline_disable(struct thread_data *t,
 498         struct thread_msg_req *req)
 499 {
 500         struct thread_msg_rsp *rsp = (struct thread_msg_rsp *) req;
 501         uint32_t n_pipelines = t->n_pipelines;
 502         struct rte_pipeline *pipeline = req->pipeline_disable.p;
 503         uint32_t i;
 504
 505         /* find pipeline */
 506         for (i = 0; i < n_pipelines; i++) {
 507                 struct pipeline_data *p = &t->pipeline_data[i];
 508
 509                 if (p->p != pipeline)
 510                         continue;
 511
 512                 if (i < n_pipelines - 1) {
 513                         struct rte_pipeline *pipeline_last =
 514                                 t->p[n_pipelines - 1];
 515                         struct pipeline_data *p_last =
 516                                 &t->pipeline_data[n_pipelines - 1];
 517
 518                         t->p[i] = pipeline_last;
 519                         memcpy(p, p_last, sizeof(*p));
 520                 }
 521
 522                 t->n_pipelines--;
 523
 524                 rsp->status = 0;
 525                 return rsp;
 526         }
 527
 528         /* should not get here */
 529         rsp->status = 0;
 530         return rsp;
 531 }
 532
 533 static void
 534 thread_msg_handle(struct thread_data *t)
 535 {
 536         for ( ; ; ) {
 537                 struct thread_msg_req *req;
 538                 struct thread_msg_rsp *rsp;
 539
 540                 req = thread_msg_recv(t->msgq_req);
 541                 if (req == NULL)
 542                         break;
 543
 544                 switch (req->type) {
 545                 case THREAD_REQ_PIPELINE_ENABLE:
 546                         rsp = thread_msg_handle_pipeline_enable(t, req);
 547                         break;
 548
 549                 case THREAD_REQ_PIPELINE_DISABLE:
 550                         rsp = thread_msg_handle_pipeline_disable(t, req);
 551                         break;
 552
 553                 default:
 554                         rsp = (struct thread_msg_rsp *) req;
 555                         rsp->status = -1;
 556                 }
 557
 558                 thread_msg_send(t->msgq_rsp, rsp);
 559         }
 560 }
 561
 562 /**
 563  * Master thread & data plane threads: message passing
 564  */
 565 enum pipeline_req_type {
 566         /* Port IN */
 567         PIPELINE_REQ_PORT_IN_STATS_READ,
 568         PIPELINE_REQ_PORT_IN_ENABLE,
 569         PIPELINE_REQ_PORT_IN_DISABLE,
 570
 571         /* Port OUT */
 572         PIPELINE_REQ_PORT_OUT_STATS_READ,
 573
 574         /* Table */
 575         PIPELINE_REQ_TABLE_STATS_READ,
 576         PIPELINE_REQ_TABLE_RULE_ADD,
 577         PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT,
 578         PIPELINE_REQ_TABLE_RULE_ADD_BULK,
 579         PIPELINE_REQ_TABLE_RULE_DELETE,
 580         PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT,
 581         PIPELINE_REQ_TABLE_RULE_STATS_READ,
 582         PIPELINE_REQ_TABLE_MTR_PROFILE_ADD,
 583         PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE,
 584         PIPELINE_REQ_TABLE_RULE_MTR_READ,
 585         PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE,
 586         PIPELINE_REQ_TABLE_RULE_TTL_READ,
 587         PIPELINE_REQ_MAX
 588 };
 589
 590 struct pipeline_msg_req_port_in_stats_read {
 591         int clear;
 592 };
 593
 594 struct pipeline_msg_req_port_out_stats_read {
 595         int clear;
 596 };
 597
 598 struct pipeline_msg_req_table_stats_read {
 599         int clear;
 600 };
 601
 602 struct pipeline_msg_req_table_rule_add {
 603         struct table_rule_match match;
 604         struct table_rule_action action;
 605 };
 606
 607 struct pipeline_msg_req_table_rule_add_default {
 608         struct table_rule_action action;
 609 };
 610
 611 struct pipeline_msg_req_table_rule_add_bulk {
 612         struct table_rule_match *match;
 613         struct table_rule_action *action;
 614         void **data;
 615         uint32_t n_rules;
 616         int bulk;
 617 };
 618
 619 struct pipeline_msg_req_table_rule_delete {
 620         struct table_rule_match match;
 621 };
 622
 623 struct pipeline_msg_req_table_rule_stats_read {
 624         void *data;
 625         int clear;
 626 };
 627
 628 struct pipeline_msg_req_table_mtr_profile_add {
 629         uint32_t meter_profile_id;
 630         struct rte_table_action_meter_profile profile;
 631 };
 632
 633 struct pipeline_msg_req_table_mtr_profile_delete {
 634         uint32_t meter_profile_id;
 635 };
 636
 637 struct pipeline_msg_req_table_rule_mtr_read {
 638         void *data;
 639         uint32_t tc_mask;
 640         int clear;
 641 };
 642
 643 struct pipeline_msg_req_table_dscp_table_update {
 644         uint64_t dscp_mask;
 645         struct rte_table_action_dscp_table dscp_table;
 646 };
 647
 648 struct pipeline_msg_req_table_rule_ttl_read {
 649         void *data;
 650         int clear;
 651 };
 652
 653 struct pipeline_msg_req {
 654         enum pipeline_req_type type;
 655         uint32_t id; /* Port IN, port OUT or table ID */
 656
 657         RTE_STD_C11
 658         union {
 659                 struct pipeline_msg_req_port_in_stats_read port_in_stats_read;
 660                 struct pipeline_msg_req_port_out_stats_read port_out_stats_read;
 661                 struct pipeline_msg_req_table_stats_read table_stats_read;
 662                 struct pipeline_msg_req_table_rule_add table_rule_add;
 663                 struct pipeline_msg_req_table_rule_add_default table_rule_add_default;
 664                 struct pipeline_msg_req_table_rule_add_bulk table_rule_add_bulk;
 665                 struct pipeline_msg_req_table_rule_delete table_rule_delete;
 666                 struct pipeline_msg_req_table_rule_stats_read table_rule_stats_read;
 667                 struct pipeline_msg_req_table_mtr_profile_add table_mtr_profile_add;
 668                 struct pipeline_msg_req_table_mtr_profile_delete table_mtr_profile_delete;
 669                 struct pipeline_msg_req_table_rule_mtr_read table_rule_mtr_read;
 670                 struct pipeline_msg_req_table_dscp_table_update table_dscp_table_update;
 671                 struct pipeline_msg_req_table_rule_ttl_read table_rule_ttl_read;
 672         };
 673 };
 674
 675 struct pipeline_msg_rsp_port_in_stats_read {
 676         struct rte_pipeline_port_in_stats stats;
 677 };
 678
 679 struct pipeline_msg_rsp_port_out_stats_read {
 680         struct rte_pipeline_port_out_stats stats;
 681 };
 682
 683 struct pipeline_msg_rsp_table_stats_read {
 684         struct rte_pipeline_table_stats stats;
 685 };
 686
 687 struct pipeline_msg_rsp_table_rule_add {
 688         void *data;
 689 };
 690
 691 struct pipeline_msg_rsp_table_rule_add_default {
 692         void *data;
 693 };
 694
 695 struct pipeline_msg_rsp_table_rule_add_bulk {
 696         uint32_t n_rules;
 697 };
 698
 699 struct pipeline_msg_rsp_table_rule_stats_read {
 700         struct rte_table_action_stats_counters stats;
 701 };
 702
 703 struct pipeline_msg_rsp_table_rule_mtr_read {
 704         struct rte_table_action_mtr_counters stats;
 705 };
 706
 707 struct pipeline_msg_rsp_table_rule_ttl_read {
 708         struct rte_table_action_ttl_counters stats;
 709 };
 710
 711 struct pipeline_msg_rsp {
 712         int status;
 713
 714         RTE_STD_C11
 715         union {
 716                 struct pipeline_msg_rsp_port_in_stats_read port_in_stats_read;
 717                 struct pipeline_msg_rsp_port_out_stats_read port_out_stats_read;
 718                 struct pipeline_msg_rsp_table_stats_read table_stats_read;
 719                 struct pipeline_msg_rsp_table_rule_add table_rule_add;
 720                 struct pipeline_msg_rsp_table_rule_add_default table_rule_add_default;
 721                 struct pipeline_msg_rsp_table_rule_add_bulk table_rule_add_bulk;
 722                 struct pipeline_msg_rsp_table_rule_stats_read table_rule_stats_read;
 723                 struct pipeline_msg_rsp_table_rule_mtr_read table_rule_mtr_read;
 724                 struct pipeline_msg_rsp_table_rule_ttl_read table_rule_ttl_read;
 725         };
 726 };
 727
 728 /**
 729  * Master thread
 730  */
 731 static struct pipeline_msg_req *
 732 pipeline_msg_alloc(void)
 733 {
 734         size_t size = RTE_MAX(sizeof(struct pipeline_msg_req),
 735                 sizeof(struct pipeline_msg_rsp));
 736
 737         return calloc(1, size);
 738 }
 739
 740 static void
 741 pipeline_msg_free(struct pipeline_msg_rsp *rsp)
 742 {
 743         free(rsp);
 744 }
 745
 746 static struct pipeline_msg_rsp *
 747 pipeline_msg_send_recv(struct pipeline *p,
 748         struct pipeline_msg_req *req)
 749 {
 750         struct rte_ring *msgq_req = p->msgq_req;
 751         struct rte_ring *msgq_rsp = p->msgq_rsp;
 752         struct pipeline_msg_rsp *rsp;
 753         int status;
 754
 755         /* send */
 756         do {
 757                 status = rte_ring_sp_enqueue(msgq_req, req);
 758         } while (status == -ENOBUFS);
 759
 760         /* recv */
 761         do {
 762                 status = rte_ring_sc_dequeue(msgq_rsp, (void **) &rsp);
 763         } while (status != 0);
 764
 765         return rsp;
 766 }
 767
 768 int
 769 pipeline_port_in_stats_read(const char *pipeline_name,
 770         uint32_t port_id,
 771         struct rte_pipeline_port_in_stats *stats,
 772         int clear)
 773 {
 774         struct pipeline *p;
 775         struct pipeline_msg_req *req;
 776         struct pipeline_msg_rsp *rsp;
 777         int status;
 778
 779         /* Check input params */
 780         if ((pipeline_name == NULL) ||
 781                 (stats == NULL))
 782                 return -1;
 783
 784         p = pipeline_find(pipeline_name);
 785         if ((p == NULL) ||
 786                 (port_id >= p->n_ports_in))
 787                 return -1;
 788
 789         if (!pipeline_is_running(p)) {
 790                 status = rte_pipeline_port_in_stats_read(p->p,
 791                         port_id,
 792                         stats,
 793                         clear);
 794
 795                 return status;
 796         }
 797
 798         /* Allocate request */
 799         req = pipeline_msg_alloc();
 800         if (req == NULL)
 801                 return -1;
 802
 803         /* Write request */
 804         req->type = PIPELINE_REQ_PORT_IN_STATS_READ;
 805         req->id = port_id;
 806         req->port_in_stats_read.clear = clear;
 807
 808         /* Send request and wait for response */
 809         rsp = pipeline_msg_send_recv(p, req);
 810         if (rsp == NULL)
 811                 return -1;
 812
 813         /* Read response */
 814         status = rsp->status;
 815         if (status)
 816                 memcpy(stats, &rsp->port_in_stats_read.stats, sizeof(*stats));
 817
 818         /* Free response */
 819         pipeline_msg_free(rsp);
 820
 821         return status;
 822 }
 823
 824 int
 825 pipeline_port_in_enable(const char *pipeline_name,
 826         uint32_t port_id)
 827 {
 828         struct pipeline *p;
 829         struct pipeline_msg_req *req;
 830         struct pipeline_msg_rsp *rsp;
 831         int status;
 832
 833         /* Check input params */
 834         if (pipeline_name == NULL)
 835                 return -1;
 836
 837         p = pipeline_find(pipeline_name);
 838         if ((p == NULL) ||
 839                 (port_id >= p->n_ports_in))
 840                 return -1;
 841
 842         if (!pipeline_is_running(p)) {
 843                 status = rte_pipeline_port_in_enable(p->p, port_id);
 844                 return status;
 845         }
 846
 847         /* Allocate request */
 848         req = pipeline_msg_alloc();
 849         if (req == NULL)
 850                 return -1;
 851
 852         /* Write request */
 853         req->type = PIPELINE_REQ_PORT_IN_ENABLE;
 854         req->id = port_id;
 855
 856         /* Send request and wait for response */
 857         rsp = pipeline_msg_send_recv(p, req);
 858         if (rsp == NULL)
 859                 return -1;
 860
 861         /* Read response */
 862         status = rsp->status;
 863
 864         /* Free response */
 865         pipeline_msg_free(rsp);
 866
 867         return status;
 868 }
 869
 870 int
 871 pipeline_port_in_disable(const char *pipeline_name,
 872         uint32_t port_id)
 873 {
 874         struct pipeline *p;
 875         struct pipeline_msg_req *req;
 876         struct pipeline_msg_rsp *rsp;
 877         int status;
 878
 879         /* Check input params */
 880         if (pipeline_name == NULL)
 881                 return -1;
 882
 883         p = pipeline_find(pipeline_name);
 884         if ((p == NULL) ||
 885                 (port_id >= p->n_ports_in))
 886                 return -1;
 887
 888         if (!pipeline_is_running(p)) {
 889                 status = rte_pipeline_port_in_disable(p->p, port_id);
 890                 return status;
 891         }
 892
 893         /* Allocate request */
 894         req = pipeline_msg_alloc();
 895         if (req == NULL)
 896                 return -1;
 897
 898         /* Write request */
 899         req->type = PIPELINE_REQ_PORT_IN_DISABLE;
 900         req->id = port_id;
 901
 902         /* Send request and wait for response */
 903         rsp = pipeline_msg_send_recv(p, req);
 904         if (rsp == NULL)
 905                 return -1;
 906
 907         /* Read response */
 908         status = rsp->status;
 909
 910         /* Free response */
 911         pipeline_msg_free(rsp);
 912
 913         return status;
 914 }
 915
 916 int
 917 pipeline_port_out_stats_read(const char *pipeline_name,
 918         uint32_t port_id,
 919         struct rte_pipeline_port_out_stats *stats,
 920         int clear)
 921 {
 922         struct pipeline *p;
 923         struct pipeline_msg_req *req;
 924         struct pipeline_msg_rsp *rsp;
 925         int status;
 926
 927         /* Check input params */
 928         if ((pipeline_name == NULL) ||
 929                 (stats == NULL))
 930                 return -1;
 931
 932         p = pipeline_find(pipeline_name);
 933         if ((p == NULL) ||
 934                 (port_id >= p->n_ports_out))
 935                 return -1;
 936
 937         if (!pipeline_is_running(p)) {
 938                 status = rte_pipeline_port_out_stats_read(p->p,
 939                         port_id,
 940                         stats,
 941                         clear);
 942
 943                 return status;
 944         }
 945
 946         /* Allocate request */
 947         req = pipeline_msg_alloc();
 948         if (req == NULL)
 949                 return -1;
 950
 951         /* Write request */
 952         req->type = PIPELINE_REQ_PORT_OUT_STATS_READ;
 953         req->id = port_id;
 954         req->port_out_stats_read.clear = clear;
 955
 956         /* Send request and wait for response */
 957         rsp = pipeline_msg_send_recv(p, req);
 958         if (rsp == NULL)
 959                 return -1;
 960
 961         /* Read response */
 962         status = rsp->status;
 963         if (status)
 964                 memcpy(stats, &rsp->port_out_stats_read.stats, sizeof(*stats));
 965
 966         /* Free response */
 967         pipeline_msg_free(rsp);
 968
 969         return status;
 970 }
 971
 972 int
 973 pipeline_table_stats_read(const char *pipeline_name,
 974         uint32_t table_id,
 975         struct rte_pipeline_table_stats *stats,
 976         int clear)
 977 {
 978         struct pipeline *p;
 979         struct pipeline_msg_req *req;
 980         struct pipeline_msg_rsp *rsp;
 981         int status;
 982
 983         /* Check input params */
 984         if ((pipeline_name == NULL) ||
 985                 (stats == NULL))
 986                 return -1;
 987
 988         p = pipeline_find(pipeline_name);
 989         if ((p == NULL) ||
 990                 (table_id >= p->n_tables))
 991                 return -1;
 992
 993         if (!pipeline_is_running(p)) {
 994                 status = rte_pipeline_table_stats_read(p->p,
 995                         table_id,
 996                         stats,
 997                         clear);
 998
 999                 return status;
1000         }
1001
1002         /* Allocate request */
1003         req = pipeline_msg_alloc();
1004         if (req == NULL)
1005                 return -1;
1006
1007         /* Write request */
1008         req->type = PIPELINE_REQ_TABLE_STATS_READ;
1009         req->id = table_id;
1010         req->table_stats_read.clear = clear;
1011
1012         /* Send request and wait for response */
1013         rsp = pipeline_msg_send_recv(p, req);
1014         if (rsp == NULL)
1015                 return -1;
1016
1017         /* Read response */
1018         status = rsp->status;
1019         if (status)
1020                 memcpy(stats, &rsp->table_stats_read.stats, sizeof(*stats));
1021
1022         /* Free response */
1023         pipeline_msg_free(rsp);
1024
1025         return status;
1026 }
1027
1028 static int
1029 match_check(struct table_rule_match *match,
1030         struct pipeline *p,
1031         uint32_t table_id)
1032 {
1033         struct table *table;
1034
1035         if ((match == NULL) ||
1036                 (p == NULL) ||
1037                 (table_id >= p->n_tables))
1038                 return -1;
1039
1040         table = &p->table[table_id];
1041         if (match->match_type != table->params.match_type)
1042                 return -1;
1043
1044         switch (match->match_type) {
1045         case TABLE_ACL:
1046         {
1047                 struct table_acl_params *t = &table->params.match.acl;
1048                 struct table_rule_match_acl *r = &match->match.acl;
1049
1050                 if ((r->ip_version && (t->ip_version == 0)) ||
1051                         ((r->ip_version == 0) && t->ip_version))
1052                         return -1;
1053
1054                 if (r->ip_version) {
1055                         if ((r->sa_depth > 32) ||
1056                                 (r->da_depth > 32))
1057                                 return -1;
1058                 } else {
1059                         if ((r->sa_depth > 128) ||
1060                                 (r->da_depth > 128))
1061                                 return -1;
1062                 }
1063                 return 0;
1064         }
1065
1066         case TABLE_ARRAY:
1067                 return 0;
1068
1069         case TABLE_HASH:
1070                 return 0;
1071
1072         case TABLE_LPM:
1073         {
1074                 struct table_lpm_params *t = &table->params.match.lpm;
1075                 struct table_rule_match_lpm *r = &match->match.lpm;
1076
1077                 if ((r->ip_version && (t->key_size != 4)) ||
1078                         ((r->ip_version == 0) && (t->key_size != 16)))
1079                         return -1;
1080
1081                 if (r->ip_version) {
1082                         if (r->depth > 32)
1083                                 return -1;
1084                 } else {
1085                         if (r->depth > 128)
1086                                 return -1;
1087                 }
1088                 return 0;
1089         }
1090
1091         case TABLE_STUB:
1092                 return -1;
1093
1094         default:
1095                 return -1;
1096         }
1097 }
1098
1099 static int
1100 action_check(struct table_rule_action *action,
1101         struct pipeline *p,
1102         uint32_t table_id)
1103 {
1104         struct table_action_profile *ap;
1105
1106         if ((action == NULL) ||
1107                 (p == NULL) ||
1108                 (table_id >= p->n_tables))
1109                 return -1;
1110
1111         ap = p->table[table_id].ap;
1112         if (action->action_mask != ap->params.action_mask)
1113                 return -1;
1114
1115         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1116                 if ((action->fwd.action == RTE_PIPELINE_ACTION_PORT) &&
1117                         (action->fwd.id >= p->n_ports_out))
1118                         return -1;
1119
1120                 if ((action->fwd.action == RTE_PIPELINE_ACTION_TABLE) &&
1121                         (action->fwd.id >= p->n_tables))
1122                         return -1;
1123         }
1124
1125         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1126                 uint32_t tc_mask0 = (1 << ap->params.mtr.n_tc) - 1;
1127                 uint32_t tc_mask1 = action->mtr.tc_mask;
1128
1129                 if (tc_mask1 != tc_mask0)
1130                         return -1;
1131         }
1132
1133         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1134                 uint32_t n_subports_per_port =
1135                         ap->params.tm.n_subports_per_port;
1136                 uint32_t n_pipes_per_subport =
1137                         ap->params.tm.n_pipes_per_subport;
1138                 uint32_t subport_id = action->tm.subport_id;
1139                 uint32_t pipe_id = action->tm.pipe_id;
1140
1141                 if ((subport_id >= n_subports_per_port) ||
1142                         (pipe_id >= n_pipes_per_subport))
1143                         return -1;
1144         }
1145
1146         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1147                 uint64_t encap_mask = ap->params.encap.encap_mask;
1148                 enum rte_table_action_encap_type type = action->encap.type;
1149
1150                 if ((encap_mask & (1LLU << type)) == 0)
1151                         return -1;
1152         }
1153
1154         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1155                 int ip_version0 = ap->params.common.ip_version;
1156                 int ip_version1 = action->nat.ip_version;
1157
1158                 if ((ip_version1 && (ip_version0 == 0)) ||
1159                         ((ip_version1 == 0) && ip_version0))
1160                         return -1;
1161         }
1162
1163         return 0;
1164 }
1165
1166 static int
1167 action_default_check(struct table_rule_action *action,
1168         struct pipeline *p,
1169         uint32_t table_id)
1170 {
1171         if ((action == NULL) ||
1172                 (action->action_mask != (1LLU << RTE_TABLE_ACTION_FWD)) ||
1173                 (p == NULL) ||
1174                 (table_id >= p->n_tables))
1175                 return -1;
1176
1177         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
1178                 if ((action->fwd.action == RTE_PIPELINE_ACTION_PORT) &&
1179                         (action->fwd.id >= p->n_ports_out))
1180                         return -1;
1181
1182                 if ((action->fwd.action == RTE_PIPELINE_ACTION_TABLE) &&
1183                         (action->fwd.id >= p->n_tables))
1184                         return -1;
1185         }
1186
1187         return 0;
1188 }
1189
1190 union table_rule_match_low_level {
1191         struct rte_table_acl_rule_add_params acl_add;
1192         struct rte_table_acl_rule_delete_params acl_delete;
1193         struct rte_table_array_key array;
1194         uint8_t hash[TABLE_RULE_MATCH_SIZE_MAX];
1195         struct rte_table_lpm_key lpm_ipv4;
1196         struct rte_table_lpm_ipv6_key lpm_ipv6;
1197 };
1198
1199 static int
1200 match_convert(struct table_rule_match *mh,
1201         union table_rule_match_low_level *ml,
1202         int add);
1203
1204 static int
1205 action_convert(struct rte_table_action *a,
1206         struct table_rule_action *action,
1207         struct rte_pipeline_table_entry *data);
1208
1209 int
1210 pipeline_table_rule_add(const char *pipeline_name,
1211         uint32_t table_id,
1212         struct table_rule_match *match,
1213         struct table_rule_action *action,
1214         void **data)
1215 {
1216         struct pipeline *p;
1217         struct pipeline_msg_req *req;
1218         struct pipeline_msg_rsp *rsp;
1219         int status;
1220
1221         /* Check input params */
1222         if ((pipeline_name == NULL) ||
1223                 (match == NULL) ||
1224                 (action == NULL) ||
1225                 (data == NULL))
1226                 return -1;
1227
1228         p = pipeline_find(pipeline_name);
1229         if ((p == NULL) ||
1230                 (table_id >= p->n_tables) ||
1231                 match_check(match, p, table_id) ||
1232                 action_check(action, p, table_id))
1233                 return -1;
1234
1235         if (!pipeline_is_running(p)) {
1236                 struct rte_table_action *a = p->table[table_id].a;
1237                 union table_rule_match_low_level match_ll;
1238                 struct rte_pipeline_table_entry *data_in, *data_out;
1239                 int key_found;
1240                 uint8_t *buffer;
1241
1242                 buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
1243                 if (buffer == NULL)
1244                         return -1;
1245
1246                 /* Table match-action rule conversion */
1247                 data_in = (struct rte_pipeline_table_entry *)buffer;
1248
1249                 status = match_convert(match, &match_ll, 1);
1250                 if (status) {
1251                         free(buffer);
1252                         return -1;
1253                 }
1254
1255                 status = action_convert(a, action, data_in);
1256                 if (status) {
1257                         free(buffer);
1258                         return -1;
1259                 }
1260
1261                 /* Add rule (match, action) to table */
1262                 status = rte_pipeline_table_entry_add(p->p,
1263                                 table_id,
1264                                 &match_ll,
1265                                 data_in,
1266                                 &key_found,
1267                                 &data_out);
1268                 if (status) {
1269                         free(buffer);
1270                         return -1;
1271                 }
1272
1273                 /* Write Response */
1274                 *data = data_out;
1275
1276                 free(buffer);
1277                 return 0;
1278         }
1279
1280         /* Allocate request */
1281         req = pipeline_msg_alloc();
1282         if (req == NULL)
1283                 return -1;
1284
1285         /* Write request */
1286         req->type = PIPELINE_REQ_TABLE_RULE_ADD;
1287         req->id = table_id;
1288         memcpy(&req->table_rule_add.match, match, sizeof(*match));
1289         memcpy(&req->table_rule_add.action, action, sizeof(*action));
1290
1291         /* Send request and wait for response */
1292         rsp = pipeline_msg_send_recv(p, req);
1293         if (rsp == NULL)
1294                 return -1;
1295
1296         /* Read response */
1297         status = rsp->status;
1298         if (status == 0)
1299                 *data = rsp->table_rule_add.data;
1300
1301         /* Free response */
1302         pipeline_msg_free(rsp);
1303
1304         return status;
1305 }
1306
1307 int
1308 pipeline_table_rule_add_default(const char *pipeline_name,
1309         uint32_t table_id,
1310         struct table_rule_action *action,
1311         void **data)
1312 {
1313         struct pipeline *p;
1314         struct pipeline_msg_req *req;
1315         struct pipeline_msg_rsp *rsp;
1316         int status;
1317
1318         /* Check input params */
1319         if ((pipeline_name == NULL) ||
1320                 (action == NULL) ||
1321                 (data == NULL))
1322                 return -1;
1323
1324         p = pipeline_find(pipeline_name);
1325         if ((p == NULL) ||
1326                 (table_id >= p->n_tables) ||
1327                 action_default_check(action, p, table_id))
1328                 return -1;
1329
1330         if (!pipeline_is_running(p)) {
1331                 struct rte_pipeline_table_entry *data_in, *data_out;
1332                 uint8_t *buffer;
1333
1334                 buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
1335                 if (buffer == NULL)
1336                         return -1;
1337
1338                 /* Apply actions */
1339                 data_in = (struct rte_pipeline_table_entry *)buffer;
1340
1341                 data_in->action = action->fwd.action;
1342                 if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
1343                         data_in->port_id = action->fwd.id;
1344                 if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
1345                         data_in->table_id = action->fwd.id;
1346
1347                 /* Add default rule to table */
1348                 status = rte_pipeline_table_default_entry_add(p->p,
1349                                 table_id,
1350                                 data_in,
1351                                 &data_out);
1352                 if (status) {
1353                         free(buffer);
1354                         return -1;
1355                 }
1356
1357                 /* Write Response */
1358                 *data = data_out;
1359
1360                 free(buffer);
1361                 return 0;
1362         }
1363
1364         /* Allocate request */
1365         req = pipeline_msg_alloc();
1366         if (req == NULL)
1367                 return -1;
1368
1369         /* Write request */
1370         req->type = PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT;
1371         req->id = table_id;
1372         memcpy(&req->table_rule_add_default.action, action, sizeof(*action));
1373
1374         /* Send request and wait for response */
1375         rsp = pipeline_msg_send_recv(p, req);
1376         if (rsp == NULL)
1377                 return -1;
1378
1379         /* Read response */
1380         status = rsp->status;
1381         if (status == 0)
1382                 *data = rsp->table_rule_add_default.data;
1383
1384         /* Free response */
1385         pipeline_msg_free(rsp);
1386
1387         return status;
1388 }
1389
1390 int
1391 pipeline_table_rule_add_bulk(const char *pipeline_name,
1392         uint32_t table_id,
1393         struct table_rule_match *match,
1394         struct table_rule_action *action,
1395         void **data,
1396         uint32_t *n_rules)
1397 {
1398         struct pipeline *p;
1399         struct pipeline_msg_req *req;
1400         struct pipeline_msg_rsp *rsp;
1401         uint32_t i;
1402         int status;
1403
1404         /* Check input params */
1405         if ((pipeline_name == NULL) ||
1406                 (match == NULL) ||
1407                 (action == NULL) ||
1408                 (data == NULL) ||
1409                 (n_rules == NULL) ||
1410                 (*n_rules == 0))
1411                 return -1;
1412
1413         p = pipeline_find(pipeline_name);
1414         if ((p == NULL) ||
1415                 (table_id >= p->n_tables))
1416                 return -1;
1417
1418         for (i = 0; i < *n_rules; i++)
1419                 if (match_check(match, p, table_id) ||
1420                         action_check(action, p, table_id))
1421                         return -1;
1422
1423         if (!pipeline_is_running(p)) {
1424                 struct rte_table_action *a = p->table[table_id].a;
1425                 union table_rule_match_low_level *match_ll;
1426                 uint8_t *action_ll;
1427                 void **match_ll_ptr;
1428                 struct rte_pipeline_table_entry **action_ll_ptr;
1429                 struct rte_pipeline_table_entry **entries_ptr =
1430                         (struct rte_pipeline_table_entry **)data;
1431                 uint32_t bulk =
1432                         (p->table[table_id].params.match_type == TABLE_ACL) ? 1 : 0;
1433                 int *found;
1434
1435                 /* Memory allocation */
1436                 match_ll = calloc(*n_rules, sizeof(union table_rule_match_low_level));
1437                 action_ll = calloc(*n_rules, TABLE_RULE_ACTION_SIZE_MAX);
1438                 match_ll_ptr = calloc(*n_rules, sizeof(void *));
1439                 action_ll_ptr =
1440                         calloc(*n_rules, sizeof(struct rte_pipeline_table_entry *));
1441                 found = calloc(*n_rules, sizeof(int));
1442
1443                 if (match_ll == NULL ||
1444                         action_ll == NULL ||
1445                         match_ll_ptr == NULL ||
1446                         action_ll_ptr == NULL ||
1447                         found == NULL)
1448                         goto fail;
1449
1450                 for (i = 0; i < *n_rules; i++) {
1451                         match_ll_ptr[i] = (void *)&match_ll[i];
1452                         action_ll_ptr[i] =
1453                                 (struct rte_pipeline_table_entry *)&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
1454                 }
1455
1456                 /* Rule match conversion */
1457                 for (i = 0; i < *n_rules; i++) {
1458                         status = match_convert(&match[i], match_ll_ptr[i], 1);
1459                         if (status)
1460                                 goto fail;
1461                 }
1462
1463                 /* Rule action conversion */
1464                 for (i = 0; i < *n_rules; i++) {
1465                         status = action_convert(a, &action[i], action_ll_ptr[i]);
1466                         if (status)
1467                                 goto fail;
1468                 }
1469
1470                 /* Add rule (match, action) to table */
1471                 if (bulk) {
1472                         status = rte_pipeline_table_entry_add_bulk(p->p,
1473                                 table_id,
1474                                 match_ll_ptr,
1475                                 action_ll_ptr,
1476                                 *n_rules,
1477                                 found,
1478                                 entries_ptr);
1479                         if (status)
1480                                 *n_rules = 0;
1481                 } else {
1482                         for (i = 0; i < *n_rules; i++) {
1483                                 status = rte_pipeline_table_entry_add(p->p,
1484                                         table_id,
1485                                         match_ll_ptr[i],
1486                                         action_ll_ptr[i],
1487                                         &found[i],
1488                                         &entries_ptr[i]);
1489                                 if (status) {
1490                                         *n_rules = i;
1491                                         break;
1492                                 }
1493                         }
1494                 }
1495
1496                 /* Free */
1497                 free(found);
1498                 free(action_ll_ptr);
1499                 free(match_ll_ptr);
1500                 free(action_ll);
1501                 free(match_ll);
1502
1503                 return status;
1504
1505 fail:
1506                 free(found);
1507                 free(action_ll_ptr);
1508                 free(match_ll_ptr);
1509                 free(action_ll);
1510                 free(match_ll);
1511
1512                 *n_rules = 0;
1513                 return -1;
1514         }
1515
1516         /* Allocate request */
1517         req = pipeline_msg_alloc();
1518         if (req == NULL)
1519                 return -1;
1520
1521         /* Write request */
1522         req->type = PIPELINE_REQ_TABLE_RULE_ADD_BULK;
1523         req->id = table_id;
1524         req->table_rule_add_bulk.match = match;
1525         req->table_rule_add_bulk.action = action;
1526         req->table_rule_add_bulk.data = data;
1527         req->table_rule_add_bulk.n_rules = *n_rules;
1528         req->table_rule_add_bulk.bulk =
1529                 (p->table[table_id].params.match_type == TABLE_ACL) ? 1 : 0;
1530
1531         /* Send request and wait for response */
1532         rsp = pipeline_msg_send_recv(p, req);
1533         if (rsp == NULL)
1534                 return -1;
1535
1536         /* Read response */
1537         status = rsp->status;
1538         if (status == 0)
1539                 *n_rules = rsp->table_rule_add_bulk.n_rules;
1540
1541         /* Free response */
1542         pipeline_msg_free(rsp);
1543
1544         return status;
1545 }
1546
1547 int
1548 pipeline_table_rule_delete(const char *pipeline_name,
1549         uint32_t table_id,
1550         struct table_rule_match *match)
1551 {
1552         struct pipeline *p;
1553         struct pipeline_msg_req *req;
1554         struct pipeline_msg_rsp *rsp;
1555         int status;
1556
1557         /* Check input params */
1558         if ((pipeline_name == NULL) ||
1559                 (match == NULL))
1560                 return -1;
1561
1562         p = pipeline_find(pipeline_name);
1563         if ((p == NULL) ||
1564                 (table_id >= p->n_tables) ||
1565                 match_check(match, p, table_id))
1566                 return -1;
1567
1568         if (!pipeline_is_running(p)) {
1569                 union table_rule_match_low_level match_ll;
1570                 int key_found;
1571
1572                 status = match_convert(match, &match_ll, 0);
1573                 if (status)
1574                         return -1;
1575
1576                 status = rte_pipeline_table_entry_delete(p->p,
1577                                 table_id,
1578                                 &match_ll,
1579                                 &key_found,
1580                                 NULL);
1581
1582                 return status;
1583         }
1584
1585         /* Allocate request */
1586         req = pipeline_msg_alloc();
1587         if (req == NULL)
1588                 return -1;
1589
1590         /* Write request */
1591         req->type = PIPELINE_REQ_TABLE_RULE_DELETE;
1592         req->id = table_id;
1593         memcpy(&req->table_rule_delete.match, match, sizeof(*match));
1594
1595         /* Send request and wait for response */
1596         rsp = pipeline_msg_send_recv(p, req);
1597         if (rsp == NULL)
1598                 return -1;
1599
1600         /* Read response */
1601         status = rsp->status;
1602
1603         /* Free response */
1604         pipeline_msg_free(rsp);
1605
1606         return status;
1607 }
1608
1609 int
1610 pipeline_table_rule_delete_default(const char *pipeline_name,
1611         uint32_t table_id)
1612 {
1613         struct pipeline *p;
1614         struct pipeline_msg_req *req;
1615         struct pipeline_msg_rsp *rsp;
1616         int status;
1617
1618         /* Check input params */
1619         if (pipeline_name == NULL)
1620                 return -1;
1621
1622         p = pipeline_find(pipeline_name);
1623         if ((p == NULL) ||
1624                 (table_id >= p->n_tables))
1625                 return -1;
1626
1627         if (!pipeline_is_running(p)) {
1628                 status = rte_pipeline_table_default_entry_delete(p->p,
1629                         table_id,
1630                         NULL);
1631
1632                 return status;
1633         }
1634
1635         /* Allocate request */
1636         req = pipeline_msg_alloc();
1637         if (req == NULL)
1638                 return -1;
1639
1640         /* Write request */
1641         req->type = PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT;
1642         req->id = table_id;
1643
1644         /* Send request and wait for response */
1645         rsp = pipeline_msg_send_recv(p, req);
1646         if (rsp == NULL)
1647                 return -1;
1648
1649         /* Read response */
1650         status = rsp->status;
1651
1652         /* Free response */
1653         pipeline_msg_free(rsp);
1654
1655         return status;
1656 }
1657
1658 int
1659 pipeline_table_rule_stats_read(const char *pipeline_name,
1660         uint32_t table_id,
1661         void *data,
1662         struct rte_table_action_stats_counters *stats,
1663         int clear)
1664 {
1665         struct pipeline *p;
1666         struct pipeline_msg_req *req;
1667         struct pipeline_msg_rsp *rsp;
1668         int status;
1669
1670         /* Check input params */
1671         if ((pipeline_name == NULL) ||
1672                 (data == NULL) ||
1673                 (stats == NULL))
1674                 return -1;
1675
1676         p = pipeline_find(pipeline_name);
1677         if ((p == NULL) ||
1678                 (table_id >= p->n_tables))
1679                 return -1;
1680
1681         if (!pipeline_is_running(p)) {
1682                 struct rte_table_action *a = p->table[table_id].a;
1683
1684                 status = rte_table_action_stats_read(a,
1685                         data,
1686                         stats,
1687                         clear);
1688
1689                 return status;
1690         }
1691
1692         /* Allocate request */
1693         req = pipeline_msg_alloc();
1694         if (req == NULL)
1695                 return -1;
1696
1697         /* Write request */
1698         req->type = PIPELINE_REQ_TABLE_RULE_STATS_READ;
1699         req->id = table_id;
1700         req->table_rule_stats_read.data = data;
1701         req->table_rule_stats_read.clear = clear;
1702
1703         /* Send request and wait for response */
1704         rsp = pipeline_msg_send_recv(p, req);
1705         if (rsp == NULL)
1706                 return -1;
1707
1708         /* Read response */
1709         status = rsp->status;
1710         if (status)
1711                 memcpy(stats, &rsp->table_rule_stats_read.stats, sizeof(*stats));
1712
1713         /* Free response */
1714         pipeline_msg_free(rsp);
1715
1716         return status;
1717 }
1718
1719 int
1720 pipeline_table_mtr_profile_add(const char *pipeline_name,
1721         uint32_t table_id,
1722         uint32_t meter_profile_id,
1723         struct rte_table_action_meter_profile *profile)
1724 {
1725         struct pipeline *p;
1726         struct pipeline_msg_req *req;
1727         struct pipeline_msg_rsp *rsp;
1728         int status;
1729
1730         /* Check input params */
1731         if ((pipeline_name == NULL) ||
1732                 (profile == NULL))
1733                 return -1;
1734
1735         p = pipeline_find(pipeline_name);
1736         if ((p == NULL) ||
1737                 (table_id >= p->n_tables))
1738                 return -1;
1739
1740         if (!pipeline_is_running(p)) {
1741                 struct rte_table_action *a = p->table[table_id].a;
1742
1743                 status = rte_table_action_meter_profile_add(a,
1744                         meter_profile_id,
1745                         profile);
1746
1747                 return status;
1748         }
1749
1750         /* Allocate request */
1751         req = pipeline_msg_alloc();
1752         if (req == NULL)
1753                 return -1;
1754
1755         /* Write request */
1756         req->type = PIPELINE_REQ_TABLE_MTR_PROFILE_ADD;
1757         req->id = table_id;
1758         req->table_mtr_profile_add.meter_profile_id = meter_profile_id;
1759         memcpy(&req->table_mtr_profile_add.profile, profile, sizeof(*profile));
1760
1761         /* Send request and wait for response */
1762         rsp = pipeline_msg_send_recv(p, req);
1763         if (rsp == NULL)
1764                 return -1;
1765
1766         /* Read response */
1767         status = rsp->status;
1768
1769         /* Free response */
1770         pipeline_msg_free(rsp);
1771
1772         return status;
1773 }
1774
1775 int
1776 pipeline_table_mtr_profile_delete(const char *pipeline_name,
1777         uint32_t table_id,
1778         uint32_t meter_profile_id)
1779 {
1780         struct pipeline *p;
1781         struct pipeline_msg_req *req;
1782         struct pipeline_msg_rsp *rsp;
1783         int status;
1784
1785         /* Check input params */
1786         if (pipeline_name == NULL)
1787                 return -1;
1788
1789         p = pipeline_find(pipeline_name);
1790         if ((p == NULL) ||
1791                 (table_id >= p->n_tables))
1792                 return -1;
1793
1794         if (!pipeline_is_running(p)) {
1795                 struct rte_table_action *a = p->table[table_id].a;
1796
1797                 status = rte_table_action_meter_profile_delete(a,
1798                                 meter_profile_id);
1799
1800                 return status;
1801         }
1802
1803         /* Allocate request */
1804         req = pipeline_msg_alloc();
1805         if (req == NULL)
1806                 return -1;
1807
1808         /* Write request */
1809         req->type = PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE;
1810         req->id = table_id;
1811         req->table_mtr_profile_delete.meter_profile_id = meter_profile_id;
1812
1813         /* Send request and wait for response */
1814         rsp = pipeline_msg_send_recv(p, req);
1815         if (rsp == NULL)
1816                 return -1;
1817
1818         /* Read response */
1819         status = rsp->status;
1820
1821         /* Free response */
1822         pipeline_msg_free(rsp);
1823
1824         return status;
1825 }
1826
1827 int
1828 pipeline_table_rule_mtr_read(const char *pipeline_name,
1829         uint32_t table_id,
1830         void *data,
1831         uint32_t tc_mask,
1832         struct rte_table_action_mtr_counters *stats,
1833         int clear)
1834 {
1835         struct pipeline *p;
1836         struct pipeline_msg_req *req;
1837         struct pipeline_msg_rsp *rsp;
1838         int status;
1839
1840         /* Check input params */
1841         if ((pipeline_name == NULL) ||
1842                 (data == NULL) ||
1843                 (stats == NULL))
1844                 return -1;
1845
1846         p = pipeline_find(pipeline_name);
1847         if ((p == NULL) ||
1848                 (table_id >= p->n_tables))
1849                 return -1;
1850
1851         if (!pipeline_is_running(p)) {
1852                 struct rte_table_action *a = p->table[table_id].a;
1853
1854                 status = rte_table_action_meter_read(a,
1855                                 data,
1856                                 tc_mask,
1857                                 stats,
1858                                 clear);
1859
1860                 return status;
1861         }
1862
1863         /* Allocate request */
1864         req = pipeline_msg_alloc();
1865         if (req == NULL)
1866                 return -1;
1867
1868         /* Write request */
1869         req->type = PIPELINE_REQ_TABLE_RULE_MTR_READ;
1870         req->id = table_id;
1871         req->table_rule_mtr_read.data = data;
1872         req->table_rule_mtr_read.tc_mask = tc_mask;
1873         req->table_rule_mtr_read.clear = clear;
1874
1875         /* Send request and wait for response */
1876         rsp = pipeline_msg_send_recv(p, req);
1877         if (rsp == NULL)
1878                 return -1;
1879
1880         /* Read response */
1881         status = rsp->status;
1882         if (status)
1883                 memcpy(stats, &rsp->table_rule_mtr_read.stats, sizeof(*stats));
1884
1885         /* Free response */
1886         pipeline_msg_free(rsp);
1887
1888         return status;
1889 }
1890
1891 int
1892 pipeline_table_dscp_table_update(const char *pipeline_name,
1893         uint32_t table_id,
1894         uint64_t dscp_mask,
1895         struct rte_table_action_dscp_table *dscp_table)
1896 {
1897         struct pipeline *p;
1898         struct pipeline_msg_req *req;
1899         struct pipeline_msg_rsp *rsp;
1900         int status;
1901
1902         /* Check input params */
1903         if ((pipeline_name == NULL) ||
1904                 (dscp_table == NULL))
1905                 return -1;
1906
1907         p = pipeline_find(pipeline_name);
1908         if ((p == NULL) ||
1909                 (table_id >= p->n_tables))
1910                 return -1;
1911
1912         if (!pipeline_is_running(p)) {
1913                 struct rte_table_action *a = p->table[table_id].a;
1914
1915                 status = rte_table_action_dscp_table_update(a,
1916                                 dscp_mask,
1917                                 dscp_table);
1918
1919                 return status;
1920         }
1921
1922         /* Allocate request */
1923         req = pipeline_msg_alloc();
1924         if (req == NULL)
1925                 return -1;
1926
1927         /* Write request */
1928         req->type = PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE;
1929         req->id = table_id;
1930         req->table_dscp_table_update.dscp_mask = dscp_mask;
1931         memcpy(&req->table_dscp_table_update.dscp_table,
1932                 dscp_table, sizeof(*dscp_table));
1933
1934         /* Send request and wait for response */
1935         rsp = pipeline_msg_send_recv(p, req);
1936         if (rsp == NULL)
1937                 return -1;
1938
1939         /* Read response */
1940         status = rsp->status;
1941
1942         /* Free response */
1943         pipeline_msg_free(rsp);
1944
1945         return status;
1946 }
1947
1948 int
1949 pipeline_table_rule_ttl_read(const char *pipeline_name,
1950         uint32_t table_id,
1951         void *data,
1952         struct rte_table_action_ttl_counters *stats,
1953         int clear)
1954 {
1955         struct pipeline *p;
1956         struct pipeline_msg_req *req;
1957         struct pipeline_msg_rsp *rsp;
1958         int status;
1959
1960         /* Check input params */
1961         if ((pipeline_name == NULL) ||
1962                 (data == NULL) ||
1963                 (stats == NULL))
1964                 return -1;
1965
1966         p = pipeline_find(pipeline_name);
1967         if ((p == NULL) ||
1968                 (table_id >= p->n_tables))
1969                 return -1;
1970
1971         if (!pipeline_is_running(p)) {
1972                 struct rte_table_action *a = p->table[table_id].a;
1973
1974                 status = rte_table_action_ttl_read(a,
1975                                 data,
1976                                 stats,
1977                                 clear);
1978
1979                 return status;
1980         }
1981
1982         /* Allocate request */
1983         req = pipeline_msg_alloc();
1984         if (req == NULL)
1985                 return -1;
1986
1987         /* Write request */
1988         req->type = PIPELINE_REQ_TABLE_RULE_TTL_READ;
1989         req->id = table_id;
1990         req->table_rule_ttl_read.data = data;
1991         req->table_rule_ttl_read.clear = clear;
1992
1993         /* Send request and wait for response */
1994         rsp = pipeline_msg_send_recv(p, req);
1995         if (rsp == NULL)
1996                 return -1;
1997
1998         /* Read response */
1999         status = rsp->status;
2000         if (status)
2001                 memcpy(stats, &rsp->table_rule_ttl_read.stats, sizeof(*stats));
2002
2003         /* Free response */
2004         pipeline_msg_free(rsp);
2005
2006         return status;
2007 }
2008
2009 /**
2010  * Data plane threads: message handling
2011  */
2012 static inline struct pipeline_msg_req *
2013 pipeline_msg_recv(struct rte_ring *msgq_req)
2014 {
2015         struct pipeline_msg_req *req;
2016
2017         int status = rte_ring_sc_dequeue(msgq_req, (void **) &req);
2018
2019         if (status != 0)
2020                 return NULL;
2021
2022         return req;
2023 }
2024
2025 static inline void
2026 pipeline_msg_send(struct rte_ring *msgq_rsp,
2027         struct pipeline_msg_rsp *rsp)
2028 {
2029         int status;
2030
2031         do {
2032                 status = rte_ring_sp_enqueue(msgq_rsp, rsp);
2033         } while (status == -ENOBUFS);
2034 }
2035
2036 static struct pipeline_msg_rsp *
2037 pipeline_msg_handle_port_in_stats_read(struct pipeline_data *p,
2038         struct pipeline_msg_req *req)
2039 {
2040         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2041         uint32_t port_id = req->id;
2042         int clear = req->port_in_stats_read.clear;
2043
2044         rsp->status = rte_pipeline_port_in_stats_read(p->p,
2045                 port_id,
2046                 &rsp->port_in_stats_read.stats,
2047                 clear);
2048
2049         return rsp;
2050 }
2051
2052 static struct pipeline_msg_rsp *
2053 pipeline_msg_handle_port_in_enable(struct pipeline_data *p,
2054         struct pipeline_msg_req *req)
2055 {
2056         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2057         uint32_t port_id = req->id;
2058
2059         rsp->status = rte_pipeline_port_in_enable(p->p,
2060                 port_id);
2061
2062         return rsp;
2063 }
2064
2065 static struct pipeline_msg_rsp *
2066 pipeline_msg_handle_port_in_disable(struct pipeline_data *p,
2067         struct pipeline_msg_req *req)
2068 {
2069         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2070         uint32_t port_id = req->id;
2071
2072         rsp->status = rte_pipeline_port_in_disable(p->p,
2073                 port_id);
2074
2075         return rsp;
2076 }
2077
2078 static struct pipeline_msg_rsp *
2079 pipeline_msg_handle_port_out_stats_read(struct pipeline_data *p,
2080         struct pipeline_msg_req *req)
2081 {
2082         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2083         uint32_t port_id = req->id;
2084         int clear = req->port_out_stats_read.clear;
2085
2086         rsp->status = rte_pipeline_port_out_stats_read(p->p,
2087                 port_id,
2088                 &rsp->port_out_stats_read.stats,
2089                 clear);
2090
2091         return rsp;
2092 }
2093
2094 static struct pipeline_msg_rsp *
2095 pipeline_msg_handle_table_stats_read(struct pipeline_data *p,
2096         struct pipeline_msg_req *req)
2097 {
2098         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2099         uint32_t port_id = req->id;
2100         int clear = req->table_stats_read.clear;
2101
2102         rsp->status = rte_pipeline_table_stats_read(p->p,
2103                 port_id,
2104                 &rsp->table_stats_read.stats,
2105                 clear);
2106
2107         return rsp;
2108 }
2109
2110 static int
2111 match_convert_ipv6_depth(uint32_t depth, uint32_t *depth32)
2112 {
2113         if (depth > 128)
2114                 return -1;
2115
2116         switch (depth / 32) {
2117         case 0:
2118                 depth32[0] = depth;
2119                 depth32[1] = 0;
2120                 depth32[2] = 0;
2121                 depth32[3] = 0;
2122                 return 0;
2123
2124         case 1:
2125                 depth32[0] = 32;
2126                 depth32[1] = depth - 32;
2127                 depth32[2] = 0;
2128                 depth32[3] = 0;
2129                 return 0;
2130
2131         case 2:
2132                 depth32[0] = 32;
2133                 depth32[1] = 32;
2134                 depth32[2] = depth - 64;
2135                 depth32[3] = 0;
2136                 return 0;
2137
2138         case 3:
2139                 depth32[0] = 32;
2140                 depth32[1] = 32;
2141                 depth32[2] = 32;
2142                 depth32[3] = depth - 96;
2143                 return 0;
2144
2145         case 4:
2146                 depth32[0] = 32;
2147                 depth32[1] = 32;
2148                 depth32[2] = 32;
2149                 depth32[3] = 32;
2150                 return 0;
2151
2152         default:
2153                 return -1;
2154         }
2155 }
2156
2157 static int
2158 match_convert(struct table_rule_match *mh,
2159         union table_rule_match_low_level *ml,
2160         int add)
2161 {
2162         memset(ml, 0, sizeof(*ml));
2163
2164         switch (mh->match_type) {
2165         case TABLE_ACL:
2166                 if (mh->match.acl.ip_version)
2167                         if (add) {
2168                                 ml->acl_add.field_value[0].value.u8 =
2169                                         mh->match.acl.proto;
2170                                 ml->acl_add.field_value[0].mask_range.u8 =
2171                                         mh->match.acl.proto_mask;
2172
2173                                 ml->acl_add.field_value[1].value.u32 =
2174                                         mh->match.acl.ipv4.sa;
2175                                 ml->acl_add.field_value[1].mask_range.u32 =
2176                                         mh->match.acl.sa_depth;
2177
2178                                 ml->acl_add.field_value[2].value.u32 =
2179                                         mh->match.acl.ipv4.da;
2180                                 ml->acl_add.field_value[2].mask_range.u32 =
2181                                         mh->match.acl.da_depth;
2182
2183                                 ml->acl_add.field_value[3].value.u16 =
2184                                         mh->match.acl.sp0;
2185                                 ml->acl_add.field_value[3].mask_range.u16 =
2186                                         mh->match.acl.sp1;
2187
2188                                 ml->acl_add.field_value[4].value.u16 =
2189                                         mh->match.acl.dp0;
2190                                 ml->acl_add.field_value[4].mask_range.u16 =
2191                                         mh->match.acl.dp1;
2192
2193                                 ml->acl_add.priority =
2194                                         (int32_t) mh->match.acl.priority;
2195                         } else {
2196                                 ml->acl_delete.field_value[0].value.u8 =
2197                                         mh->match.acl.proto;
2198                                 ml->acl_delete.field_value[0].mask_range.u8 =
2199                                         mh->match.acl.proto_mask;
2200
2201                                 ml->acl_delete.field_value[1].value.u32 =
2202                                         mh->match.acl.ipv4.sa;
2203                                 ml->acl_delete.field_value[1].mask_range.u32 =
2204                                         mh->match.acl.sa_depth;
2205
2206                                 ml->acl_delete.field_value[2].value.u32 =
2207                                         mh->match.acl.ipv4.da;
2208                                 ml->acl_delete.field_value[2].mask_range.u32 =
2209                                         mh->match.acl.da_depth;
2210
2211                                 ml->acl_delete.field_value[3].value.u16 =
2212                                         mh->match.acl.sp0;
2213                                 ml->acl_delete.field_value[3].mask_range.u16 =
2214                                         mh->match.acl.sp1;
2215
2216                                 ml->acl_delete.field_value[4].value.u16 =
2217                                         mh->match.acl.dp0;
2218                                 ml->acl_delete.field_value[4].mask_range.u16 =
2219                                         mh->match.acl.dp1;
2220                         }
2221                 else
2222                         if (add) {
2223                                 uint32_t *sa32 =
2224                                         (uint32_t *) mh->match.acl.ipv6.sa;
2225                                 uint32_t *da32 =
2226                                         (uint32_t *) mh->match.acl.ipv6.da;
2227                                 uint32_t sa32_depth[4], da32_depth[4];
2228                                 int status;
2229
2230                                 status = match_convert_ipv6_depth(
2231                                         mh->match.acl.sa_depth,
2232                                         sa32_depth);
2233                                 if (status)
2234                                         return status;
2235
2236                                 status = match_convert_ipv6_depth(
2237                                         mh->match.acl.da_depth,
2238                                         da32_depth);
2239                                 if (status)
2240                                         return status;
2241
2242                                 ml->acl_add.field_value[0].value.u8 =
2243                                         mh->match.acl.proto;
2244                                 ml->acl_add.field_value[0].mask_range.u8 =
2245                                         mh->match.acl.proto_mask;
2246
2247                                 ml->acl_add.field_value[1].value.u32 = sa32[0];
2248                                 ml->acl_add.field_value[1].mask_range.u32 =
2249                                         sa32_depth[0];
2250                                 ml->acl_add.field_value[2].value.u32 = sa32[1];
2251                                 ml->acl_add.field_value[2].mask_range.u32 =
2252                                         sa32_depth[1];
2253                                 ml->acl_add.field_value[3].value.u32 = sa32[2];
2254                                 ml->acl_add.field_value[3].mask_range.u32 =
2255                                         sa32_depth[2];
2256                                 ml->acl_add.field_value[4].value.u32 = sa32[3];
2257                                 ml->acl_add.field_value[4].mask_range.u32 =
2258                                         sa32_depth[3];
2259
2260                                 ml->acl_add.field_value[5].value.u32 = da32[0];
2261                                 ml->acl_add.field_value[5].mask_range.u32 =
2262                                         da32_depth[0];
2263                                 ml->acl_add.field_value[6].value.u32 = da32[1];
2264                                 ml->acl_add.field_value[6].mask_range.u32 =
2265                                         da32_depth[1];
2266                                 ml->acl_add.field_value[7].value.u32 = da32[2];
2267                                 ml->acl_add.field_value[7].mask_range.u32 =
2268                                         da32_depth[2];
2269                                 ml->acl_add.field_value[8].value.u32 = da32[3];
2270                                 ml->acl_add.field_value[8].mask_range.u32 =
2271                                         da32_depth[3];
2272
2273                                 ml->acl_add.field_value[9].value.u16 =
2274                                         mh->match.acl.sp0;
2275                                 ml->acl_add.field_value[9].mask_range.u16 =
2276                                         mh->match.acl.sp1;
2277
2278                                 ml->acl_add.field_value[10].value.u16 =
2279                                         mh->match.acl.dp0;
2280                                 ml->acl_add.field_value[10].mask_range.u16 =
2281                                         mh->match.acl.dp1;
2282
2283                                 ml->acl_add.priority =
2284                                         (int32_t) mh->match.acl.priority;
2285                         } else {
2286                                 uint32_t *sa32 =
2287                                         (uint32_t *) mh->match.acl.ipv6.sa;
2288                                 uint32_t *da32 =
2289                                         (uint32_t *) mh->match.acl.ipv6.da;
2290                                 uint32_t sa32_depth[4], da32_depth[4];
2291                                 int status;
2292
2293                                 status = match_convert_ipv6_depth(
2294                                         mh->match.acl.sa_depth,
2295                                         sa32_depth);
2296                                 if (status)
2297                                         return status;
2298
2299                                 status = match_convert_ipv6_depth(
2300                                         mh->match.acl.da_depth,
2301                                         da32_depth);
2302                                 if (status)
2303                                         return status;
2304
2305                                 ml->acl_delete.field_value[0].value.u8 =
2306                                         mh->match.acl.proto;
2307                                 ml->acl_delete.field_value[0].mask_range.u8 =
2308                                         mh->match.acl.proto_mask;
2309
2310                                 ml->acl_delete.field_value[1].value.u32 =
2311                                         sa32[0];
2312                                 ml->acl_delete.field_value[1].mask_range.u32 =
2313                                         sa32_depth[0];
2314                                 ml->acl_delete.field_value[2].value.u32 =
2315                                         sa32[1];
2316                                 ml->acl_delete.field_value[2].mask_range.u32 =
2317                                         sa32_depth[1];
2318                                 ml->acl_delete.field_value[3].value.u32 =
2319                                         sa32[2];
2320                                 ml->acl_delete.field_value[3].mask_range.u32 =
2321                                         sa32_depth[2];
2322                                 ml->acl_delete.field_value[4].value.u32 =
2323                                         sa32[3];
2324                                 ml->acl_delete.field_value[4].mask_range.u32 =
2325                                         sa32_depth[3];
2326
2327                                 ml->acl_delete.field_value[5].value.u32 =
2328                                         da32[0];
2329                                 ml->acl_delete.field_value[5].mask_range.u32 =
2330                                         da32_depth[0];
2331                                 ml->acl_delete.field_value[6].value.u32 =
2332                                         da32[1];
2333                                 ml->acl_delete.field_value[6].mask_range.u32 =
2334                                         da32_depth[1];
2335                                 ml->acl_delete.field_value[7].value.u32 =
2336                                         da32[2];
2337                                 ml->acl_delete.field_value[7].mask_range.u32 =
2338                                         da32_depth[2];
2339                                 ml->acl_delete.field_value[8].value.u32 =
2340                                         da32[3];
2341                                 ml->acl_delete.field_value[8].mask_range.u32 =
2342                                         da32_depth[3];
2343
2344                                 ml->acl_delete.field_value[9].value.u16 =
2345                                         mh->match.acl.sp0;
2346                                 ml->acl_delete.field_value[9].mask_range.u16 =
2347                                         mh->match.acl.sp1;
2348
2349                                 ml->acl_delete.field_value[10].value.u16 =
2350                                         mh->match.acl.dp0;
2351                                 ml->acl_delete.field_value[10].mask_range.u16 =
2352                                         mh->match.acl.dp1;
2353                         }
2354                 return 0;
2355
2356         case TABLE_ARRAY:
2357                 ml->array.pos = mh->match.array.pos;
2358                 return 0;
2359
2360         case TABLE_HASH:
2361                 memcpy(ml->hash, mh->match.hash.key, sizeof(ml->hash));
2362                 return 0;
2363
2364         case TABLE_LPM:
2365                 if (mh->match.lpm.ip_version) {
2366                         ml->lpm_ipv4.ip = mh->match.lpm.ipv4;
2367                         ml->lpm_ipv4.depth = mh->match.lpm.depth;
2368                 } else {
2369                         memcpy(ml->lpm_ipv6.ip,
2370                                 mh->match.lpm.ipv6, sizeof(ml->lpm_ipv6.ip));
2371                         ml->lpm_ipv6.depth = mh->match.lpm.depth;
2372                 }
2373
2374                 return 0;
2375
2376         default:
2377                 return -1;
2378         }
2379 }
2380
2381 static int
2382 action_convert(struct rte_table_action *a,
2383         struct table_rule_action *action,
2384         struct rte_pipeline_table_entry *data)
2385 {
2386         int status;
2387
2388         /* Apply actions */
2389         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
2390                 status = rte_table_action_apply(a,
2391                         data,
2392                         RTE_TABLE_ACTION_FWD,
2393                         &action->fwd);
2394
2395                 if (status)
2396                         return status;
2397         }
2398
2399         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) {
2400                 status = rte_table_action_apply(a,
2401                         data,
2402                         RTE_TABLE_ACTION_LB,
2403                         &action->lb);
2404
2405                 if (status)
2406                         return status;
2407         }
2408
2409         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
2410                 status = rte_table_action_apply(a,
2411                         data,
2412                         RTE_TABLE_ACTION_MTR,
2413                         &action->mtr);
2414
2415                 if (status)
2416                         return status;
2417         }
2418
2419         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
2420                 status = rte_table_action_apply(a,
2421                         data,
2422                         RTE_TABLE_ACTION_TM,
2423                         &action->tm);
2424
2425                 if (status)
2426                         return status;
2427         }
2428
2429         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
2430                 status = rte_table_action_apply(a,
2431                         data,
2432                         RTE_TABLE_ACTION_ENCAP,
2433                         &action->encap);
2434
2435                 if (status)
2436                         return status;
2437         }
2438
2439         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
2440                 status = rte_table_action_apply(a,
2441                         data,
2442                         RTE_TABLE_ACTION_NAT,
2443                         &action->nat);
2444
2445                 if (status)
2446                         return status;
2447         }
2448
2449         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
2450                 status = rte_table_action_apply(a,
2451                         data,
2452                         RTE_TABLE_ACTION_TTL,
2453                         &action->ttl);
2454
2455                 if (status)
2456                         return status;
2457         }
2458
2459         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
2460                 status = rte_table_action_apply(a,
2461                         data,
2462                         RTE_TABLE_ACTION_STATS,
2463                         &action->stats);
2464
2465                 if (status)
2466                         return status;
2467         }
2468
2469         if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
2470                 status = rte_table_action_apply(a,
2471                         data,
2472                         RTE_TABLE_ACTION_TIME,
2473                         &action->time);
2474
2475                 if (status)
2476                         return status;
2477         }
2478
2479         return 0;
2480 }
2481
2482 static struct pipeline_msg_rsp *
2483 pipeline_msg_handle_table_rule_add(struct pipeline_data *p,
2484         struct pipeline_msg_req *req)
2485 {
2486         union table_rule_match_low_level match_ll;
2487         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2488         struct table_rule_match *match = &req->table_rule_add.match;
2489         struct table_rule_action *action = &req->table_rule_add.action;
2490         struct rte_pipeline_table_entry *data_in, *data_out;
2491         uint32_t table_id = req->id;
2492         int key_found, status;
2493         struct rte_table_action *a = p->table_data[table_id].a;
2494
2495         /* Apply actions */
2496         memset(p->buffer, 0, sizeof(p->buffer));
2497         data_in = (struct rte_pipeline_table_entry *) p->buffer;
2498
2499         status = match_convert(match, &match_ll, 1);
2500         if (status) {
2501                 rsp->status = -1;
2502                 return rsp;
2503         }
2504
2505         status = action_convert(a, action, data_in);
2506         if (status) {
2507                 rsp->status = -1;
2508                 return rsp;
2509         }
2510
2511         status = rte_pipeline_table_entry_add(p->p,
2512                 table_id,
2513                 &match_ll,
2514                 data_in,
2515                 &key_found,
2516                 &data_out);
2517         if (status) {
2518                 rsp->status = -1;
2519                 return rsp;
2520         }
2521
2522         /* Write response */
2523         rsp->status = 0;
2524         rsp->table_rule_add.data = data_out;
2525
2526         return rsp;
2527 }
2528
2529 static struct pipeline_msg_rsp *
2530 pipeline_msg_handle_table_rule_add_default(struct pipeline_data *p,
2531         struct pipeline_msg_req *req)
2532 {
2533         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2534         struct table_rule_action *action = &req->table_rule_add_default.action;
2535         struct rte_pipeline_table_entry *data_in, *data_out;
2536         uint32_t table_id = req->id;
2537         int status;
2538
2539         /* Apply actions */
2540         memset(p->buffer, 0, sizeof(p->buffer));
2541         data_in = (struct rte_pipeline_table_entry *) p->buffer;
2542
2543         data_in->action = action->fwd.action;
2544         if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
2545                 data_in->port_id = action->fwd.id;
2546         if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
2547                 data_in->table_id = action->fwd.id;
2548
2549         /* Add default rule to table */
2550         status = rte_pipeline_table_default_entry_add(p->p,
2551                 table_id,
2552                 data_in,
2553                 &data_out);
2554         if (status) {
2555                 rsp->status = -1;
2556                 return rsp;
2557         }
2558
2559         /* Write response */
2560         rsp->status = 0;
2561         rsp->table_rule_add_default.data = data_out;
2562
2563         return rsp;
2564 }
2565
2566 static struct pipeline_msg_rsp *
2567 pipeline_msg_handle_table_rule_add_bulk(struct pipeline_data *p,
2568         struct pipeline_msg_req *req)
2569 {
2570
2571         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2572
2573         uint32_t table_id = req->id;
2574         struct table_rule_match *match = req->table_rule_add_bulk.match;
2575         struct table_rule_action *action = req->table_rule_add_bulk.action;
2576         struct rte_pipeline_table_entry **data =
2577                 (struct rte_pipeline_table_entry **)req->table_rule_add_bulk.data;
2578         uint32_t n_rules = req->table_rule_add_bulk.n_rules;
2579         uint32_t bulk = req->table_rule_add_bulk.bulk;
2580
2581         struct rte_table_action *a = p->table_data[table_id].a;
2582         union table_rule_match_low_level *match_ll;
2583         uint8_t *action_ll;
2584         void **match_ll_ptr;
2585         struct rte_pipeline_table_entry **action_ll_ptr;
2586         int *found, status;
2587         uint32_t i;
2588
2589         /* Memory allocation */
2590         match_ll = calloc(n_rules, sizeof(union table_rule_match_low_level));
2591         action_ll = calloc(n_rules, TABLE_RULE_ACTION_SIZE_MAX);
2592         match_ll_ptr = calloc(n_rules, sizeof(void *));
2593         action_ll_ptr =
2594                 calloc(n_rules, sizeof(struct rte_pipeline_table_entry *));
2595         found = calloc(n_rules, sizeof(int));
2596
2597         if ((match_ll == NULL) ||
2598                 (action_ll == NULL) ||
2599                 (match_ll_ptr == NULL) ||
2600                 (action_ll_ptr == NULL) ||
2601                 (found == NULL))
2602                 goto fail;
2603
2604         for (i = 0; i < n_rules; i++) {
2605                 match_ll_ptr[i] = (void *)&match_ll[i];
2606                 action_ll_ptr[i] =
2607                         (struct rte_pipeline_table_entry *)&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
2608         }
2609
2610         /* Rule match conversion */
2611         for (i = 0; i < n_rules; i++) {
2612                 status = match_convert(&match[i], match_ll_ptr[i], 1);
2613                 if (status)
2614                         goto fail;
2615         }
2616
2617         /* Rule action conversion */
2618         for (i = 0; i < n_rules; i++) {
2619                 status = action_convert(a, &action[i], action_ll_ptr[i]);
2620                 if (status)
2621                         goto fail;
2622         }
2623
2624         /* Add rule (match, action) to table */
2625         if (bulk) {
2626                 status = rte_pipeline_table_entry_add_bulk(p->p,
2627                         table_id,
2628                         match_ll_ptr,
2629                         action_ll_ptr,
2630                         n_rules,
2631                         found,
2632                         data);
2633                 if (status)
2634                         n_rules = 0;
2635         } else
2636                 for (i = 0; i < n_rules; i++) {
2637                         status = rte_pipeline_table_entry_add(p->p,
2638                                 table_id,
2639                                 match_ll_ptr[i],
2640                                 action_ll_ptr[i],
2641                                 &found[i],
2642                                 &data[i]);
2643                         if (status) {
2644                                 n_rules = i;
2645                                 break;
2646                         }
2647                 }
2648
2649         /* Write response */
2650         rsp->status = 0;
2651         rsp->table_rule_add_bulk.n_rules = n_rules;
2652
2653         /* Free */
2654         free(found);
2655         free(action_ll_ptr);
2656         free(match_ll_ptr);
2657         free(action_ll);
2658         free(match_ll);
2659
2660         return rsp;
2661
2662 fail:
2663         free(found);
2664         free(action_ll_ptr);
2665         free(match_ll_ptr);
2666         free(action_ll);
2667         free(match_ll);
2668
2669         rsp->status = -1;
2670         rsp->table_rule_add_bulk.n_rules = 0;
2671         return rsp;
2672 }
2673
2674 static struct pipeline_msg_rsp *
2675 pipeline_msg_handle_table_rule_delete(struct pipeline_data *p,
2676         struct pipeline_msg_req *req)
2677 {
2678         union table_rule_match_low_level match_ll;
2679         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2680         struct table_rule_match *match = &req->table_rule_delete.match;
2681         uint32_t table_id = req->id;
2682         int key_found, status;
2683
2684         status = match_convert(match, &match_ll, 0);
2685         if (status) {
2686                 rsp->status = -1;
2687                 return rsp;
2688         }
2689
2690         rsp->status = rte_pipeline_table_entry_delete(p->p,
2691                 table_id,
2692                 &match_ll,
2693                 &key_found,
2694                 NULL);
2695
2696         return rsp;
2697 }
2698
2699 static struct pipeline_msg_rsp *
2700 pipeline_msg_handle_table_rule_delete_default(struct pipeline_data *p,
2701         struct pipeline_msg_req *req)
2702 {
2703         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2704         uint32_t table_id = req->id;
2705
2706         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
2707                 table_id,
2708                 NULL);
2709
2710         return rsp;
2711 }
2712
2713 static struct pipeline_msg_rsp *
2714 pipeline_msg_handle_table_rule_stats_read(struct pipeline_data *p,
2715         struct pipeline_msg_req *req)
2716 {
2717         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2718         uint32_t table_id = req->id;
2719         void *data = req->table_rule_stats_read.data;
2720         int clear = req->table_rule_stats_read.clear;
2721         struct rte_table_action *a = p->table_data[table_id].a;
2722
2723         rsp->status = rte_table_action_stats_read(a,
2724                 data,
2725                 &rsp->table_rule_stats_read.stats,
2726                 clear);
2727
2728         return rsp;
2729 }
2730
2731 static struct pipeline_msg_rsp *
2732 pipeline_msg_handle_table_mtr_profile_add(struct pipeline_data *p,
2733         struct pipeline_msg_req *req)
2734 {
2735         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2736         uint32_t table_id = req->id;
2737         uint32_t meter_profile_id = req->table_mtr_profile_add.meter_profile_id;
2738         struct rte_table_action_meter_profile *profile =
2739                 &req->table_mtr_profile_add.profile;
2740         struct rte_table_action *a = p->table_data[table_id].a;
2741
2742         rsp->status = rte_table_action_meter_profile_add(a,
2743                 meter_profile_id,
2744                 profile);
2745
2746         return rsp;
2747 }
2748
2749 static struct pipeline_msg_rsp *
2750 pipeline_msg_handle_table_mtr_profile_delete(struct pipeline_data *p,
2751         struct pipeline_msg_req *req)
2752 {
2753         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2754         uint32_t table_id = req->id;
2755         uint32_t meter_profile_id =
2756                 req->table_mtr_profile_delete.meter_profile_id;
2757         struct rte_table_action *a = p->table_data[table_id].a;
2758
2759         rsp->status = rte_table_action_meter_profile_delete(a,
2760                 meter_profile_id);
2761
2762         return rsp;
2763 }
2764
2765 static struct pipeline_msg_rsp *
2766 pipeline_msg_handle_table_rule_mtr_read(struct pipeline_data *p,
2767         struct pipeline_msg_req *req)
2768 {
2769         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2770         uint32_t table_id = req->id;
2771         void *data = req->table_rule_mtr_read.data;
2772         uint32_t tc_mask = req->table_rule_mtr_read.tc_mask;
2773         int clear = req->table_rule_mtr_read.clear;
2774         struct rte_table_action *a = p->table_data[table_id].a;
2775
2776         rsp->status = rte_table_action_meter_read(a,
2777                 data,
2778                 tc_mask,
2779                 &rsp->table_rule_mtr_read.stats,
2780                 clear);
2781
2782         return rsp;
2783 }
2784
2785 static struct pipeline_msg_rsp *
2786 pipeline_msg_handle_table_dscp_table_update(struct pipeline_data *p,
2787         struct pipeline_msg_req *req)
2788 {
2789         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2790         uint32_t table_id = req->id;
2791         uint64_t dscp_mask = req->table_dscp_table_update.dscp_mask;
2792         struct rte_table_action_dscp_table *dscp_table =
2793                 &req->table_dscp_table_update.dscp_table;
2794         struct rte_table_action *a = p->table_data[table_id].a;
2795
2796         rsp->status = rte_table_action_dscp_table_update(a,
2797                 dscp_mask,
2798                 dscp_table);
2799
2800         return rsp;
2801 }
2802
2803 static struct pipeline_msg_rsp *
2804 pipeline_msg_handle_table_rule_ttl_read(struct pipeline_data *p,
2805         struct pipeline_msg_req *req)
2806 {
2807         struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
2808         uint32_t table_id = req->id;
2809         void *data = req->table_rule_ttl_read.data;
2810         int clear = req->table_rule_ttl_read.clear;
2811         struct rte_table_action *a = p->table_data[table_id].a;
2812
2813         rsp->status = rte_table_action_ttl_read(a,
2814                 data,
2815                 &rsp->table_rule_ttl_read.stats,
2816                 clear);
2817
2818         return rsp;
2819 }
2820
2821 static void
2822 pipeline_msg_handle(struct pipeline_data *p)
2823 {
2824         for ( ; ; ) {
2825                 struct pipeline_msg_req *req;
2826                 struct pipeline_msg_rsp *rsp;
2827
2828                 req = pipeline_msg_recv(p->msgq_req);
2829                 if (req == NULL)
2830                         break;
2831
2832                 switch (req->type) {
2833                 case PIPELINE_REQ_PORT_IN_STATS_READ:
2834                         rsp = pipeline_msg_handle_port_in_stats_read(p, req);
2835                         break;
2836
2837                 case PIPELINE_REQ_PORT_IN_ENABLE:
2838                         rsp = pipeline_msg_handle_port_in_enable(p, req);
2839                         break;
2840
2841                 case PIPELINE_REQ_PORT_IN_DISABLE:
2842                         rsp = pipeline_msg_handle_port_in_disable(p, req);
2843                         break;
2844
2845                 case PIPELINE_REQ_PORT_OUT_STATS_READ:
2846                         rsp = pipeline_msg_handle_port_out_stats_read(p, req);
2847                         break;
2848
2849                 case PIPELINE_REQ_TABLE_STATS_READ:
2850                         rsp = pipeline_msg_handle_table_stats_read(p, req);
2851                         break;
2852
2853                 case PIPELINE_REQ_TABLE_RULE_ADD:
2854                         rsp = pipeline_msg_handle_table_rule_add(p, req);
2855                         break;
2856
2857                 case PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT:
2858                         rsp = pipeline_msg_handle_table_rule_add_default(p,     req);
2859                         break;
2860
2861                 case PIPELINE_REQ_TABLE_RULE_ADD_BULK:
2862                         rsp = pipeline_msg_handle_table_rule_add_bulk(p, req);
2863                         break;
2864
2865                 case PIPELINE_REQ_TABLE_RULE_DELETE:
2866                         rsp = pipeline_msg_handle_table_rule_delete(p, req);
2867                         break;
2868
2869                 case PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT:
2870                         rsp = pipeline_msg_handle_table_rule_delete_default(p, req);
2871                         break;
2872
2873                 case PIPELINE_REQ_TABLE_RULE_STATS_READ:
2874                         rsp = pipeline_msg_handle_table_rule_stats_read(p, req);
2875                         break;
2876
2877                 case PIPELINE_REQ_TABLE_MTR_PROFILE_ADD:
2878                         rsp = pipeline_msg_handle_table_mtr_profile_add(p, req);
2879                         break;
2880
2881                 case PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE:
2882                         rsp = pipeline_msg_handle_table_mtr_profile_delete(p, req);
2883                         break;
2884
2885                 case PIPELINE_REQ_TABLE_RULE_MTR_READ:
2886                         rsp = pipeline_msg_handle_table_rule_mtr_read(p, req);
2887                         break;
2888
2889                 case PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE:
2890                         rsp = pipeline_msg_handle_table_dscp_table_update(p, req);
2891                         break;
2892
2893                 case PIPELINE_REQ_TABLE_RULE_TTL_READ:
2894                         rsp = pipeline_msg_handle_table_rule_ttl_read(p, req);
2895                         break;
2896
2897                 default:
2898                         rsp = (struct pipeline_msg_rsp *) req;
2899                         rsp->status = -1;
2900                 }
2901
2902                 pipeline_msg_send(p->msgq_rsp, rsp);
2903         }
2904 }
2905
2906 /**
2907  * Data plane threads: main
2908  */
2909 int
2910 thread_main(void *arg __rte_unused)
2911 {
2912         struct thread_data *t;
2913         uint32_t thread_id, i;
2914
2915         thread_id = rte_lcore_id();
2916         t = &thread_data[thread_id];
2917
2918         /* Dispatch loop */
2919         for (i = 0; ; i++) {
2920                 uint32_t j;
2921
2922                 /* Data Plane */
2923                 for (j = 0; j < t->n_pipelines; j++)
2924                         rte_pipeline_run(t->p[j]);
2925
2926                 /* Control Plane */
2927                 if ((i & 0xF) == 0) {
2928                         uint64_t time = rte_get_tsc_cycles();
2929                         uint64_t time_next_min = UINT64_MAX;
2930
2931                         if (time < t->time_next_min)
2932                                 continue;
2933
2934                         /* Pipeline message queues */
2935                         for (j = 0; j < t->n_pipelines; j++) {
2936                                 struct pipeline_data *p =
2937                                         &t->pipeline_data[j];
2938                                 uint64_t time_next = p->time_next;
2939
2940                                 if (time_next <= time) {
2941                                         pipeline_msg_handle(p);
2942                                         rte_pipeline_flush(p->p);
2943                                         time_next = time + p->timer_period;
2944                                         p->time_next = time_next;
2945                                 }
2946
2947                                 if (time_next < time_next_min)
2948                                         time_next_min = time_next;
2949                         }
2950
2951                         /* Thread message queues */
2952                         {
2953                                 uint64_t time_next = t->time_next;
2954
2955                                 if (time_next <= time) {
2956                                         thread_msg_handle(t);
2957                                         time_next = time + t->timer_period;
2958                                         t->time_next = time_next;
2959                                 }
2960
2961                                 if (time_next < time_next_min)
2962                                         time_next_min = time_next;
2963                         }
2964
2965                         t->time_next_min = time_next_min;
2966                 }
2967         }
2968
2969         return 0;
2970 }