2 * Zebra dataplane layer.
3 * Copyright (c) 2018 Volta Networks, Inc.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; see the file COPYING; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 #include "lib/libfrr.h"
21 #include "lib/debug.h"
22 #include "lib/frratomic.h"
23 #include "lib/frr_pthread.h"
24 #include "lib/memory.h"
25 #include "lib/queue.h"
26 #include "lib/zebra.h"
27 #include "zebra/zebra_memory.h"
28 #include "zebra/zserv.h"
29 #include "zebra/zebra_dplane.h"
31 #include "zebra/debug.h"
33 /* Memory type for context blocks */
34 DEFINE_MTYPE(ZEBRA
, DP_CTX
, "Zebra DPlane Ctx")
35 DEFINE_MTYPE(ZEBRA
, DP_PROV
, "Zebra DPlane Provider")
41 /* Default value for max queued incoming updates */
42 const uint32_t DPLANE_DEFAULT_MAX_QUEUED
= 200;
45 /* Validation check macro for context blocks */
46 /* #define DPLANE_DEBUG 1 */
50 # define DPLANE_CTX_VALID(p) \
55 # define DPLANE_CTX_VALID(p)
57 #endif /* DPLANE_DEBUG */
60 * The context block used to exchange info about route updates across
61 * the boundary between the zebra main context (and pthread) and the
62 * dataplane layer (and pthread).
64 struct zebra_dplane_ctx
{
67 enum dplane_op_e zd_op
;
69 /* Status on return */
70 enum zebra_dplane_result zd_status
;
72 /* TODO -- internal/sub-operation status? */
73 enum zebra_dplane_result zd_remote_status
;
74 enum zebra_dplane_result zd_kernel_status
;
76 /* Dest and (optional) source prefixes */
77 struct prefix zd_dest
;
94 route_tag_t zd_old_tag
;
96 uint32_t zd_old_metric
;
98 uint16_t zd_old_instance
;
101 uint8_t zd_old_distance
;
104 uint32_t zd_nexthop_mtu
;
107 struct zebra_dplane_info zd_ns_info
;
110 struct nexthop_group zd_ng
;
112 /* "Previous" nexthops, used only in route updates without netlink */
113 struct nexthop_group zd_old_ng
;
115 /* TODO -- use fixed array of nexthops, to avoid mallocs? */
117 /* Embedded list linkage */
118 TAILQ_ENTRY(zebra_dplane_ctx
) zd_q_entries
;
122 * Registration block for one dataplane provider.
124 struct zebra_dplane_provider
{
126 char dp_name
[DPLANE_PROVIDER_NAMELEN
+ 1];
128 /* Priority, for ordering among providers */
134 dplane_provider_process_fp dp_fp
;
136 dplane_provider_fini_fp dp_fini
;
138 _Atomic
uint32_t dp_in_counter
;
139 _Atomic
uint32_t dp_error_counter
;
141 /* Embedded list linkage */
142 TAILQ_ENTRY(zebra_dplane_provider
) dp_q_providers
;
149 static struct zebra_dplane_globals
{
150 /* Mutex to control access to dataplane components */
151 pthread_mutex_t dg_mutex
;
153 /* Results callback registered by zebra 'core' */
154 dplane_results_fp dg_results_cb
;
156 /* Sentinel for beginning of shutdown */
157 volatile bool dg_is_shutdown
;
159 /* Sentinel for end of shutdown */
160 volatile bool dg_run
;
162 /* Route-update context queue inbound to the dataplane */
163 TAILQ_HEAD(zdg_ctx_q
, zebra_dplane_ctx
) dg_route_ctx_q
;
165 /* Ordered list of providers */
166 TAILQ_HEAD(zdg_prov_q
, zebra_dplane_provider
) dg_providers_q
;
168 /* Counter used to assign internal ids to providers */
169 uint32_t dg_provider_id
;
171 /* Limit number of pending, unprocessed updates */
172 _Atomic
uint32_t dg_max_queued_updates
;
174 _Atomic
uint32_t dg_routes_in
;
175 _Atomic
uint32_t dg_routes_queued
;
176 _Atomic
uint32_t dg_routes_queued_max
;
177 _Atomic
uint32_t dg_route_errors
;
179 /* Dataplane pthread */
180 struct frr_pthread
*dg_pthread
;
182 /* Event-delivery context 'master' for the dplane */
183 struct thread_master
*dg_master
;
185 /* Event/'thread' pointer for queued updates */
186 struct thread
*dg_t_update
;
188 /* Event pointer for pending shutdown check loop */
189 struct thread
*dg_t_shutdown_check
;
194 * Lock and unlock for interactions with the zebra 'core'
196 #define DPLANE_LOCK() pthread_mutex_lock(&zdplane_info.dg_mutex)
198 #define DPLANE_UNLOCK() pthread_mutex_unlock(&zdplane_info.dg_mutex)
201 static int dplane_route_process(struct thread
*event
);
208 * Allocate a dataplane update context
210 static struct zebra_dplane_ctx
*dplane_ctx_alloc(void)
212 struct zebra_dplane_ctx
*p
;
214 /* TODO -- just alloc'ing memory, but would like to maintain
217 p
= XCALLOC(MTYPE_DP_CTX
, sizeof(struct zebra_dplane_ctx
));
223 * Free a dataplane results context.
225 static void dplane_ctx_free(struct zebra_dplane_ctx
**pctx
)
228 DPLANE_CTX_VALID(*pctx
);
230 /* TODO -- just freeing memory, but would like to maintain
234 /* Free embedded nexthops */
235 if ((*pctx
)->zd_ng
.nexthop
) {
236 /* This deals with recursive nexthops too */
237 nexthops_free((*pctx
)->zd_ng
.nexthop
);
240 if ((*pctx
)->zd_old_ng
.nexthop
) {
241 /* This deals with recursive nexthops too */
242 nexthops_free((*pctx
)->zd_old_ng
.nexthop
);
245 XFREE(MTYPE_DP_CTX
, *pctx
);
251 * Return a context block to the dplane module after processing
253 void dplane_ctx_fini(struct zebra_dplane_ctx
**pctx
)
255 /* TODO -- enqueue for next provider; for now, just free */
256 dplane_ctx_free(pctx
);
259 /* Enqueue a context block */
260 void dplane_ctx_enqueue_tail(struct dplane_ctx_q
*q
,
261 const struct zebra_dplane_ctx
*ctx
)
263 TAILQ_INSERT_TAIL(q
, (struct zebra_dplane_ctx
*)ctx
, zd_q_entries
);
266 /* Dequeue a context block from the head of a list */
267 void dplane_ctx_dequeue(struct dplane_ctx_q
*q
, struct zebra_dplane_ctx
**ctxp
)
269 struct zebra_dplane_ctx
*ctx
= TAILQ_FIRST(q
);
272 TAILQ_REMOVE(q
, ctx
, zd_q_entries
);
278 * Accessors for information from the context object
280 enum zebra_dplane_result
dplane_ctx_get_status(
281 const struct zebra_dplane_ctx
*ctx
)
283 DPLANE_CTX_VALID(ctx
);
285 return ctx
->zd_status
;
288 enum dplane_op_e
dplane_ctx_get_op(const struct zebra_dplane_ctx
*ctx
)
290 DPLANE_CTX_VALID(ctx
);
295 const char *dplane_op2str(enum dplane_op_e op
)
297 const char *ret
= "UNKNOWN";
305 case DPLANE_OP_ROUTE_INSTALL
:
306 ret
= "ROUTE_INSTALL";
308 case DPLANE_OP_ROUTE_UPDATE
:
309 ret
= "ROUTE_UPDATE";
311 case DPLANE_OP_ROUTE_DELETE
:
312 ret
= "ROUTE_DELETE";
320 const char *dplane_res2str(enum zebra_dplane_result res
)
322 const char *ret
= "<Unknown>";
325 case ZEBRA_DPLANE_REQUEST_FAILURE
:
328 case ZEBRA_DPLANE_REQUEST_QUEUED
:
331 case ZEBRA_DPLANE_REQUEST_SUCCESS
:
339 const struct prefix
*dplane_ctx_get_dest(const struct zebra_dplane_ctx
*ctx
)
341 DPLANE_CTX_VALID(ctx
);
343 return &(ctx
->zd_dest
);
346 /* Source prefix is a little special - return NULL for "no src prefix" */
347 const struct prefix
*dplane_ctx_get_src(const struct zebra_dplane_ctx
*ctx
)
349 DPLANE_CTX_VALID(ctx
);
351 if (ctx
->zd_src
.prefixlen
== 0 &&
352 IN6_IS_ADDR_UNSPECIFIED(&(ctx
->zd_src
.u
.prefix6
))) {
355 return &(ctx
->zd_src
);
359 bool dplane_ctx_is_update(const struct zebra_dplane_ctx
*ctx
)
361 DPLANE_CTX_VALID(ctx
);
363 return ctx
->zd_is_update
;
366 uint32_t dplane_ctx_get_seq(const struct zebra_dplane_ctx
*ctx
)
368 DPLANE_CTX_VALID(ctx
);
373 uint32_t dplane_ctx_get_old_seq(const struct zebra_dplane_ctx
*ctx
)
375 DPLANE_CTX_VALID(ctx
);
377 return ctx
->zd_old_seq
;
380 vrf_id_t
dplane_ctx_get_vrf(const struct zebra_dplane_ctx
*ctx
)
382 DPLANE_CTX_VALID(ctx
);
384 return ctx
->zd_vrf_id
;
387 int dplane_ctx_get_type(const struct zebra_dplane_ctx
*ctx
)
389 DPLANE_CTX_VALID(ctx
);
394 int dplane_ctx_get_old_type(const struct zebra_dplane_ctx
*ctx
)
396 DPLANE_CTX_VALID(ctx
);
398 return ctx
->zd_old_type
;
401 afi_t
dplane_ctx_get_afi(const struct zebra_dplane_ctx
*ctx
)
403 DPLANE_CTX_VALID(ctx
);
408 safi_t
dplane_ctx_get_safi(const struct zebra_dplane_ctx
*ctx
)
410 DPLANE_CTX_VALID(ctx
);
415 uint32_t dplane_ctx_get_table(const struct zebra_dplane_ctx
*ctx
)
417 DPLANE_CTX_VALID(ctx
);
419 return ctx
->zd_table_id
;
422 route_tag_t
dplane_ctx_get_tag(const struct zebra_dplane_ctx
*ctx
)
424 DPLANE_CTX_VALID(ctx
);
429 route_tag_t
dplane_ctx_get_old_tag(const struct zebra_dplane_ctx
*ctx
)
431 DPLANE_CTX_VALID(ctx
);
433 return ctx
->zd_old_tag
;
436 uint16_t dplane_ctx_get_instance(const struct zebra_dplane_ctx
*ctx
)
438 DPLANE_CTX_VALID(ctx
);
440 return ctx
->zd_instance
;
443 uint16_t dplane_ctx_get_old_instance(const struct zebra_dplane_ctx
*ctx
)
445 DPLANE_CTX_VALID(ctx
);
447 return ctx
->zd_old_instance
;
450 uint32_t dplane_ctx_get_metric(const struct zebra_dplane_ctx
*ctx
)
452 DPLANE_CTX_VALID(ctx
);
454 return ctx
->zd_metric
;
457 uint32_t dplane_ctx_get_old_metric(const struct zebra_dplane_ctx
*ctx
)
459 DPLANE_CTX_VALID(ctx
);
461 return ctx
->zd_old_metric
;
464 uint32_t dplane_ctx_get_mtu(const struct zebra_dplane_ctx
*ctx
)
466 DPLANE_CTX_VALID(ctx
);
471 uint32_t dplane_ctx_get_nh_mtu(const struct zebra_dplane_ctx
*ctx
)
473 DPLANE_CTX_VALID(ctx
);
475 return ctx
->zd_nexthop_mtu
;
478 uint8_t dplane_ctx_get_distance(const struct zebra_dplane_ctx
*ctx
)
480 DPLANE_CTX_VALID(ctx
);
482 return ctx
->zd_distance
;
485 uint8_t dplane_ctx_get_old_distance(const struct zebra_dplane_ctx
*ctx
)
487 DPLANE_CTX_VALID(ctx
);
489 return ctx
->zd_old_distance
;
492 const struct nexthop_group
*dplane_ctx_get_ng(
493 const struct zebra_dplane_ctx
*ctx
)
495 DPLANE_CTX_VALID(ctx
);
497 return &(ctx
->zd_ng
);
500 const struct nexthop_group
*dplane_ctx_get_old_ng(
501 const struct zebra_dplane_ctx
*ctx
)
503 DPLANE_CTX_VALID(ctx
);
505 return &(ctx
->zd_old_ng
);
508 const struct zebra_dplane_info
*dplane_ctx_get_ns(
509 const struct zebra_dplane_ctx
*ctx
)
511 DPLANE_CTX_VALID(ctx
);
513 return &(ctx
->zd_ns_info
);
517 * End of dplane context accessors
521 * Retrieve the limit on the number of pending, unprocessed updates.
523 uint32_t dplane_get_in_queue_limit(void)
525 return atomic_load_explicit(&zdplane_info
.dg_max_queued_updates
,
526 memory_order_relaxed
);
530 * Configure limit on the number of pending, queued updates.
532 void dplane_set_in_queue_limit(uint32_t limit
, bool set
)
534 /* Reset to default on 'unset' */
536 limit
= DPLANE_DEFAULT_MAX_QUEUED
;
538 atomic_store_explicit(&zdplane_info
.dg_max_queued_updates
, limit
,
539 memory_order_relaxed
);
543 * Retrieve the current queue depth of incoming, unprocessed updates
545 uint32_t dplane_get_in_queue_len(void)
547 return atomic_load_explicit(&zdplane_info
.dg_routes_queued
,
548 memory_order_seq_cst
);
552 * Initialize a context block for a route update from zebra data structs.
554 static int dplane_ctx_route_init(struct zebra_dplane_ctx
*ctx
,
556 struct route_node
*rn
,
557 struct route_entry
*re
)
560 const struct route_table
*table
= NULL
;
561 const rib_table_info_t
*info
;
562 const struct prefix
*p
, *src_p
;
563 struct zebra_ns
*zns
;
564 struct zebra_vrf
*zvrf
;
565 struct nexthop
*nexthop
;
567 if (!ctx
|| !rn
|| !re
)
572 ctx
->zd_type
= re
->type
;
573 ctx
->zd_old_type
= re
->type
;
575 /* Prefixes: dest, and optional source */
576 srcdest_rnode_prefixes(rn
, &p
, &src_p
);
578 prefix_copy(&(ctx
->zd_dest
), p
);
581 prefix_copy(&(ctx
->zd_src
), src_p
);
583 memset(&(ctx
->zd_src
), 0, sizeof(ctx
->zd_src
));
585 ctx
->zd_table_id
= re
->table
;
587 ctx
->zd_metric
= re
->metric
;
588 ctx
->zd_old_metric
= re
->metric
;
589 ctx
->zd_vrf_id
= re
->vrf_id
;
590 ctx
->zd_mtu
= re
->mtu
;
591 ctx
->zd_nexthop_mtu
= re
->nexthop_mtu
;
592 ctx
->zd_instance
= re
->instance
;
593 ctx
->zd_tag
= re
->tag
;
594 ctx
->zd_old_tag
= re
->tag
;
595 ctx
->zd_distance
= re
->distance
;
597 table
= srcdest_rnode_table(rn
);
600 ctx
->zd_afi
= info
->afi
;
601 ctx
->zd_safi
= info
->safi
;
603 /* Extract ns info - can't use pointers to 'core' structs */
604 zvrf
= vrf_info_lookup(re
->vrf_id
);
607 zebra_dplane_info_from_zns(&(ctx
->zd_ns_info
), zns
, true /*is_cmd*/);
609 #if defined(HAVE_NETLINK)
610 /* Increment message counter after copying to context struct - may need
611 * two messages in some 'update' cases.
613 if (op
== DPLANE_OP_ROUTE_UPDATE
)
614 zns
->netlink_cmd
.seq
+= 2;
616 zns
->netlink_cmd
.seq
++;
619 /* Copy nexthops; recursive info is included too */
620 copy_nexthops(&(ctx
->zd_ng
.nexthop
), re
->ng
.nexthop
, NULL
);
622 /* TODO -- maybe use array of nexthops to avoid allocs? */
624 /* Ensure that the dplane's nexthop flag is clear. */
625 for (ALL_NEXTHOPS(ctx
->zd_ng
, nexthop
))
626 UNSET_FLAG(nexthop
->flags
, NEXTHOP_FLAG_FIB
);
628 /* Trying out the sequence number idea, so we can try to detect
629 * when a result is stale.
631 re
->dplane_sequence
++;
632 ctx
->zd_seq
= re
->dplane_sequence
;
641 * Enqueue a new route update,
642 * and ensure an event is active for the dataplane thread.
644 static int dplane_route_enqueue(struct zebra_dplane_ctx
*ctx
)
649 /* Enqueue for processing by the dataplane thread */
652 TAILQ_INSERT_TAIL(&zdplane_info
.dg_route_ctx_q
, ctx
,
657 curr
= atomic_add_fetch_explicit(
659 /* TODO -- issue with the clang atomic/intrinsics currently;
660 * casting away the 'Atomic'-ness of the variable works.
662 (uint32_t *)&(zdplane_info
.dg_routes_queued
),
664 &(zdplane_info
.dg_routes_queued
),
666 1, memory_order_seq_cst
);
668 /* Maybe update high-water counter also */
669 high
= atomic_load_explicit(&zdplane_info
.dg_routes_queued_max
,
670 memory_order_seq_cst
);
671 while (high
< curr
) {
672 if (atomic_compare_exchange_weak_explicit(
673 &zdplane_info
.dg_routes_queued_max
,
675 memory_order_seq_cst
,
676 memory_order_seq_cst
))
680 /* Ensure that an event for the dataplane thread is active */
681 thread_add_event(zdplane_info
.dg_master
, dplane_route_process
, NULL
, 0,
682 &zdplane_info
.dg_t_update
);
690 * Attempt to dequeue a route-update block
692 static struct zebra_dplane_ctx
*dplane_route_dequeue(void)
694 struct zebra_dplane_ctx
*ctx
= NULL
;
698 ctx
= TAILQ_FIRST(&zdplane_info
.dg_route_ctx_q
);
700 TAILQ_REMOVE(&zdplane_info
.dg_route_ctx_q
,
710 * Utility that prepares a route update and enqueues it for processing
712 static enum zebra_dplane_result
713 dplane_route_update_internal(struct route_node
*rn
,
714 struct route_entry
*re
,
715 struct route_entry
*old_re
,
718 enum zebra_dplane_result result
= ZEBRA_DPLANE_REQUEST_FAILURE
;
720 struct zebra_dplane_ctx
*ctx
= NULL
;
722 /* Obtain context block */
723 ctx
= dplane_ctx_alloc();
729 /* Init context with info from zebra data structs */
730 ret
= dplane_ctx_route_init(ctx
, op
, rn
, re
);
732 /* Capture some extra info for update case
733 * where there's a different 'old' route.
735 if ((op
== DPLANE_OP_ROUTE_UPDATE
) &&
736 old_re
&& (old_re
!= re
)) {
737 ctx
->zd_is_update
= true;
739 old_re
->dplane_sequence
++;
740 ctx
->zd_old_seq
= old_re
->dplane_sequence
;
742 ctx
->zd_old_tag
= old_re
->tag
;
743 ctx
->zd_old_type
= old_re
->type
;
744 ctx
->zd_old_instance
= old_re
->instance
;
745 ctx
->zd_old_distance
= old_re
->distance
;
746 ctx
->zd_old_metric
= old_re
->metric
;
749 /* For bsd, capture previous re's nexthops too, sigh.
750 * We'll need these to do per-nexthop deletes.
752 copy_nexthops(&(ctx
->zd_old_ng
.nexthop
),
753 old_re
->ng
.nexthop
, NULL
);
754 #endif /* !HAVE_NETLINK */
757 /* Enqueue context for processing */
758 ret
= dplane_route_enqueue(ctx
);
763 atomic_fetch_add_explicit(&zdplane_info
.dg_routes_in
, 1,
764 memory_order_relaxed
);
767 result
= ZEBRA_DPLANE_REQUEST_QUEUED
;
769 atomic_fetch_add_explicit(&zdplane_info
.dg_route_errors
, 1,
770 memory_order_relaxed
);
771 dplane_ctx_free(&ctx
);
778 * Enqueue a route 'add' for the dataplane.
780 enum zebra_dplane_result
dplane_route_add(struct route_node
*rn
,
781 struct route_entry
*re
)
783 enum zebra_dplane_result ret
= ZEBRA_DPLANE_REQUEST_FAILURE
;
785 if (rn
== NULL
|| re
== NULL
)
788 ret
= dplane_route_update_internal(rn
, re
, NULL
,
789 DPLANE_OP_ROUTE_INSTALL
);
796 * Enqueue a route update for the dataplane.
798 enum zebra_dplane_result
dplane_route_update(struct route_node
*rn
,
799 struct route_entry
*re
,
800 struct route_entry
*old_re
)
802 enum zebra_dplane_result ret
= ZEBRA_DPLANE_REQUEST_FAILURE
;
804 if (rn
== NULL
|| re
== NULL
)
807 ret
= dplane_route_update_internal(rn
, re
, old_re
,
808 DPLANE_OP_ROUTE_UPDATE
);
814 * Enqueue a route removal for the dataplane.
816 enum zebra_dplane_result
dplane_route_delete(struct route_node
*rn
,
817 struct route_entry
*re
)
819 enum zebra_dplane_result ret
= ZEBRA_DPLANE_REQUEST_FAILURE
;
821 if (rn
== NULL
|| re
== NULL
)
824 ret
= dplane_route_update_internal(rn
, re
, NULL
,
825 DPLANE_OP_ROUTE_DELETE
);
832 * Event handler function for routing updates
834 static int dplane_route_process(struct thread
*event
)
836 enum zebra_dplane_result res
;
837 struct zebra_dplane_ctx
*ctx
;
840 /* Check for shutdown */
841 if (!zdplane_info
.dg_run
)
844 /* TODO -- limit number of updates per cycle? */
845 ctx
= dplane_route_dequeue();
850 atomic_fetch_sub_explicit(&zdplane_info
.dg_routes_queued
, 1,
851 memory_order_relaxed
);
853 if (IS_ZEBRA_DEBUG_DPLANE_DETAIL
) {
854 char dest_str
[PREFIX_STRLEN
];
856 prefix2str(dplane_ctx_get_dest(ctx
),
857 dest_str
, sizeof(dest_str
));
859 zlog_debug("%u:%s Dplane route update ctx %p op %s",
860 dplane_ctx_get_vrf(ctx
), dest_str
,
861 ctx
, dplane_op2str(dplane_ctx_get_op(ctx
)));
864 /* TODO -- support series of providers */
866 /* Initially, just doing kernel-facing update here */
867 res
= kernel_route_update(ctx
);
869 if (res
!= ZEBRA_DPLANE_REQUEST_SUCCESS
)
870 atomic_fetch_add_explicit(&zdplane_info
.dg_route_errors
,
871 1, memory_order_relaxed
);
873 ctx
->zd_status
= res
;
875 /* Enqueue result to zebra main context */
876 zdplane_info
.dg_results_cb(ctx
);
885 * Handler for 'show dplane'
887 int dplane_show_helper(struct vty
*vty
, bool detailed
)
889 uint64_t queued
, limit
, queue_max
, errs
, incoming
;
891 /* Using atomics because counters are being changed in different
894 incoming
= atomic_load_explicit(&zdplane_info
.dg_routes_in
,
895 memory_order_relaxed
);
896 limit
= atomic_load_explicit(&zdplane_info
.dg_max_queued_updates
,
897 memory_order_relaxed
);
898 queued
= atomic_load_explicit(&zdplane_info
.dg_routes_queued
,
899 memory_order_relaxed
);
900 queue_max
= atomic_load_explicit(&zdplane_info
.dg_routes_queued_max
,
901 memory_order_relaxed
);
902 errs
= atomic_load_explicit(&zdplane_info
.dg_route_errors
,
903 memory_order_relaxed
);
905 vty_out(vty
, "Route updates: %"PRIu64
"\n", incoming
);
906 vty_out(vty
, "Route update errors: %"PRIu64
"\n", errs
);
907 vty_out(vty
, "Route update queue limit: %"PRIu64
"\n", limit
);
908 vty_out(vty
, "Route update queue depth: %"PRIu64
"\n", queued
);
909 vty_out(vty
, "Route update queue max: %"PRIu64
"\n", queue_max
);
915 * Handler for 'show dplane providers'
917 int dplane_show_provs_helper(struct vty
*vty
, bool detailed
)
919 vty_out(vty
, "Zebra dataplane providers:%s\n",
920 (detailed
? " (detailed)" : ""));
926 * Provider registration
928 int dplane_provider_register(const char *name
,
929 enum dplane_provider_prio_e prio
,
930 dplane_provider_process_fp fp
,
931 dplane_provider_fini_fp fini_fp
)
934 struct zebra_dplane_provider
*p
, *last
;
942 if (prio
<= DPLANE_PRIO_NONE
||
943 prio
> DPLANE_PRIO_LAST
) {
948 /* Allocate and init new provider struct */
949 p
= XCALLOC(MTYPE_DP_PROV
, sizeof(struct zebra_dplane_provider
));
955 strncpy(p
->dp_name
, name
, DPLANE_PROVIDER_NAMELEN
);
956 p
->dp_name
[DPLANE_PROVIDER_NAMELEN
] = '\0'; /* Belt-and-suspenders */
958 p
->dp_priority
= prio
;
960 p
->dp_fini
= fini_fp
;
962 /* Lock the lock - the dplane pthread may be running */
965 p
->dp_id
= ++zdplane_info
.dg_provider_id
;
967 /* Insert into list ordered by priority */
968 TAILQ_FOREACH(last
, &zdplane_info
.dg_providers_q
, dp_q_providers
) {
969 if (last
->dp_priority
> p
->dp_priority
)
974 TAILQ_INSERT_BEFORE(last
, p
, dp_q_providers
);
976 TAILQ_INSERT_TAIL(&zdplane_info
.dg_providers_q
, p
,
987 * Zebra registers a results callback with the dataplane system
989 int dplane_results_register(dplane_results_fp fp
)
991 zdplane_info
.dg_results_cb
= fp
;
996 * Initialize the dataplane module during startup, internal/private version
998 static void zebra_dplane_init_internal(struct zebra_t
*zebra
)
1000 memset(&zdplane_info
, 0, sizeof(zdplane_info
));
1002 pthread_mutex_init(&zdplane_info
.dg_mutex
, NULL
);
1004 TAILQ_INIT(&zdplane_info
.dg_route_ctx_q
);
1005 TAILQ_INIT(&zdplane_info
.dg_providers_q
);
1007 zdplane_info
.dg_max_queued_updates
= DPLANE_DEFAULT_MAX_QUEUED
;
1009 /* TODO -- register default kernel 'provider' during init */
1010 zdplane_info
.dg_run
= true;
1012 /* Start dataplane pthread */
1014 zdplane_info
.dg_run
= true;
1016 struct frr_pthread_attr pattr
= {
1017 .start
= frr_pthread_attr_default
.start
,
1018 .stop
= frr_pthread_attr_default
.stop
1021 zdplane_info
.dg_pthread
= frr_pthread_new(&pattr
, "Zebra dplane thread",
1024 zdplane_info
.dg_master
= zdplane_info
.dg_pthread
->master
;
1026 frr_pthread_run(zdplane_info
.dg_pthread
, NULL
);
1029 /* Indicates zebra shutdown/exit is in progress. Some operations may be
1030 * simplified or skipped during shutdown processing.
1032 bool dplane_is_in_shutdown(void)
1034 return zdplane_info
.dg_is_shutdown
;
1038 * Early or pre-shutdown, de-init notification api. This runs pretty
1039 * early during zebra shutdown, as a signal to stop new work and prepare
1040 * for updates generated by shutdown/cleanup activity, as zebra tries to
1041 * remove everything it's responsible for.
1042 * NB: This runs in the main zebra thread context.
1044 void zebra_dplane_pre_finish(void)
1046 if (IS_ZEBRA_DEBUG_DPLANE
)
1047 zlog_debug("Zebra dataplane pre-fini called");
1049 zdplane_info
.dg_is_shutdown
= true;
1051 /* Notify provider(s) of pending shutdown */
1055 * Utility to determine whether work remains enqueued within the dplane;
1056 * used during system shutdown processing.
1058 static bool dplane_work_pending(void)
1060 struct zebra_dplane_ctx
*ctx
;
1062 /* TODO -- just checking incoming/pending work for now */
1065 ctx
= TAILQ_FIRST(&zdplane_info
.dg_route_ctx_q
);
1069 return (ctx
!= NULL
);
1073 * Shutdown-time intermediate callback, used to determine when all pending
1074 * in-flight updates are done. If there's still work to do, reschedules itself.
1075 * If all work is done, schedules an event to the main zebra thread for
1076 * final zebra shutdown.
1077 * This runs in the dplane pthread context.
1079 static int dplane_check_shutdown_status(struct thread
*event
)
1081 if (IS_ZEBRA_DEBUG_DPLANE
)
1082 zlog_debug("Zebra dataplane shutdown status check called");
1084 if (dplane_work_pending()) {
1085 /* Reschedule dplane check on a short timer */
1086 thread_add_timer_msec(zdplane_info
.dg_master
,
1087 dplane_check_shutdown_status
,
1089 &zdplane_info
.dg_t_shutdown_check
);
1091 /* TODO - give up and stop waiting after a short time? */
1094 /* We appear to be done - schedule a final callback event
1095 * for the zebra main pthread.
1097 thread_add_event(zebrad
.master
, zebra_finalize
, NULL
, 0, NULL
);
1104 * Shutdown, de-init api. This runs pretty late during shutdown,
1105 * after zebra has tried to free/remove/uninstall all routes during shutdown.
1106 * At this point, dplane work may still remain to be done, so we can't just
1107 * blindly terminate. If there's still work to do, we'll periodically check
1108 * and when done, we'll enqueue a task to the zebra main thread for final
1109 * termination processing.
1111 * NB: This runs in the main zebra thread context.
1113 void zebra_dplane_finish(void)
1115 if (IS_ZEBRA_DEBUG_DPLANE
)
1116 zlog_debug("Zebra dataplane fini called");
1118 thread_add_event(zdplane_info
.dg_master
,
1119 dplane_check_shutdown_status
, NULL
, 0,
1120 &zdplane_info
.dg_t_shutdown_check
);
1124 * Final phase of shutdown, after all work enqueued to dplane has been
1125 * processed. This is called from the zebra main pthread context.
1127 void zebra_dplane_shutdown(void)
1129 if (IS_ZEBRA_DEBUG_DPLANE
)
1130 zlog_debug("Zebra dataplane shutdown called");
1132 /* Stop dplane thread, if it's running */
1134 zdplane_info
.dg_run
= false;
1136 THREAD_OFF(zdplane_info
.dg_t_update
);
1138 frr_pthread_stop(zdplane_info
.dg_pthread
, NULL
);
1140 /* Destroy pthread */
1141 frr_pthread_destroy(zdplane_info
.dg_pthread
);
1142 zdplane_info
.dg_pthread
= NULL
;
1143 zdplane_info
.dg_master
= NULL
;
1145 /* Notify provider(s) of final shutdown */
1147 /* Clean-up provider objects */
1149 /* Clean queue(s) */
1153 * Initialize the dataplane module at startup; called by zebra rib_init()
1155 void zebra_dplane_init(void)
1157 zebra_dplane_init_internal(&zebrad
);