]> git.proxmox.com Git - mirror_frr.git/blobdiff - zebra/zebra_rnh.c
zebra, lib: fix the ZEBRA_INTERFACE_VRF_UPDATE zapi message
[mirror_frr.git] / zebra / zebra_rnh.c
index c9fb782ba615076f2092ea656dca5814c756d206..b1fbe8a6536de6dbdf2bf91b6f520ec8664a7266 100644 (file)
@@ -36,6 +36,7 @@
 #include "nexthop.h"
 #include "vrf.h"
 
+#include "zebra/zebra_router.h"
 #include "zebra/rib.h"
 #include "zebra/rt.h"
 #include "zebra/zserv.h"
@@ -47,6 +48,7 @@
 #include "zebra/zebra_routemap.h"
 #include "zebra/interface.h"
 #include "zebra/zebra_memory.h"
+#include "zebra/zebra_errors.h"
 
 static void free_state(vrf_id_t vrf_id, struct route_entry *re,
                       struct route_node *rn);
@@ -66,10 +68,16 @@ static int compare_state(struct route_entry *r1, struct route_entry *r2);
 static int send_client(struct rnh *rnh, struct zserv *client, rnh_type_t type,
                       vrf_id_t vrf_id);
 static void print_rnh(struct route_node *rn, struct vty *vty);
+static int zebra_client_cleanup_rnh(struct zserv *client);
 
 int zebra_rnh_ip_default_route = 0;
 int zebra_rnh_ipv6_default_route = 0;
 
+void zebra_rnh_init(void)
+{
+       hook_register(zserv_client_close, zebra_client_cleanup_rnh);
+}
+
 static inline struct route_table *get_rnh_table(vrf_id_t vrfid, int family,
                                                rnh_type_t type)
 {
@@ -96,7 +104,8 @@ char *rnh_str(struct rnh *rnh, char *buf, int size)
        return buf;
 }
 
-struct rnh *zebra_add_rnh(struct prefix *p, vrf_id_t vrfid, rnh_type_t type)
+struct rnh *zebra_add_rnh(struct prefix *p, vrf_id_t vrfid, rnh_type_t type,
+                         bool *exists)
 {
        struct route_table *table;
        struct route_node *rn;
@@ -110,8 +119,10 @@ struct rnh *zebra_add_rnh(struct prefix *p, vrf_id_t vrfid, rnh_type_t type)
        table = get_rnh_table(vrfid, PREFIX_FAMILY(p), type);
        if (!table) {
                prefix2str(p, buf, sizeof(buf));
-               zlog_warn("%u: Add RNH %s type %d - table not found", vrfid,
+               flog_warn(EC_ZEBRA_RNH_NO_TABLE,
+                         "%u: Add RNH %s type %d - table not found", vrfid,
                          buf, type);
+               exists = false;
                return NULL;
        }
 
@@ -125,12 +136,13 @@ struct rnh *zebra_add_rnh(struct prefix *p, vrf_id_t vrfid, rnh_type_t type)
                rnh = XCALLOC(MTYPE_RNH, sizeof(struct rnh));
                rnh->client_list = list_new();
                rnh->vrf_id = vrfid;
-               rnh->zebra_static_route_list = list_new();
                rnh->zebra_pseudowire_list = list_new();
                route_lock_node(rn);
                rn->info = rnh;
                rnh->node = rn;
-       }
+               *exists = false;
+       } else
+               *exists = true;
 
        route_unlock_node(rn);
        return (rn->info);
@@ -160,9 +172,8 @@ struct rnh *zebra_lookup_rnh(struct prefix *p, vrf_id_t vrfid, rnh_type_t type)
 void zebra_free_rnh(struct rnh *rnh)
 {
        rnh->flags |= ZEBRA_NHT_DELETED;
-       list_delete_and_null(&rnh->client_list);
-       list_delete_and_null(&rnh->zebra_static_route_list);
-       list_delete_and_null(&rnh->zebra_pseudowire_list);
+       list_delete(&rnh->client_list);
+       list_delete(&rnh->zebra_pseudowire_list);
        free_state(rnh->vrf_id, rnh->state, rnh->node);
        XFREE(MTYPE_RNH, rnh);
 }
@@ -185,6 +196,14 @@ void zebra_delete_rnh(struct rnh *rnh, rnh_type_t type)
        route_unlock_node(rn);
 }
 
+/*
+ * This code will send to the registering client
+ * the looked up rnh.
+ * For a rnh that was created, there is no data
+ * so it will send an empty nexthop group
+ * If rnh exists then we know it has been evaluated
+ * and as such it will have a resolved rnh.
+ */
 void zebra_add_rnh_client(struct rnh *rnh, struct zserv *client,
                          rnh_type_t type, vrf_id_t vrf_id)
 {
@@ -194,11 +213,14 @@ void zebra_add_rnh_client(struct rnh *rnh, struct zserv *client,
                           zebra_route_string(client->proto),
                           rnh_str(rnh, buf, sizeof(buf)), type);
        }
-       if (!listnode_lookup(rnh->client_list, client)) {
+       if (!listnode_lookup(rnh->client_list, client))
                listnode_add(rnh->client_list, client);
-               send_client(rnh, client, type,
-                           vrf_id); // Pending: check if its needed
-       }
+
+       /*
+        * We always need to respond with known information,
+        * currently multiple daemons expect this behavior
+        */
+       send_client(rnh, client, type, vrf_id);
 }
 
 void zebra_remove_rnh_client(struct rnh *rnh, struct zserv *client,
@@ -212,82 +234,10 @@ void zebra_remove_rnh_client(struct rnh *rnh, struct zserv *client,
        }
        listnode_delete(rnh->client_list, client);
        if (list_isempty(rnh->client_list)
-           && list_isempty(rnh->zebra_static_route_list)
            && list_isempty(rnh->zebra_pseudowire_list))
                zebra_delete_rnh(rnh, type);
 }
 
-void zebra_register_rnh_static_nh(vrf_id_t vrf_id, struct prefix *nh,
-                                 struct route_node *static_rn)
-{
-       struct rnh *rnh;
-
-       rnh = zebra_add_rnh(nh, vrf_id, RNH_NEXTHOP_TYPE);
-       if (rnh && !listnode_lookup(rnh->zebra_static_route_list, static_rn)) {
-               listnode_add(rnh->zebra_static_route_list, static_rn);
-       }
-}
-
-void zebra_deregister_rnh_static_nh(vrf_id_t vrf_id, struct prefix *nh,
-                                   struct route_node *static_rn)
-{
-       struct rnh *rnh;
-
-       rnh = zebra_lookup_rnh(nh, vrf_id, RNH_NEXTHOP_TYPE);
-       if (!rnh || (rnh->flags & ZEBRA_NHT_DELETED))
-               return;
-
-       listnode_delete(rnh->zebra_static_route_list, static_rn);
-
-       if (list_isempty(rnh->client_list)
-           && list_isempty(rnh->zebra_static_route_list)
-           && list_isempty(rnh->zebra_pseudowire_list))
-               zebra_delete_rnh(rnh, RNH_NEXTHOP_TYPE);
-}
-
-void zebra_deregister_rnh_static_nexthops(vrf_id_t vrf_id,
-                                         struct nexthop *nexthop,
-                                         struct route_node *rn)
-{
-       struct nexthop *nh;
-       struct prefix nh_p;
-
-       for (nh = nexthop; nh; nh = nh->next) {
-               switch (nh->type) {
-               case NEXTHOP_TYPE_IPV4:
-               case NEXTHOP_TYPE_IPV4_IFINDEX:
-                       nh_p.family = AF_INET;
-                       nh_p.prefixlen = IPV4_MAX_BITLEN;
-                       nh_p.u.prefix4 = nh->gate.ipv4;
-                       break;
-               case NEXTHOP_TYPE_IPV6:
-               case NEXTHOP_TYPE_IPV6_IFINDEX:
-                       nh_p.family = AF_INET6;
-                       nh_p.prefixlen = IPV6_MAX_BITLEN;
-                       nh_p.u.prefix6 = nh->gate.ipv6;
-                       break;
-               /*
-                * Not sure what really to do here, we are not
-                * supposed to have either of these for NHT
-                * and the code has no way to know what prefix
-                * to use.  So I'm going to just continue
-                * for the moment, which is preferable to
-                * what is currently happening which is a
-                * CRASH and BURN.
-                * Some simple testing shows that we
-                * are not leaving slag around for these
-                * skipped static routes.  Since
-                * they don't appear to be installed
-                */
-               case NEXTHOP_TYPE_IFINDEX:
-               case NEXTHOP_TYPE_BLACKHOLE:
-                       continue;
-                       break;
-               }
-               zebra_deregister_rnh_static_nh(vrf_id, &nh_p, rn);
-       }
-}
-
 /* XXX move this utility function elsewhere? */
 static void addr2hostprefix(int af, const union g_addr *addr,
                            struct prefix *prefix)
@@ -305,7 +255,7 @@ static void addr2hostprefix(int af, const union g_addr *addr,
                break;
        default:
                memset(prefix, 0, sizeof(*prefix));
-               zlog_warn("%s: unknown address family %d", __func__, af);
+               zlog_debug("%s: unknown address family %d", __func__, af);
                break;
        }
 }
@@ -314,13 +264,19 @@ void zebra_register_rnh_pseudowire(vrf_id_t vrf_id, struct zebra_pw *pw)
 {
        struct prefix nh;
        struct rnh *rnh;
+       bool exists;
+       struct zebra_vrf *zvrf;
+
+       zvrf = vrf_info_lookup(vrf_id);
+       if (!zvrf)
+               return;
 
        addr2hostprefix(pw->af, &pw->nexthop, &nh);
-       rnh = zebra_add_rnh(&nh, vrf_id, RNH_NEXTHOP_TYPE);
+       rnh = zebra_add_rnh(&nh, vrf_id, RNH_NEXTHOP_TYPE, &exists);
        if (rnh && !listnode_lookup(rnh->zebra_pseudowire_list, pw)) {
                listnode_add(rnh->zebra_pseudowire_list, pw);
                pw->rnh = rnh;
-               zebra_evaluate_rnh(vrf_id, pw->af, 1, RNH_NEXTHOP_TYPE, &nh);
+               zebra_evaluate_rnh(zvrf, pw->af, 1, RNH_NEXTHOP_TYPE, &nh);
        }
 }
 
@@ -336,7 +292,6 @@ void zebra_deregister_rnh_pseudowire(vrf_id_t vrf_id, struct zebra_pw *pw)
        pw->rnh = NULL;
 
        if (list_isempty(rnh->client_list)
-           && list_isempty(rnh->zebra_static_route_list)
            && list_isempty(rnh->zebra_pseudowire_list))
                zebra_delete_rnh(rnh, RNH_NEXTHOP_TYPE);
 }
@@ -344,7 +299,8 @@ void zebra_deregister_rnh_pseudowire(vrf_id_t vrf_id, struct zebra_pw *pw)
 /* Apply the NHT route-map for a client to the route (and nexthops)
  * resolving a NH.
  */
-static int zebra_rnh_apply_nht_rmap(int family, struct route_node *prn,
+static int zebra_rnh_apply_nht_rmap(int family, struct zebra_vrf *zvrf,
+                                   struct route_node *prn,
                                    struct route_entry *re, int proto)
 {
        int at_least_one = 0;
@@ -357,8 +313,8 @@ static int zebra_rnh_apply_nht_rmap(int family, struct route_node *prn,
        if (prn && re) {
                for (nexthop = re->ng.nexthop; nexthop;
                     nexthop = nexthop->next) {
-                       ret = zebra_nht_route_map_check(rmap_family, proto,
-                                                       &prn->p, re, nexthop);
+                       ret = zebra_nht_route_map_check(
+                               rmap_family, proto, &prn->p, zvrf, re, nexthop);
                        if (ret != RMAP_DENYMATCH) {
                                SET_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE);
                                at_least_one++; /* at least one valid NH */
@@ -375,7 +331,7 @@ static int zebra_rnh_apply_nht_rmap(int family, struct route_node *prn,
  * for BGP route for import.
  */
 static struct route_entry *
-zebra_rnh_resolve_import_entry(vrf_id_t vrfid, int family,
+zebra_rnh_resolve_import_entry(struct zebra_vrf *zvrf, int family,
                               struct route_node *nrn, struct rnh *rnh,
                               struct route_node **prn)
 {
@@ -385,7 +341,7 @@ zebra_rnh_resolve_import_entry(vrf_id_t vrfid, int family,
 
        *prn = NULL;
 
-       route_table = zebra_vrf_table(family2afi(family), SAFI_UNICAST, vrfid);
+       route_table = zvrf->table[family2afi(family)][SAFI_UNICAST];
        if (!route_table) // unexpected
                return NULL;
 
@@ -457,11 +413,9 @@ static void zebra_rnh_eval_import_check_entry(vrf_id_t vrfid, int family,
 /*
  * Notify clients registered for this nexthop about a change.
  */
-static void zebra_rnh_notify_protocol_clients(vrf_id_t vrfid, int family,
-                                             struct route_node *nrn,
-                                             struct rnh *rnh,
-                                             struct route_node *prn,
-                                             struct route_entry *re)
+static void zebra_rnh_notify_protocol_clients(
+       struct zebra_vrf *zvrf, int family, struct route_node *nrn,
+       struct rnh *rnh, struct route_node *prn, struct route_entry *re)
 {
        struct listnode *node;
        struct zserv *client;
@@ -473,11 +427,11 @@ static void zebra_rnh_notify_protocol_clients(vrf_id_t vrfid, int family,
                prefix2str(&nrn->p, bufn, INET6_ADDRSTRLEN);
                if (prn && re) {
                        prefix2str(&prn->p, bufp, INET6_ADDRSTRLEN);
-                       zlog_debug("%u:%s: NH resolved over route %s", vrfid,
-                                  bufn, bufp);
+                       zlog_debug("%u:%s: NH resolved over route %s",
+                                  zvrf->vrf->vrf_id, bufn, bufp);
                } else
-                       zlog_debug("%u:%s: NH has become unresolved", vrfid,
-                                  bufn);
+                       zlog_debug("%u:%s: NH has become unresolved",
+                                  zvrf->vrf->vrf_id, bufn);
        }
 
        for (ALL_LIST_ELEMENTS_RO(rnh->client_list, node, client)) {
@@ -487,7 +441,7 @@ static void zebra_rnh_notify_protocol_clients(vrf_id_t vrfid, int family,
                         * nexthop to see if it is filtered or not.
                         */
                        num_resolving_nh = zebra_rnh_apply_nht_rmap(
-                               family, prn, re, client->proto);
+                               family, zvrf, prn, re, client->proto);
                        if (num_resolving_nh)
                                rnh->filtered[client->proto] = 0;
                        else
@@ -496,7 +450,7 @@ static void zebra_rnh_notify_protocol_clients(vrf_id_t vrfid, int family,
                        if (IS_ZEBRA_DEBUG_NHT)
                                zlog_debug(
                                        "%u:%s: Notifying client %s about NH %s",
-                                       vrfid, bufn,
+                                       zvrf->vrf->vrf_id, bufn,
                                        zebra_route_string(client->proto),
                                        num_resolving_nh
                                                ? ""
@@ -506,139 +460,95 @@ static void zebra_rnh_notify_protocol_clients(vrf_id_t vrfid, int family,
                        if (IS_ZEBRA_DEBUG_NHT)
                                zlog_debug(
                                        "%u:%s: Notifying client %s about NH (unreachable)",
-                                       vrfid, bufn,
+                                       zvrf->vrf->vrf_id, bufn,
                                        zebra_route_string(client->proto));
                }
 
-               send_client(rnh, client, RNH_NEXTHOP_TYPE, vrfid);
+               send_client(rnh, client, RNH_NEXTHOP_TYPE, zvrf->vrf->vrf_id);
        }
 }
 
-static void zebra_rnh_process_static_routes(vrf_id_t vrfid, int family,
-                                           struct route_node *nrn,
-                                           struct rnh *rnh,
-                                           struct route_node *prn,
-                                           struct route_entry *re)
+static void zebra_rnh_process_pbr_tables(int family,
+                                        struct route_node *nrn,
+                                        struct rnh *rnh,
+                                        struct route_node *prn,
+                                        struct route_entry *re)
 {
+       struct zebra_router_table *zrt;
+       struct route_entry *o_re;
+       struct route_node *o_rn;
        struct listnode *node;
-       int num_resolving_nh = 0;
-       struct route_node *static_rn;
-       struct route_entry *sre;
-       struct nexthop *nexthop;
-       char bufn[INET6_ADDRSTRLEN];
-       char bufp[INET6_ADDRSTRLEN];
-       char bufs[INET6_ADDRSTRLEN];
+       struct zserv *client;
+       afi_t afi = AFI_IP;
 
-       if (IS_ZEBRA_DEBUG_NHT) {
-               prefix2str(&nrn->p, bufn, INET6_ADDRSTRLEN);
-               if (prn)
-                       prefix2str(&prn->p, bufp, INET6_ADDRSTRLEN);
+       if (family == AF_INET6)
+               afi = AFI_IP6;
+
+       /*
+        * We are only concerned about nexthops that change for
+        * anyone using PBR
+        */
+       for (ALL_LIST_ELEMENTS_RO(rnh->client_list, node, client)) {
+               if (client->proto == ZEBRA_ROUTE_PBR)
+                       break;
        }
 
-       if (prn && re) {
-               /* Apply route-map for "static" to route resolving this
-                * nexthop to see if it is filtered or not.
-                */
-               num_resolving_nh = zebra_rnh_apply_nht_rmap(family, prn, re,
-                                                           ZEBRA_ROUTE_STATIC);
-               if (num_resolving_nh)
-                       rnh->filtered[ZEBRA_ROUTE_STATIC] = 0;
-               else
-                       rnh->filtered[ZEBRA_ROUTE_STATIC] = 1;
-       } else
-               rnh->filtered[ZEBRA_ROUTE_STATIC] = 0;
+       if (!client)
+               return;
 
-       /* Evaluate each static route associated with this nexthop. */
-       for (ALL_LIST_ELEMENTS_RO(rnh->zebra_static_route_list, node,
-                                 static_rn)) {
-               RNODE_FOREACH_RE (static_rn, sre) {
-                       if (sre->type != ZEBRA_ROUTE_STATIC)
-                               continue;
+       RB_FOREACH (zrt, zebra_router_table_head, &zrouter.tables) {
+               if (afi != zrt->afi)
+                       continue;
 
-                       /* Set the filter flag for the correct nexthop - static
-                        * route may
-                        * be having multiple. We care here only about
-                        * registered nexthops.
-                        */
-                       for (nexthop = sre->ng.nexthop; nexthop;
-                            nexthop = nexthop->next) {
-                               switch (nexthop->type) {
-                               case NEXTHOP_TYPE_IPV4:
-                               case NEXTHOP_TYPE_IPV4_IFINDEX:
-                                       if (nexthop->gate.ipv4.s_addr
-                                           == nrn->p.u.prefix4.s_addr) {
-                                               if (num_resolving_nh)
-                                                       UNSET_FLAG(
-                                                               nexthop->flags,
-                                                               NEXTHOP_FLAG_FILTERED);
-                                               else
-                                                       SET_FLAG(
-                                                               nexthop->flags,
-                                                               NEXTHOP_FLAG_FILTERED);
-                                       }
+               for (o_rn = route_top(zrt->table); o_rn;
+                    o_rn = srcdest_route_next(o_rn)) {
+                       RNODE_FOREACH_RE (o_rn, o_re) {
+                               if (o_re->type == ZEBRA_ROUTE_PBR)
                                        break;
-                               case NEXTHOP_TYPE_IPV6:
-                               case NEXTHOP_TYPE_IPV6_IFINDEX:
 
-                                       if (memcmp(&nexthop->gate.ipv6,
-                                                  &nrn->p.u.prefix6, 16)
-                                           == 0) {
-                                               if (num_resolving_nh)
-                                                       UNSET_FLAG(
-                                                               nexthop->flags,
-                                                               NEXTHOP_FLAG_FILTERED);
-                                               else
-                                                       SET_FLAG(
-                                                               nexthop->flags,
-                                                               NEXTHOP_FLAG_FILTERED);
-                                       }
-                                       break;
-                               default:
-                                       break;
-                               }
                        }
 
-                       if (IS_ZEBRA_DEBUG_NHT) {
-                               prefix2str(&static_rn->p, bufs,
-                                          INET6_ADDRSTRLEN);
-                               if (prn && re)
-                                       zlog_debug(
-                                               "%u:%s: NH change %s, scheduling static route %s",
-                                               vrfid, bufn,
-                                               num_resolving_nh
-                                                       ? ""
-                                                       : "(filtered by route-map)",
-                                               bufs);
-                               else
-                                       zlog_debug(
-                                               "%u:%s: NH unreachable, scheduling static route %s",
-                                               vrfid, bufn, bufs);
+                       /*
+                        * If we have a PBR route and a nexthop changes
+                        * just rethink it.  Yes this is a hammer, but
+                        * a small one
+                        */
+                       if (o_re) {
+                               SET_FLAG(o_re->status, ROUTE_ENTRY_CHANGED);
+                               rib_queue_add(o_rn);
                        }
-
-                       SET_FLAG(sre->status, ROUTE_ENTRY_CHANGED);
-                       SET_FLAG(sre->status, ROUTE_ENTRY_NEXTHOPS_CHANGED);
                }
-
-               rib_queue_add(static_rn);
        }
 }
 
+/*
+ * Utility to determine whether a candidate nexthop is useable. We make this
+ * check in a couple of places, so this is a single home for the logic we
+ * use.
+ */
+static bool rnh_nexthop_valid(const struct nexthop *nh)
+{
+       return (CHECK_FLAG(nh->flags, NEXTHOP_FLAG_FIB)
+               && CHECK_FLAG(nh->flags, NEXTHOP_FLAG_ACTIVE));
+}
+
 /*
  * Determine appropriate route (route entry) resolving a tracked
  * nexthop.
  */
 static struct route_entry *
-zebra_rnh_resolve_nexthop_entry(vrf_id_t vrfid, int family,
+zebra_rnh_resolve_nexthop_entry(struct zebra_vrf *zvrf, int family,
                                struct route_node *nrn, struct rnh *rnh,
                                struct route_node **prn)
 {
        struct route_table *route_table;
        struct route_node *rn;
        struct route_entry *re;
+       struct nexthop *nexthop;
 
        *prn = NULL;
 
-       route_table = zebra_vrf_table(family2afi(family), SAFI_UNICAST, vrfid);
+       route_table = zvrf->table[family2afi(family)][SAFI_UNICAST];
        if (!route_table)
                return NULL;
 
@@ -667,12 +577,22 @@ zebra_rnh_resolve_nexthop_entry(vrf_id_t vrfid, int family,
                        if (!CHECK_FLAG(re->flags, ZEBRA_FLAG_SELECTED))
                                continue;
 
+                       /* Just being SELECTED isn't quite enough - must
+                        * have an installed nexthop to be useful.
+                        */
+                       for (ALL_NEXTHOPS(re->ng, nexthop)) {
+                               if (rnh_nexthop_valid(nexthop))
+                                       break;
+                       }
+
+                       if (nexthop == NULL)
+                               continue;
+
                        if (CHECK_FLAG(rnh->flags, ZEBRA_NHT_CONNECTED)) {
                                if ((re->type == ZEBRA_ROUTE_CONNECT)
                                    || (re->type == ZEBRA_ROUTE_STATIC))
                                        break;
                                if (re->type == ZEBRA_ROUTE_NHRP) {
-                                       struct nexthop *nexthop;
 
                                        for (nexthop = re->ng.nexthop; nexthop;
                                             nexthop = nexthop->next)
@@ -715,8 +635,8 @@ static void zebra_rnh_process_pseudowires(vrf_id_t vrfid, struct rnh *rnh)
  * take appropriate action; this involves notifying any clients and/or
  * scheduling dependent static routes for processing.
  */
-static void zebra_rnh_eval_nexthop_entry(vrf_id_t vrfid, int family, int force,
-                                        struct route_node *nrn,
+static void zebra_rnh_eval_nexthop_entry(struct zebra_vrf *zvrf, int family,
+                                        int force, struct route_node *nrn,
                                         struct rnh *rnh,
                                         struct route_node *prn,
                                         struct route_entry *re)
@@ -745,21 +665,20 @@ static void zebra_rnh_eval_nexthop_entry(vrf_id_t vrfid, int family, int force,
                 * rnh->state.
                 */
                /* Notify registered protocol clients. */
-               zebra_rnh_notify_protocol_clients(vrfid, family, nrn, rnh, prn,
+               zebra_rnh_notify_protocol_clients(zvrf, family, nrn, rnh, prn,
                                                  rnh->state);
 
-               /* Process static routes attached to this nexthop */
-               zebra_rnh_process_static_routes(vrfid, family, nrn, rnh, prn,
-                                               rnh->state);
+               zebra_rnh_process_pbr_tables(family, nrn, rnh, prn, rnh->state);
 
                /* Process pseudowires attached to this nexthop */
-               zebra_rnh_process_pseudowires(vrfid, rnh);
+               zebra_rnh_process_pseudowires(zvrf->vrf->vrf_id, rnh);
        }
 }
 
 /* Evaluate one tracked entry */
-static void zebra_rnh_evaluate_entry(vrf_id_t vrfid, int family, int force,
-                                    rnh_type_t type, struct route_node *nrn)
+static void zebra_rnh_evaluate_entry(struct zebra_vrf *zvrf, int family,
+                                    int force, rnh_type_t type,
+                                    struct route_node *nrn)
 {
        struct rnh *rnh;
        struct route_entry *re;
@@ -768,18 +687,18 @@ static void zebra_rnh_evaluate_entry(vrf_id_t vrfid, int family, int force,
 
        if (IS_ZEBRA_DEBUG_NHT) {
                prefix2str(&nrn->p, bufn, INET6_ADDRSTRLEN);
-               zlog_debug("%u:%s: Evaluate RNH, type %d %s", vrfid, bufn, type,
-                          force ? "(force)" : "");
+               zlog_debug("%u:%s: Evaluate RNH, type %d %s", zvrf->vrf->vrf_id,
+                          bufn, type, force ? "(force)" : "");
        }
 
        rnh = nrn->info;
 
        /* Identify route entry (RE) resolving this tracked entry. */
        if (type == RNH_IMPORT_CHECK_TYPE)
-               re = zebra_rnh_resolve_import_entry(vrfid, family, nrn, rnh,
+               re = zebra_rnh_resolve_import_entry(zvrf, family, nrn, rnh,
                                                    &prn);
        else
-               re = zebra_rnh_resolve_nexthop_entry(vrfid, family, nrn, rnh,
+               re = zebra_rnh_resolve_nexthop_entry(zvrf, family, nrn, rnh,
                                                     &prn);
 
        /* If the entry cannot be resolved and that is also the existing state,
@@ -790,11 +709,11 @@ static void zebra_rnh_evaluate_entry(vrf_id_t vrfid, int family, int force,
 
        /* Process based on type of entry. */
        if (type == RNH_IMPORT_CHECK_TYPE)
-               zebra_rnh_eval_import_check_entry(vrfid, family, force, nrn,
-                                                 rnh, re);
+               zebra_rnh_eval_import_check_entry(zvrf->vrf->vrf_id, family,
+                                                 force, nrn, rnh, re);
        else
-               zebra_rnh_eval_nexthop_entry(vrfid, family, force, nrn, rnh,
-                                            prn, re);
+               zebra_rnh_eval_nexthop_entry(zvrf, family, force, nrn, rnh, prn,
+                                            re);
 }
 
 /*
@@ -806,7 +725,7 @@ static void zebra_rnh_evaluate_entry(vrf_id_t vrfid, int family, int force,
  * we can have a situation where one re entry
  * covers multiple nexthops we are interested in.
  */
-static void zebra_rnh_clear_nhc_flag(vrf_id_t vrfid, int family,
+static void zebra_rnh_clear_nhc_flag(struct zebra_vrf *zvrf, int family,
                                     rnh_type_t type, struct route_node *nrn)
 {
        struct rnh *rnh;
@@ -817,10 +736,10 @@ static void zebra_rnh_clear_nhc_flag(vrf_id_t vrfid, int family,
 
        /* Identify route entry (RIB) resolving this tracked entry. */
        if (type == RNH_IMPORT_CHECK_TYPE)
-               re = zebra_rnh_resolve_import_entry(vrfid, family, nrn, rnh,
+               re = zebra_rnh_resolve_import_entry(zvrf, family, nrn, rnh,
                                                    &prn);
        else
-               re = zebra_rnh_resolve_nexthop_entry(vrfid, family, nrn, rnh,
+               re = zebra_rnh_resolve_nexthop_entry(zvrf, family, nrn, rnh,
                                                     &prn);
 
        if (re) {
@@ -832,13 +751,13 @@ static void zebra_rnh_clear_nhc_flag(vrf_id_t vrfid, int family,
 /* Evaluate all tracked entries (nexthops or routes for import into BGP)
  * of a particular VRF and address-family or a specific prefix.
  */
-void zebra_evaluate_rnh(vrf_id_t vrfid, int family, int force, rnh_type_t type,
-                       struct prefix *p)
+void zebra_evaluate_rnh(struct zebra_vrf *zvrf, int family, int force,
+                       rnh_type_t type, struct prefix *p)
 {
        struct route_table *rnh_table;
        struct route_node *nrn;
 
-       rnh_table = get_rnh_table(vrfid, family, type);
+       rnh_table = get_rnh_table(zvrf->vrf->vrf_id, family, type);
        if (!rnh_table) // unexpected
                return;
 
@@ -846,7 +765,7 @@ void zebra_evaluate_rnh(vrf_id_t vrfid, int family, int force, rnh_type_t type,
                /* Evaluating a specific entry, make sure it exists. */
                nrn = route_node_lookup(rnh_table, p);
                if (nrn && nrn->info)
-                       zebra_rnh_evaluate_entry(vrfid, family, force, type,
+                       zebra_rnh_evaluate_entry(zvrf, family, force, type,
                                                 nrn);
 
                if (nrn)
@@ -856,14 +775,14 @@ void zebra_evaluate_rnh(vrf_id_t vrfid, int family, int force, rnh_type_t type,
                nrn = route_top(rnh_table);
                while (nrn) {
                        if (nrn->info)
-                               zebra_rnh_evaluate_entry(vrfid, family, force,
+                               zebra_rnh_evaluate_entry(zvrf, family, force,
                                                         type, nrn);
                        nrn = route_next(nrn); /* this will also unlock nrn */
                }
                nrn = route_top(rnh_table);
                while (nrn) {
                        if (nrn->info)
-                               zebra_rnh_clear_nhc_flag(vrfid, family, type,
+                               zebra_rnh_clear_nhc_flag(zvrf, family, type,
                                                         nrn);
                        nrn = route_next(nrn); /* this will also unlock nrn */
                }
@@ -887,34 +806,6 @@ void zebra_print_rnh_table(vrf_id_t vrfid, int af, struct vty *vty,
                        print_rnh(rn, vty);
 }
 
-int zebra_cleanup_rnh_client(vrf_id_t vrf_id, int family, struct zserv *client,
-                            rnh_type_t type)
-{
-       struct route_table *ntable;
-       struct route_node *nrn;
-       struct rnh *rnh;
-
-       if (IS_ZEBRA_DEBUG_NHT)
-               zlog_debug("%u: Client %s RNH cleanup for family %d type %d",
-                          vrf_id, zebra_route_string(client->proto), family,
-                          type);
-
-       ntable = get_rnh_table(vrf_id, family, type);
-       if (!ntable) {
-               zlog_debug("cleanup_rnh_client: rnh table not found\n");
-               return -1;
-       }
-
-       for (nrn = route_top(ntable); nrn; nrn = route_next(nrn)) {
-               if (!nrn->info)
-                       continue;
-
-               rnh = nrn->info;
-               zebra_remove_rnh_client(rnh, client, type);
-       }
-       return 1;
-}
-
 /**
  * free_state - free up the re structure associated with the rnh.
  */
@@ -926,7 +817,6 @@ static void free_state(vrf_id_t vrf_id, struct route_entry *re,
                return;
 
        /* free RE and nexthops */
-       zebra_deregister_rnh_static_nexthops(vrf_id, re->ng.nexthop, rn);
        nexthops_free(re->ng.nexthop);
        XFREE(MTYPE_RE, re);
 }
@@ -985,8 +875,8 @@ static int send_client(struct rnh *rnh, struct zserv *client, rnh_type_t type,
        struct stream *s;
        struct route_entry *re;
        unsigned long nump;
-       u_char num;
-       struct nexthop *nexthop;
+       uint8_t num;
+       struct nexthop *nh;
        struct route_node *rn;
        int cmd = (type == RNH_IMPORT_CHECK_TYPE) ? ZEBRA_IMPORT_CHECK_UPDATE
                                                  : ZEBRA_NEXTHOP_UPDATE;
@@ -1010,7 +900,8 @@ static int send_client(struct rnh *rnh, struct zserv *client, rnh_type_t type,
                stream_put(s, &rn->p.u.prefix6, IPV6_MAX_BYTELEN);
                break;
        default:
-               zlog_err("%s: Unknown family (%d) notification attempted\n",
+               flog_err(EC_ZEBRA_RNH_UNKNOWN_FAMILY,
+                        "%s: Unknown family (%d) notification attempted\n",
                         __FUNCTION__, rn->p.family);
                break;
        }
@@ -1022,32 +913,39 @@ static int send_client(struct rnh *rnh, struct zserv *client, rnh_type_t type,
                num = 0;
                nump = stream_get_endp(s);
                stream_putc(s, 0);
-               for (nexthop = re->ng.nexthop; nexthop; nexthop = nexthop->next)
-                       if ((CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB)
-                            || CHECK_FLAG(nexthop->flags,
-                                          NEXTHOP_FLAG_RECURSIVE))
-                           && CHECK_FLAG(nexthop->flags,
-                                         NEXTHOP_FLAG_ACTIVE)) {
-                               stream_putc(s, nexthop->type);
-                               switch (nexthop->type) {
+               for (ALL_NEXTHOPS(re->ng, nh))
+                       if (rnh_nexthop_valid(nh)) {
+                               stream_putl(s, nh->vrf_id);
+                               stream_putc(s, nh->type);
+                               switch (nh->type) {
                                case NEXTHOP_TYPE_IPV4:
                                case NEXTHOP_TYPE_IPV4_IFINDEX:
-                                       stream_put_in_addr(s,
-                                                          &nexthop->gate.ipv4);
-                                       stream_putl(s, nexthop->ifindex);
+                                       stream_put_in_addr(s, &nh->gate.ipv4);
+                                       stream_putl(s, nh->ifindex);
                                        break;
                                case NEXTHOP_TYPE_IFINDEX:
-                                       stream_putl(s, nexthop->ifindex);
+                                       stream_putl(s, nh->ifindex);
                                        break;
                                case NEXTHOP_TYPE_IPV6:
                                case NEXTHOP_TYPE_IPV6_IFINDEX:
-                                       stream_put(s, &nexthop->gate.ipv6, 16);
-                                       stream_putl(s, nexthop->ifindex);
+                                       stream_put(s, &nh->gate.ipv6, 16);
+                                       stream_putl(s, nh->ifindex);
                                        break;
                                default:
                                        /* do nothing */
                                        break;
                                }
+                               if (nh->nh_label) {
+                                       stream_putc(s,
+                                                   nh->nh_label->num_labels);
+                                       if (nh->nh_label->num_labels)
+                                               stream_put(
+                                                       s,
+                                                       &nh->nh_label->label[0],
+                                                       nh->nh_label->num_labels
+                                                               * sizeof(mpls_label_t));
+                               } else
+                                       stream_putc(s, 0);
                                num++;
                        }
                stream_putc_at(s, nump, num);
@@ -1062,7 +960,7 @@ static int send_client(struct rnh *rnh, struct zserv *client, rnh_type_t type,
 
        client->nh_last_upd_time = monotime(NULL);
        client->last_write_cmd = cmd;
-       return zebra_server_send_message(client, s);
+       return zserv_send_message(client, s);
 }
 
 static void print_nh(struct nexthop *nexthop, struct vty *vty)
@@ -1129,10 +1027,65 @@ static void print_rnh(struct route_node *rn, struct vty *vty)
                vty_out(vty, " %s(fd %d)%s", zebra_route_string(client->proto),
                        client->sock,
                        rnh->filtered[client->proto] ? "(filtered)" : "");
-       if (!list_isempty(rnh->zebra_static_route_list))
-               vty_out(vty, " zebra%s",
-                       rnh->filtered[ZEBRA_ROUTE_STATIC] ? "(filtered)" : "");
        if (!list_isempty(rnh->zebra_pseudowire_list))
                vty_out(vty, " zebra[pseudowires]");
        vty_out(vty, "\n");
 }
+
+static int zebra_cleanup_rnh_client(vrf_id_t vrf_id, int family,
+                                   struct zserv *client, rnh_type_t type)
+{
+       struct route_table *ntable;
+       struct route_node *nrn;
+       struct rnh *rnh;
+
+       if (IS_ZEBRA_DEBUG_NHT)
+               zlog_debug("%u: Client %s RNH cleanup for family %d type %d",
+                          vrf_id, zebra_route_string(client->proto), family,
+                          type);
+
+       ntable = get_rnh_table(vrf_id, family, type);
+       if (!ntable) {
+               zlog_debug("cleanup_rnh_client: rnh table not found\n");
+               return -1;
+       }
+
+       for (nrn = route_top(ntable); nrn; nrn = route_next(nrn)) {
+               if (!nrn->info)
+                       continue;
+
+               rnh = nrn->info;
+               zebra_remove_rnh_client(rnh, client, type);
+       }
+       return 1;
+}
+
+/* Cleanup registered nexthops (across VRFs) upon client disconnect. */
+static int zebra_client_cleanup_rnh(struct zserv *client)
+{
+       struct vrf *vrf;
+       struct zebra_vrf *zvrf;
+
+       RB_FOREACH (vrf, vrf_id_head, &vrfs_by_id) {
+               zvrf = vrf->info;
+               if (zvrf) {
+                       zebra_cleanup_rnh_client(zvrf_id(zvrf), AF_INET, client,
+                                                RNH_NEXTHOP_TYPE);
+                       zebra_cleanup_rnh_client(zvrf_id(zvrf), AF_INET6,
+                                                client, RNH_NEXTHOP_TYPE);
+                       zebra_cleanup_rnh_client(zvrf_id(zvrf), AF_INET, client,
+                                                RNH_IMPORT_CHECK_TYPE);
+                       zebra_cleanup_rnh_client(zvrf_id(zvrf), AF_INET6,
+                                                client, RNH_IMPORT_CHECK_TYPE);
+                       if (client->proto == ZEBRA_ROUTE_LDP) {
+                               hash_iterate(zvrf->lsp_table,
+                                            mpls_ldp_lsp_uninstall_all,
+                                            zvrf->lsp_table);
+                               mpls_ldp_ftn_uninstall_all(zvrf, AFI_IP);
+                               mpls_ldp_ftn_uninstall_all(zvrf, AFI_IP6);
+                       }
+               }
+       }
+
+       return 0;
+}