]> git.proxmox.com Git - mirror_frr.git/blobdiff - zebra/zebra_rib.c
Merge pull request #4622 from donaldsharp/import_table_fix
[mirror_frr.git] / zebra / zebra_rib.c
index 1878ac39c3dd7fff7ef0426ae7afe750bf6f76b2..85bca53d746225a41aaffefc3c79ebbe969aa97e 100644 (file)
@@ -108,10 +108,6 @@ static const struct {
        /* no entry/default: 150 */
 };
 
-/* RPF lookup behaviour */
-static enum multicast_mode ipv4_multicast_mode = MCAST_NO_CONFIG;
-
-
 static void __attribute__((format(printf, 5, 6)))
 _rnode_zlog(const char *_func, vrf_id_t vrf_id, struct route_node *rn,
            int priority, const char *msgfmt, ...)
@@ -129,7 +125,7 @@ _rnode_zlog(const char *_func, vrf_id_t vrf_id, struct route_node *rn,
                srcdest_rnode2str(rn, buf, sizeof(buf));
 
                if (info->safi == SAFI_MULTICAST)
-                       strcat(buf, " (MRIB)");
+                       strlcat(buf, " (MRIB)", sizeof(buf));
        } else {
                snprintf(buf, sizeof(buf), "{(route_node *) NULL}");
        }
@@ -404,7 +400,7 @@ struct route_entry *rib_match_ipv4_multicast(vrf_id_t vrf_id,
        struct route_node *m_rn = NULL, *u_rn = NULL;
        union g_addr gaddr = {.ipv4 = addr};
 
-       switch (ipv4_multicast_mode) {
+       switch (zrouter.ipv4_multicast_mode) {
        case MCAST_MRIB_ONLY:
                return rib_match(AFI_IP, SAFI_MULTICAST, vrf_id, &gaddr,
                                 rn_out);
@@ -456,19 +452,6 @@ struct route_entry *rib_match_ipv4_multicast(vrf_id_t vrf_id,
        return re;
 }
 
-void multicast_mode_ipv4_set(enum multicast_mode mode)
-{
-       if (IS_ZEBRA_DEBUG_RIB)
-               zlog_debug("%s: multicast lookup mode set (%d)", __func__,
-                          mode);
-       ipv4_multicast_mode = mode;
-}
-
-enum multicast_mode multicast_mode_ipv4_get(void)
-{
-       return ipv4_multicast_mode;
-}
-
 struct route_entry *rib_lookup_ipv4(struct prefix_ipv4 *p, vrf_id_t vrf_id)
 {
        struct route_table *table;
@@ -587,8 +570,25 @@ void rib_install_kernel(struct route_node *rn, struct route_entry *re,
        switch (ret) {
        case ZEBRA_DPLANE_REQUEST_QUEUED:
                SET_FLAG(re->status, ROUTE_ENTRY_QUEUED);
-               if (old)
+
+               if (old) {
                        SET_FLAG(old->status, ROUTE_ENTRY_QUEUED);
+
+                       /* Free old FIB nexthop group */
+                       if (old->fib_ng.nexthop) {
+                               nexthops_free(old->fib_ng.nexthop);
+                               old->fib_ng.nexthop = NULL;
+                       }
+
+                       if (!RIB_SYSTEM_ROUTE(old)) {
+                               /* Clear old route's FIB flags */
+                               for (ALL_NEXTHOPS(old->ng, nexthop)) {
+                                       UNSET_FLAG(nexthop->flags,
+                                                  NEXTHOP_FLAG_FIB);
+                               }
+                       }
+               }
+
                if (zvrf)
                        zvrf->installs_queued++;
                break;
@@ -674,6 +674,12 @@ static void rib_uninstall(struct route_node *rn, struct route_entry *re)
 
                dest->selected_fib = NULL;
 
+               /* Free FIB nexthop group, if present */
+               if (re->fib_ng.nexthop) {
+                       nexthops_free(re->fib_ng.nexthop);
+                       re->fib_ng.nexthop = NULL;
+               }
+
                for (ALL_NEXTHOPS(re->ng, nexthop))
                        UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
        }
@@ -691,7 +697,7 @@ static void rib_uninstall(struct route_node *rn, struct route_entry *re)
 /*
  * rib_can_delete_dest
  *
- * Returns TRUE if the given dest can be deleted from the table.
+ * Returns true if the given dest can be deleted from the table.
  */
 static int rib_can_delete_dest(rib_dest_t *dest)
 {
@@ -738,9 +744,10 @@ void zebra_rib_evaluate_rn_nexthops(struct route_node *rn, uint32_t seq)
                if (IS_ZEBRA_DEBUG_NHT_DETAILED) {
                        char buf[PREFIX_STRLEN];
 
-                       zlog_debug("%s: %s Being examined for Nexthop Tracking",
+                       zlog_debug("%s: %s Being examined for Nexthop Tracking Count: %zd",
                                   __PRETTY_FUNCTION__,
-                                  srcdest_rnode2str(rn, buf, sizeof(buf)));
+                                  srcdest_rnode2str(rn, buf, sizeof(buf)),
+                                  dest ? rnh_list_count(&dest->nht) : 0);
                }
                if (!dest) {
                        rn = rn->parent;
@@ -754,7 +761,7 @@ void zebra_rib_evaluate_rn_nexthops(struct route_node *rn, uint32_t seq)
                 * nht resolution and as such we need to call the
                 * nexthop tracking evaluation code
                 */
-               frr_each (rnh_list, &dest->nht, rnh) {
+               frr_each_safe(rnh_list, &dest->nht, rnh) {
                        struct zebra_vrf *zvrf =
                                zebra_vrf_lookup_by_id(rnh->vrf_id);
                        struct prefix *p = &rnh->node->p;
@@ -763,11 +770,12 @@ void zebra_rib_evaluate_rn_nexthops(struct route_node *rn, uint32_t seq)
                                char buf1[PREFIX_STRLEN];
                                char buf2[PREFIX_STRLEN];
 
-                               zlog_debug("%u:%s has Nexthop(%s) depending on it, evaluating %u:%u",
+                               zlog_debug("%u:%s has Nexthop(%s) Type: %s depending on it, evaluating %u:%u",
                                           zvrf->vrf->vrf_id,
                                           srcdest_rnode2str(rn, buf1,
                                                      sizeof(buf1)),
                                           prefix2str(p, buf2, sizeof(buf2)),
+                                          rnh_type2str(rnh->type),
                                           seq, rnh->seqno);
                        }
 
@@ -807,7 +815,7 @@ void zebra_rib_evaluate_rn_nexthops(struct route_node *rn, uint32_t seq)
  * Garbage collect the rib dest corresponding to the given route node
  * if appropriate.
  *
- * Returns TRUE if the dest was deleted, FALSE otherwise.
+ * Returns true if the dest was deleted, false otherwise.
  */
 int rib_gc_dest(struct route_node *rn)
 {
@@ -1366,21 +1374,239 @@ static void zebra_rib_fixup_system(struct route_node *rn)
 }
 
 /*
- * Route-update results processing after async dataplane update.
+ * Update a route from a dplane context. This consolidates common code
+ * that can be used in processing of results from FIB updates, and in
+ * async notification processing.
+ * The return is 'true' if the installed nexthops changed; 'false' otherwise.
  */
-static void rib_process_result(struct zebra_dplane_ctx *ctx)
+static bool rib_update_re_from_ctx(struct route_entry *re,
+                                  struct route_node *rn,
+                                  struct zebra_dplane_ctx *ctx)
+{
+       char dest_str[PREFIX_STRLEN] = "";
+       char nh_str[NEXTHOP_STRLEN];
+       struct nexthop *nexthop, *ctx_nexthop;
+       bool matched;
+       const struct nexthop_group *ctxnhg;
+       bool is_selected = false; /* Is 're' currently the selected re? */
+       bool changed_p = false; /* Change to nexthops? */
+       rib_dest_t *dest;
+
+       /* Note well: only capturing the prefix string if debug is enabled here;
+        * unconditional log messages will have to generate the string.
+        */
+       if (IS_ZEBRA_DEBUG_RIB)
+               prefix2str(&(rn->p), dest_str, sizeof(dest_str));
+
+       dest = rib_dest_from_rnode(rn);
+       if (dest)
+               is_selected = (re == dest->selected_fib);
+
+       if (IS_ZEBRA_DEBUG_RIB_DETAILED)
+               zlog_debug("update_from_ctx: %u:%s: %sSELECTED",
+                          re->vrf_id, dest_str, (is_selected ? "" : "NOT "));
+
+       /* Update zebra's nexthop FIB flag for each nexthop that was installed.
+        * If the installed set differs from the set requested by the rib/owner,
+        * we use the fib-specific nexthop-group to record the actual FIB
+        * status.
+        */
+
+       /*
+        * First check the fib nexthop-group, if it's present. The comparison
+        * here is quite strict: we require that the fib sets match exactly.
+        */
+       matched = false;
+       do {
+               if (re->fib_ng.nexthop == NULL)
+                       break;
+
+               matched = true;
+
+               /* First check the route's fib nexthops */
+               for (ALL_NEXTHOPS(re->fib_ng, nexthop)) {
+
+                       if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_RECURSIVE))
+                               continue;
+
+                       ctx_nexthop = NULL;
+                       for (ALL_NEXTHOPS_PTR(dplane_ctx_get_ng(ctx),
+                                             ctx_nexthop)) {
+                               if (nexthop_same(ctx_nexthop, nexthop))
+                                       break;
+                       }
+
+                       if (ctx_nexthop == NULL) {
+                               /* Nexthop not in the new installed set */
+                               if (IS_ZEBRA_DEBUG_RIB_DETAILED) {
+                                       nexthop2str(nexthop, nh_str,
+                                                   sizeof(nh_str));
+                                       zlog_debug("update_from_ctx: no match for fib nh %s",
+                                                  nh_str);
+                               }
+
+                               matched = false;
+                               break;
+                       }
+               }
+
+               if (!matched)
+                       break;
+
+               /* Check the new installed set */
+               ctx_nexthop = NULL;
+               for (ALL_NEXTHOPS_PTR(dplane_ctx_get_ng(ctx), ctx_nexthop)) {
+
+                       if (CHECK_FLAG(ctx_nexthop->flags,
+                                      NEXTHOP_FLAG_RECURSIVE))
+                               continue;
+
+                       /* Compare with the current group's nexthops */
+                       nexthop = NULL;
+                       for (ALL_NEXTHOPS(re->fib_ng, nexthop)) {
+                               if (nexthop_same(nexthop, ctx_nexthop))
+                                       break;
+                       }
+
+                       if (nexthop == NULL) {
+                               /* Nexthop not in the old installed set */
+                               if (IS_ZEBRA_DEBUG_RIB_DETAILED) {
+                                       nexthop2str(ctx_nexthop, nh_str,
+                                                   sizeof(nh_str));
+                                       zlog_debug("update_from_ctx: no fib match for notif nh %s",
+                                                  nh_str);
+                               }
+                               matched = false;
+                               break;
+                       }
+               }
+
+       } while (0);
+
+       /* If the new FIB set matches the existing FIB set, we're done. */
+       if (matched) {
+               if (IS_ZEBRA_DEBUG_RIB)
+                       zlog_debug("%u:%s update_from_ctx(): existing fib nhg, no change",
+                                  re->vrf_id, dest_str);
+               goto done;
+
+       } else if (re->fib_ng.nexthop) {
+               /*
+                * Free stale fib list and move on to check the rib nhg.
+                */
+               if (IS_ZEBRA_DEBUG_RIB)
+                       zlog_debug("%u:%s update_from_ctx(): replacing fib nhg",
+                                  re->vrf_id, dest_str);
+               nexthops_free(re->fib_ng.nexthop);
+               re->fib_ng.nexthop = NULL;
+
+               /* Note that the installed nexthops have changed */
+               changed_p = true;
+       } else {
+               if (IS_ZEBRA_DEBUG_RIB)
+                       zlog_debug("%u:%s update_from_ctx(): no fib nhg",
+                                  re->vrf_id, dest_str);
+       }
+
+       /*
+        * Compare with the rib nexthop group. The comparison here is different:
+        * the RIB group may be a superset of the list installed in the FIB. We
+        * walk the RIB group, looking for the 'installable' candidate
+        * nexthops, and then check those against the set
+        * that is actually installed.
+        */
+       matched = true;
+       for (ALL_NEXTHOPS(re->ng, nexthop)) {
+
+               if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_RECURSIVE))
+                       continue;
+
+               if (!CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE))
+                       continue;
+
+               /* Check for a FIB nexthop corresponding to the RIB nexthop */
+               ctx_nexthop = NULL;
+               for (ALL_NEXTHOPS_PTR(dplane_ctx_get_ng(ctx), ctx_nexthop)) {
+                       if (nexthop_same(ctx_nexthop, nexthop))
+                               break;
+               }
+
+               /* If the FIB doesn't know about the nexthop,
+                * it's not installed
+                */
+               if (ctx_nexthop == NULL) {
+                       if (IS_ZEBRA_DEBUG_RIB_DETAILED) {
+                               nexthop2str(nexthop, nh_str, sizeof(nh_str));
+                               zlog_debug("update_from_ctx: no notif match for rib nh %s",
+                                          nh_str);
+                       }
+                       matched = false;
+
+                       if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB))
+                               changed_p = true;
+
+                       UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
+                       break;
+               }
+
+               if (CHECK_FLAG(ctx_nexthop->flags, NEXTHOP_FLAG_FIB)) {
+                       if (!CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB))
+                               changed_p = true;
+
+                       SET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
+               } else {
+                       if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB))
+                               changed_p = true;
+
+                       UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
+               }
+       }
+
+       /* If all nexthops were processed, we're done */
+       if (matched) {
+               if (IS_ZEBRA_DEBUG_RIB)
+                       zlog_debug("%u:%s update_from_ctx(): rib nhg matched, changed '%s'",
+                                  re->vrf_id, dest_str,
+                                  (changed_p ? "true" : "false"));
+               goto done;
+       }
+
+       /* FIB nexthop set differs from the RIB set:
+        * create a fib-specific nexthop-group
+        */
+       if (IS_ZEBRA_DEBUG_RIB)
+               zlog_debug("%u:%s update_from_ctx(): changed %s, adding new fib nhg",
+                          re->vrf_id, dest_str,
+                          (changed_p ? "true" : "false"));
+
+       ctxnhg = dplane_ctx_get_ng(ctx);
+
+       if (ctxnhg->nexthop)
+               copy_nexthops(&(re->fib_ng.nexthop), ctxnhg->nexthop, NULL);
+       else {
+               /* Bit of a special case when the fib has _no_ installed
+                * nexthops.
+                */
+               nexthop = nexthop_new();
+               nexthop->type = NEXTHOP_TYPE_IPV4;
+               nexthop_add(&(re->fib_ng.nexthop), nexthop);
+       }
+
+done:
+       return changed_p;
+}
+
+/*
+ * Helper to locate a zebra route-node from a dplane context. This is used
+ * when processing dplane results, e.g. Note well: the route-node is returned
+ * with a ref held - route_unlock_node() must be called eventually.
+ */
+static struct route_node *
+rib_find_rn_from_ctx(const struct zebra_dplane_ctx *ctx)
 {
        struct route_table *table = NULL;
-       struct zebra_vrf *zvrf = NULL;
        struct route_node *rn = NULL;
-       struct route_entry *re = NULL, *old_re = NULL, *rib;
-       bool is_update = false;
-       struct nexthop *nexthop, *ctx_nexthop;
-       char dest_str[PREFIX_STRLEN] = "";
-       enum dplane_op_e op;
-       enum zebra_dplane_result status;
        const struct prefix *dest_pfx, *src_pfx;
-       uint32_t seq;
 
        /* Locate rn and re(s) from ctx */
 
@@ -1390,7 +1616,7 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
                                              dplane_ctx_get_table(ctx));
        if (table == NULL) {
                if (IS_ZEBRA_DEBUG_DPLANE) {
-                       zlog_debug("Failed to process dplane results: no table for afi %d, safi %d, vrf %u",
+                       zlog_debug("Failed to find route for ctx: no table for afi %d, safi %d, vrf %u",
                                   dplane_ctx_get_afi(ctx),
                                   dplane_ctx_get_safi(ctx),
                                   dplane_ctx_get_vrf(ctx));
@@ -1398,8 +1624,35 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
                goto done;
        }
 
-       zvrf = vrf_info_lookup(dplane_ctx_get_vrf(ctx));
+       dest_pfx = dplane_ctx_get_dest(ctx);
+       src_pfx = dplane_ctx_get_src(ctx);
+
+       rn = srcdest_rnode_get(table, dest_pfx,
+                              src_pfx ? (struct prefix_ipv6 *)src_pfx : NULL);
+
+done:
+       return rn;
+}
+
+
+
+/*
+ * Route-update results processing after async dataplane update.
+ */
+static void rib_process_result(struct zebra_dplane_ctx *ctx)
+{
+       struct zebra_vrf *zvrf = NULL;
+       struct route_node *rn = NULL;
+       struct route_entry *re = NULL, *old_re = NULL, *rib;
+       bool is_update = false;
+       char dest_str[PREFIX_STRLEN] = "";
+       enum dplane_op_e op;
+       enum zebra_dplane_result status;
+       const struct prefix *dest_pfx, *src_pfx;
+       uint32_t seq;
+       bool fib_changed = false;
 
+       zvrf = vrf_info_lookup(dplane_ctx_get_vrf(ctx));
        dest_pfx = dplane_ctx_get_dest(ctx);
 
        /* Note well: only capturing the prefix string if debug is enabled here;
@@ -1408,9 +1661,8 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
        if (IS_ZEBRA_DEBUG_DPLANE)
                prefix2str(dest_pfx, dest_str, sizeof(dest_str));
 
-       src_pfx = dplane_ctx_get_src(ctx);
-       rn = srcdest_rnode_get(table, dplane_ctx_get_dest(ctx),
-                              src_pfx ? (struct prefix_ipv6 *)src_pfx : NULL);
+       /* Locate rn and re(s) from ctx */
+       rn = rib_find_rn_from_ctx(ctx);
        if (rn == NULL) {
                if (IS_ZEBRA_DEBUG_DPLANE) {
                        zlog_debug("Failed to process dplane results: no route for %u:%s",
@@ -1504,34 +1756,25 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
                                UNSET_FLAG(old_re->status,
                                           ROUTE_ENTRY_INSTALLED);
                        }
-                       /* Update zebra nexthop FIB flag for each
-                        * nexthop that was installed.
-                        */
-                       for (ALL_NEXTHOPS_PTR(dplane_ctx_get_ng(ctx),
-                                             ctx_nexthop)) {
-
-                               if (!re)
-                                       continue;
 
-                               for (ALL_NEXTHOPS(re->ng, nexthop)) {
-                                       if (nexthop_same(ctx_nexthop, nexthop))
-                                               break;
+                       /* Update zebra route based on the results in
+                        * the context struct.
+                        */
+                       if (re) {
+                               fib_changed =
+                                       rib_update_re_from_ctx(re, rn, ctx);
+
+                               if (!fib_changed) {
+                                       if (IS_ZEBRA_DEBUG_DPLANE_DETAIL)
+                                               zlog_debug("%u:%s no fib change for re",
+                                                          dplane_ctx_get_vrf(
+                                                                  ctx),
+                                                          dest_str);
                                }
 
-                               if (nexthop == NULL)
-                                       continue;
-
-                               if (CHECK_FLAG(nexthop->flags,
-                                              NEXTHOP_FLAG_RECURSIVE))
-                                       continue;
-
-                               if (CHECK_FLAG(ctx_nexthop->flags,
-                                              NEXTHOP_FLAG_FIB))
-                                       SET_FLAG(nexthop->flags,
-                                                NEXTHOP_FLAG_FIB);
-                               else
-                                       UNSET_FLAG(nexthop->flags,
-                                                  NEXTHOP_FLAG_FIB);
+                               /* Redistribute */
+                               redistribute_update(dest_pfx, src_pfx,
+                                                   re, NULL);
                        }
 
                        /*
@@ -1548,19 +1791,6 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
                        if (zvrf)
                                zvrf->installs++;
 
-                       /* Redistribute */
-                       /*
-                        * TODO -- still calling the redist api using the
-                        * route_entries, and there's a corner-case here:
-                        * if there's no client for the 'new' route, a redist
-                        * deleting the 'old' route will be sent. But if the
-                        * 'old' context info was stale, 'old_re' will be
-                        * NULL here and that delete will not be sent.
-                        */
-                       if (re)
-                               redistribute_update(dest_pfx, src_pfx,
-                                                   re, old_re);
-
                        /* Notify route owner */
                        zsend_route_notify_owner_ctx(ctx, ZAPI_ROUTE_INSTALLED);
 
@@ -1635,6 +1865,179 @@ done:
        dplane_ctx_fini(&ctx);
 }
 
+/*
+ * Handle notification from async dataplane: the dataplane has detected
+ * some change to a route, and notifies zebra so that the control plane
+ * can reflect that change.
+ */
+static void rib_process_dplane_notify(struct zebra_dplane_ctx *ctx)
+{
+       struct route_node *rn = NULL;
+       struct route_entry *re = NULL;
+       struct nexthop *nexthop;
+       char dest_str[PREFIX_STRLEN] = "";
+       const struct prefix *dest_pfx, *src_pfx;
+       rib_dest_t *dest;
+       bool fib_changed = false;
+       bool debug_p = IS_ZEBRA_DEBUG_DPLANE | IS_ZEBRA_DEBUG_RIB;
+       int start_count, end_count;
+       dest_pfx = dplane_ctx_get_dest(ctx);
+
+       /* Note well: only capturing the prefix string if debug is enabled here;
+        * unconditional log messages will have to generate the string.
+        */
+       if (debug_p)
+               prefix2str(dest_pfx, dest_str, sizeof(dest_str));
+
+       /* Locate rn and re(s) from ctx */
+       rn = rib_find_rn_from_ctx(ctx);
+       if (rn == NULL) {
+               if (debug_p) {
+                       zlog_debug("Failed to process dplane notification: no routes for %u:%s",
+                                  dplane_ctx_get_vrf(ctx), dest_str);
+               }
+               goto done;
+       }
+
+       dest = rib_dest_from_rnode(rn);
+       srcdest_rnode_prefixes(rn, &dest_pfx, &src_pfx);
+
+       if (debug_p)
+               zlog_debug("%u:%s Processing dplane notif ctx %p",
+                          dplane_ctx_get_vrf(ctx), dest_str, ctx);
+
+       /*
+        * Take a pass through the routes, look for matches with the context
+        * info.
+        */
+       RNODE_FOREACH_RE(rn, re) {
+               if (rib_route_match_ctx(re, ctx, false /*!update*/))
+                       break;
+       }
+
+       /* No match? Nothing we can do */
+       if (re == NULL) {
+               if (debug_p)
+                       zlog_debug("%u:%s Unable to process dplane notification: no entry for type %s",
+                                  dplane_ctx_get_vrf(ctx), dest_str,
+                                  zebra_route_string(
+                                          dplane_ctx_get_type(ctx)));
+
+               goto done;
+       }
+
+       /* Is this a notification that ... matters? We only really care about
+        * the route that is currently selected for installation.
+        */
+       if (re != dest->selected_fib) {
+               /* TODO -- don't skip processing entirely? We might like to
+                * at least report on the event.
+                */
+               if (debug_p)
+                       zlog_debug("%u:%s dplane notif, but type %s not selected_fib",
+                                  dplane_ctx_get_vrf(ctx), dest_str,
+                                  zebra_route_string(
+                                          dplane_ctx_get_type(ctx)));
+               goto done;
+       }
+
+       /* We'll want to determine whether the installation status of the
+        * route has changed: we'll check the status before processing,
+        * and then again if there's been a change.
+        */
+       start_count = 0;
+       for (ALL_NEXTHOPS_PTR(rib_active_nhg(re), nexthop)) {
+               if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB))
+                       start_count++;
+       }
+
+       /* Update zebra's nexthop FIB flags based on the context struct's
+        * nexthops.
+        */
+       fib_changed = rib_update_re_from_ctx(re, rn, ctx);
+
+       if (!fib_changed) {
+               if (debug_p)
+                       zlog_debug("%u:%s No change from dplane notification",
+                                  dplane_ctx_get_vrf(ctx), dest_str);
+
+               goto done;
+       }
+
+       /*
+        * Perform follow-up work if the actual status of the prefix
+        * changed.
+        */
+
+       end_count = 0;
+       for (ALL_NEXTHOPS_PTR(rib_active_nhg(re), nexthop)) {
+               if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB))
+                       end_count++;
+       }
+
+       /* Various fib transitions: changed nexthops; from installed to
+        * not-installed; or not-installed to installed.
+        */
+       if (start_count > 0 && end_count > 0) {
+
+               /* Changed nexthops - update kernel/others */
+               dplane_route_notif_update(rn, re,
+                                         DPLANE_OP_ROUTE_UPDATE, ctx);
+
+       } else if (start_count == 0 && end_count > 0) {
+               if (debug_p)
+                       zlog_debug("%u:%s installed transition from dplane notification",
+                                  dplane_ctx_get_vrf(ctx), dest_str);
+
+               /* We expect this to be the selected route, so we want
+                * to tell others about this transistion.
+                */
+               SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
+
+               /* Changed nexthops - update kernel/others */
+               dplane_route_notif_update(rn, re, DPLANE_OP_ROUTE_INSTALL, ctx);
+
+               /* Redistribute, lsp, and nht update */
+               redistribute_update(dest_pfx, src_pfx, re, NULL);
+
+               zebra_rib_evaluate_rn_nexthops(
+                       rn, zebra_router_get_next_sequence());
+
+               zebra_rib_evaluate_mpls(rn);
+
+       } else if (start_count > 0 && end_count == 0) {
+               if (debug_p)
+                       zlog_debug("%u:%s un-installed transition from dplane notification",
+                                  dplane_ctx_get_vrf(ctx), dest_str);
+
+               /* Transition from _something_ installed to _nothing_
+                * installed.
+                */
+               /* We expect this to be the selected route, so we want
+                * to tell others about this transistion.
+                */
+               UNSET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
+
+               /* Changed nexthops - update kernel/others */
+               dplane_route_notif_update(rn, re, DPLANE_OP_ROUTE_DELETE, ctx);
+
+               /* Redistribute, lsp, and nht update */
+               redistribute_delete(dest_pfx, src_pfx, re);
+
+               zebra_rib_evaluate_rn_nexthops(
+                       rn, zebra_router_get_next_sequence());
+
+               zebra_rib_evaluate_mpls(rn);
+       }
+
+done:
+       if (rn)
+               route_unlock_node(rn);
+
+       /* Return context to dataplane module */
+       dplane_ctx_fini(&ctx);
+}
+
 /* Take a list of route_node structs and return 1, if there was a record
  * picked from it and processed by rib_process(). Don't process more,
  * than one RN record; operate only in the specified sub-queue.
@@ -1658,6 +2061,7 @@ static unsigned int process_subq(struct list *subq, uint8_t qindex)
 
        if (IS_ZEBRA_DEBUG_RIB_DETAILED) {
                char buf[SRCDEST2STR_BUFFER];
+
                srcdest_rnode2str(rnode, buf, sizeof(buf));
                zlog_debug("%u:%s: rn %p dequeued from sub-queue %u",
                           zvrf ? zvrf_id(zvrf) : 0, buf, rnode, qindex);
@@ -1942,9 +2346,11 @@ static void rib_link(struct route_node *rn, struct route_entry *re, int process)
        afi = (rn->p.family == AF_INET)
                      ? AFI_IP
                      : (rn->p.family == AF_INET6) ? AFI_IP6 : AFI_MAX;
-       if (is_zebra_import_table_enabled(afi, re->table)) {
+       if (is_zebra_import_table_enabled(afi, re->vrf_id, re->table)) {
+               struct zebra_vrf *zvrf = zebra_vrf_lookup_by_id(re->vrf_id);
+
                rmap_name = zebra_get_import_table_route_map(afi, re->table);
-               zebra_add_import_table_entry(rn, re, rmap_name);
+               zebra_add_import_table_entry(zvrf, rn, re, rmap_name);
        } else if (process)
                rib_queue_add(rn);
 }
@@ -1993,6 +2399,8 @@ void rib_unlink(struct route_node *rn, struct route_entry *re)
                dest->selected_fib = NULL;
 
        nexthops_free(re->ng.nexthop);
+       nexthops_free(re->fib_ng.nexthop);
+
        XFREE(MTYPE_RE, re);
 }
 
@@ -2008,8 +2416,10 @@ void rib_delnode(struct route_node *rn, struct route_entry *re)
        afi = (rn->p.family == AF_INET)
                      ? AFI_IP
                      : (rn->p.family == AF_INET6) ? AFI_IP6 : AFI_MAX;
-       if (is_zebra_import_table_enabled(afi, re->table)) {
-               zebra_del_import_table_entry(rn, re);
+       if (is_zebra_import_table_enabled(afi, re->vrf_id, re->table)) {
+               struct zebra_vrf *zvrf = zebra_vrf_lookup_by_id(re->vrf_id);
+
+               zebra_del_import_table_entry(zvrf, rn, re);
                /* Just clean up if non main table */
                if (IS_ZEBRA_DEBUG_RIB) {
                        char buf[SRCDEST2STR_BUFFER];
@@ -2038,6 +2448,7 @@ void _route_entry_dump(const char *func, union prefixconstptr pp,
        bool is_srcdst = src_p && src_p->prefixlen;
        char straddr[PREFIX_STRLEN];
        char srcaddr[PREFIX_STRLEN];
+       char nhname[PREFIX_STRLEN];
        struct nexthop *nexthop;
 
        zlog_debug("%s: dumping RE entry %p for %s%s%s vrf %u", func,
@@ -2047,12 +2458,12 @@ void _route_entry_dump(const char *func, union prefixconstptr pp,
                             : "",
                   re->vrf_id);
        zlog_debug("%s: uptime == %lu, type == %u, instance == %d, table == %d",
-                  func, (unsigned long)re->uptime, re->type, re->instance,
+                  straddr, (unsigned long)re->uptime, re->type, re->instance,
                   re->table);
        zlog_debug(
                "%s: metric == %u, mtu == %u, distance == %u, flags == %u, status == %u",
-               func, re->metric, re->mtu, re->distance, re->flags, re->status);
-       zlog_debug("%s: nexthop_num == %u, nexthop_active_num == %u", func,
+               straddr, re->metric, re->mtu, re->distance, re->flags, re->status);
+       zlog_debug("%s: nexthop_num == %u, nexthop_active_num == %u", straddr,
                   re->nexthop_num, re->nexthop_active_num);
 
        for (ALL_NEXTHOPS(re->ng, nexthop)) {
@@ -2061,27 +2472,27 @@ void _route_entry_dump(const char *func, union prefixconstptr pp,
 
                switch (nexthop->type) {
                case NEXTHOP_TYPE_BLACKHOLE:
-                       sprintf(straddr, "Blackhole");
+                       sprintf(nhname, "Blackhole");
                        break;
                case NEXTHOP_TYPE_IFINDEX:
                        ifp = if_lookup_by_index(nexthop->ifindex,
                                                 nexthop->vrf_id);
-                       sprintf(straddr, "%s", ifp ? ifp->name : "Unknown");
+                       sprintf(nhname, "%s", ifp ? ifp->name : "Unknown");
                        break;
                case NEXTHOP_TYPE_IPV4:
                        /* fallthrough */
                case NEXTHOP_TYPE_IPV4_IFINDEX:
-                       inet_ntop(AF_INET, &nexthop->gate, straddr,
+                       inet_ntop(AF_INET, &nexthop->gate, nhname,
                                  INET6_ADDRSTRLEN);
                        break;
                case NEXTHOP_TYPE_IPV6:
                case NEXTHOP_TYPE_IPV6_IFINDEX:
-                       inet_ntop(AF_INET6, &nexthop->gate, straddr,
+                       inet_ntop(AF_INET6, &nexthop->gate, nhname,
                                  INET6_ADDRSTRLEN);
                        break;
                }
                zlog_debug("%s: %s %s[%u] vrf %s(%u) with flags %s%s%s%s%s%s",
-                          func, (nexthop->rparent ? "  NH" : "NH"), straddr,
+                          straddr, (nexthop->rparent ? "  NH" : "NH"), nhname,
                           nexthop->ifindex, vrf ? vrf->name : "Unknown",
                           nexthop->vrf_id,
                           (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE)
@@ -2103,7 +2514,7 @@ void _route_entry_dump(const char *func, union prefixconstptr pp,
                                    ? "DUPLICATE "
                                    : ""));
        }
-       zlog_debug("%s: dump complete", func);
+       zlog_debug("%s: dump complete", straddr);
 }
 
 /* This is an exported helper to rtm_read() to dump the strange
@@ -2229,15 +2640,9 @@ int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
                apply_mask_ipv6(src_p);
 
        /* Set default distance by route type. */
-       if (re->distance == 0) {
+       if (re->distance == 0)
                re->distance = route_distance(re->type);
 
-               /* iBGP distance is 200. */
-               if (re->type == ZEBRA_ROUTE_BGP
-                   && CHECK_FLAG(re->flags, ZEBRA_FLAG_IBGP))
-                       re->distance = 200;
-       }
-
        /* Lookup route node.*/
        rn = srcdest_rnode_get(table, p, src_p);
 
@@ -2340,11 +2745,14 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
                else
                        src_buf[0] = '\0';
 
-               if (IS_ZEBRA_DEBUG_RIB)
-                       zlog_debug("%u:%s%s%s doesn't exist in rib", vrf_id,
-                                  dst_buf,
+               if (IS_ZEBRA_DEBUG_RIB) {
+                       struct vrf *vrf = vrf_lookup_by_id(vrf_id);
+
+                       zlog_debug("%s[%d]:%s%s%s doesn't exist in rib",
+                                  vrf->name, table_id, dst_buf,
                                   (src_buf[0] != '\0') ? " from " : "",
                                   src_buf);
+               }
                return;
        }
 
@@ -2380,7 +2788,11 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
                                break;
                        }
                        for (ALL_NEXTHOPS(re->ng, rtnh))
-                               if (nexthop_same_no_recurse(rtnh, nh)) {
+                               /*
+                                * No guarantee all kernel send nh with labels
+                                * on delete.
+                                */
+                               if (nexthop_same_no_labels(rtnh, nh)) {
                                        same = re;
                                        break;
                                }
@@ -2652,6 +3064,7 @@ void rib_sweep_table(struct route_table *table)
 
        for (rn = route_top(table); rn; rn = srcdest_route_next(rn)) {
                RNODE_FOREACH_RE_SAFE (rn, re, next) {
+
                        if (IS_ZEBRA_DEBUG_RIB)
                                route_entry_dump(&rn->p, NULL, re);
 
@@ -2661,6 +3074,14 @@ void rib_sweep_table(struct route_table *table)
                        if (!CHECK_FLAG(re->flags, ZEBRA_FLAG_SELFROUTE))
                                continue;
 
+                       /*
+                        * If routes are older than startup_time then
+                        * we know we read them in from the kernel.
+                        * As such we can safely remove them.
+                        */
+                       if (zrouter.startup_time < re->uptime)
+                               continue;
+
                        /*
                         * So we are starting up and have received
                         * routes from the kernel that we have installed
@@ -2690,7 +3111,7 @@ void rib_sweep_table(struct route_table *table)
 }
 
 /* Sweep all RIB tables.  */
-void rib_sweep_route(void)
+int rib_sweep_route(struct thread *t)
 {
        struct vrf *vrf;
        struct zebra_vrf *zvrf;
@@ -2704,6 +3125,8 @@ void rib_sweep_route(void)
        }
 
        zebra_router_sweep_route();
+
+       return 0;
 }
 
 /* Remove specific by protocol routes from 'table'. */
@@ -2841,13 +3264,40 @@ static int rib_process_dplane_results(struct thread *thread)
                        case DPLANE_OP_ROUTE_INSTALL:
                        case DPLANE_OP_ROUTE_UPDATE:
                        case DPLANE_OP_ROUTE_DELETE:
-                               rib_process_result(ctx);
+                       {
+                               /* Bit of special case for route updates
+                                * that were generated by async notifications:
+                                * we don't want to continue processing these
+                                * in the rib.
+                                */
+                               if (dplane_ctx_get_notif_provider(ctx) == 0)
+                                       rib_process_result(ctx);
+                               else
+                                       dplane_ctx_fini(&ctx);
+                       }
+                       break;
+
+                       case DPLANE_OP_ROUTE_NOTIFY:
+                               rib_process_dplane_notify(ctx);
                                break;
 
                        case DPLANE_OP_LSP_INSTALL:
                        case DPLANE_OP_LSP_UPDATE:
                        case DPLANE_OP_LSP_DELETE:
-                               zebra_mpls_lsp_dplane_result(ctx);
+                       {
+                               /* Bit of special case for LSP updates
+                                * that were generated by async notifications:
+                                * we don't want to continue processing these.
+                                */
+                               if (dplane_ctx_get_notif_provider(ctx) == 0)
+                                       zebra_mpls_lsp_dplane_result(ctx);
+                               else
+                                       dplane_ctx_fini(&ctx);
+                       }
+                       break;
+
+                       case DPLANE_OP_LSP_NOTIFY:
+                               zebra_mpls_process_dplane_notify(ctx);
                                break;
 
                        case DPLANE_OP_PW_INSTALL:
@@ -2940,7 +3390,7 @@ void rib_init(void)
  *
  * Get the first vrf id that is greater than the given vrf id if any.
  *
- * Returns TRUE if a vrf id was found, FALSE otherwise.
+ * Returns true if a vrf id was found, false otherwise.
  */
 static inline int vrf_id_get_next(vrf_id_t vrf_id, vrf_id_t *next_id_p)
 {