]> git.proxmox.com Git - mirror_frr.git/blob - bgpd/rfapi/vnc_export_bgp.c
*: auto-convert to SPDX License IDs
[mirror_frr.git] / bgpd / rfapi / vnc_export_bgp.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 *
4 * Copyright 2009-2016, LabN Consulting, L.L.C.
5 *
6 */
7
8 /*
9 * File: vnc_export_bgp.c
10 * Purpose: Export routes to BGP directly (not via zebra)
11 */
12
13 #include "lib/zebra.h"
14 #include "lib/prefix.h"
15 #include "lib/agg_table.h"
16 #include "lib/vty.h"
17 #include "lib/log.h"
18 #include "lib/stream.h"
19 #include "lib/memory.h"
20 #include "lib/linklist.h"
21 #include "lib/plist.h"
22 #include "lib/routemap.h"
23 #include "lib/lib_errors.h"
24
25 #include "bgpd/bgpd.h"
26 #include "bgpd/bgp_ecommunity.h"
27 #include "bgpd/bgp_attr.h"
28 #include "bgpd/bgp_aspath.h"
29
30 #include "bgpd/rfapi/vnc_export_bgp.h"
31 #include "bgpd/rfapi/vnc_export_bgp_p.h"
32 #include "bgpd/rfapi/vnc_export_table.h"
33 #include "bgpd/rfapi/bgp_rfapi_cfg.h"
34 #include "bgpd/rfapi/rfapi.h"
35 #include "bgpd/rfapi/rfapi_import.h"
36 #include "bgpd/rfapi/rfapi_private.h"
37 #include "bgpd/rfapi/rfapi_backend.h"
38 #include "bgpd/rfapi/rfapi_vty.h"
39 #include "bgpd/rfapi/vnc_debug.h"
40
41
42 static void vnc_direct_add_rn_group_rd(struct bgp *bgp,
43 struct rfapi_nve_group_cfg *rfg,
44 struct agg_node *rn, struct attr *attr,
45 afi_t afi,
46 struct rfapi_descriptor *irfd);
47
48 /***********************************************************************
49 * Export methods that set nexthop to CE (from 5226 roo EC) BEGIN
50 ***********************************************************************/
51
52 /*
53 * Memory allocation approach: make a ghost attr that
54 * has non-interned parts for the modifications. ghost attr
55 * memory is allocated by caller.
56 *
57 * - extract ce (=5226) EC and use as new nexthop
58 * - strip Tunnel Encap attr
59 * - copy all ECs
60 */
61 static void encap_attr_export_ce(struct attr *new, struct attr *orig,
62 struct prefix *use_nexthop)
63 {
64 /*
65 * Make "new" a ghost attr copy of "orig"
66 */
67 memset(new, 0, sizeof(struct attr));
68 *new = *orig;
69
70 /*
71 * Set nexthop
72 */
73 switch (use_nexthop->family) {
74 case AF_INET:
75 new->nexthop = use_nexthop->u.prefix4;
76 new->mp_nexthop_len = BGP_ATTR_NHLEN_IPV4; /* bytes */
77 new->flag |= ATTR_FLAG_BIT(BGP_ATTR_NEXT_HOP);
78 break;
79
80 case AF_INET6:
81 new->mp_nexthop_global = use_nexthop->u.prefix6;
82 new->mp_nexthop_len = BGP_ATTR_NHLEN_IPV6_GLOBAL; /* bytes */
83 break;
84
85 default:
86 assert(0);
87 break;
88 }
89
90 /*
91 * Set MED
92 *
93 * Note that it will be deleted when BGP sends to any eBGP
94 * peer unless PEER_FLAG_MED_UNCHANGED is set:
95 *
96 * neighbor NEIGHBOR attribute-unchanged med
97 */
98 if (!CHECK_FLAG(new->flag, BGP_ATTR_MULTI_EXIT_DISC)) {
99 if (CHECK_FLAG(new->flag, BGP_ATTR_LOCAL_PREF)) {
100 if (new->local_pref > 255)
101 new->med = 0;
102 else
103 new->med = 255 - new->local_pref;
104 } else {
105 new->med = 255; /* shouldn't happen */
106 }
107 new->flag |= ATTR_FLAG_BIT(BGP_ATTR_MULTI_EXIT_DISC);
108 }
109
110 /*
111 * "new" is now a ghost attr:
112 * - it owns an "extra" struct
113 * - it owns any non-interned parts
114 * - any references to interned parts are not counted
115 *
116 * Caller should, after using the attr, call:
117 * - bgp_attr_flush() to free non-interned parts
118 */
119 }
120
121 static int getce(struct bgp *bgp, struct attr *attr, struct prefix *pfx_ce)
122 {
123 uint8_t *ecp;
124 uint32_t i;
125 uint16_t localadmin = bgp->rfapi_cfg->resolve_nve_roo_local_admin;
126 struct ecommunity *ecomm = bgp_attr_get_ecommunity(attr);
127
128 for (ecp = ecomm->val, i = 0; i < ecomm->size;
129 ++i, ecp += ECOMMUNITY_SIZE) {
130
131 if (VNC_DEBUG(EXPORT_BGP_GETCE)) {
132 vnc_zlog_debug_any(
133 "%s: %02x %02x %02x %02x %02x %02x %02x %02x",
134 __func__, ecp[0], ecp[1], ecp[2], ecp[3],
135 ecp[4], ecp[5], ecp[6], ecp[7]);
136 }
137
138 /*
139 * is it ROO?
140 */
141 if (ecp[0] != 1 || ecp[1] != 3) {
142 continue;
143 }
144
145 /*
146 * Match local admin value?
147 */
148 if (ecp[6] != ((localadmin & 0xff00) >> 8)
149 || ecp[7] != (localadmin & 0xff))
150 continue;
151
152 memset((uint8_t *)pfx_ce, 0, sizeof(*pfx_ce));
153 memcpy(&pfx_ce->u.prefix4, ecp + 2, 4);
154 pfx_ce->family = AF_INET;
155 pfx_ce->prefixlen = IPV4_MAX_BITLEN;
156
157 return 0;
158 }
159 return -1;
160 }
161
162
163 void vnc_direct_bgp_add_route_ce(struct bgp *bgp, struct agg_node *rn,
164 struct bgp_path_info *bpi)
165 {
166 struct attr *attr = bpi->attr;
167 struct peer *peer = bpi->peer;
168 const struct prefix *prefix = agg_node_get_prefix(rn);
169 afi_t afi = family2afi(prefix->family);
170 struct bgp_dest *udest;
171 struct bgp_path_info *ubpi;
172 struct attr hattr;
173 struct attr *iattr;
174 struct prefix ce_nexthop;
175 struct prefix post_routemap_nexthop;
176
177
178 if (!afi) {
179 flog_err(EC_LIB_DEVELOPMENT, "%s: can't get afi of route node",
180 __func__);
181 return;
182 }
183
184 if ((bpi->type != ZEBRA_ROUTE_BGP)
185 || (bpi->sub_type != BGP_ROUTE_NORMAL
186 && bpi->sub_type != BGP_ROUTE_RFP
187 && bpi->sub_type != BGP_ROUTE_STATIC)) {
188
189 vnc_zlog_debug_verbose(
190 "%s: wrong route type/sub_type for export, skipping",
191 __func__);
192 return;
193 }
194
195 /* check bgp redist flag for vnc direct ("vpn") routes */
196 if (!bgp->redist[afi][ZEBRA_ROUTE_VNC_DIRECT]) {
197 vnc_zlog_debug_verbose(
198 "%s: bgp redistribution of VNC direct routes is off",
199 __func__);
200 return;
201 }
202
203 if (!bgp->rfapi_cfg) {
204 vnc_zlog_debug_verbose("%s: bgp->rfapi_cfg is NULL, skipping",
205 __func__);
206 return;
207 }
208
209 if (!VNC_EXPORT_BGP_CE_ENABLED(bgp->rfapi_cfg)) {
210 vnc_zlog_debug_verbose(
211 "%s: export-to-bgp ce mode not enabled, skipping",
212 __func__);
213 return;
214 }
215
216 /*
217 * prefix list check
218 */
219 if (bgp->rfapi_cfg->plist_export_bgp[afi]) {
220 if (prefix_list_apply(bgp->rfapi_cfg->plist_export_bgp[afi],
221 prefix)
222 == PREFIX_DENY) {
223 vnc_zlog_debug_verbose(
224 "%s: prefix list denied, skipping", __func__);
225 return;
226 }
227 }
228
229
230 /*
231 * Extract CE
232 * This works only for IPv4 because IPv6 addresses are too big
233 * to fit in an extended community
234 */
235 if (getce(bgp, attr, &ce_nexthop)) {
236 vnc_zlog_debug_verbose("%s: EC has no encoded CE, skipping",
237 __func__);
238 return;
239 }
240
241 /*
242 * Is this route already represented in the unicast RIB?
243 * (look up prefix; compare route type, sub_type, peer, nexthop)
244 */
245 udest = bgp_afi_node_get(bgp->rib[afi][SAFI_UNICAST], afi, SAFI_UNICAST,
246 prefix, NULL);
247 for (ubpi = bgp_dest_get_bgp_path_info(udest); ubpi;
248 ubpi = ubpi->next) {
249 struct prefix unicast_nexthop;
250
251 if (CHECK_FLAG(ubpi->flags, BGP_PATH_REMOVED))
252 continue;
253
254 rfapiUnicastNexthop2Prefix(afi, ubpi->attr, &unicast_nexthop);
255
256 if (ubpi->type == ZEBRA_ROUTE_VNC_DIRECT
257 && ubpi->sub_type == BGP_ROUTE_REDISTRIBUTE
258 && ubpi->peer == peer
259 && prefix_same(&unicast_nexthop, &ce_nexthop)) {
260
261 vnc_zlog_debug_verbose(
262 "%s: already have matching exported unicast route, skipping",
263 __func__);
264 return;
265 }
266 }
267
268 /*
269 * Construct new attribute set with CE addr as
270 * nexthop and without Tunnel Encap attr
271 */
272 encap_attr_export_ce(&hattr, attr, &ce_nexthop);
273 if (bgp->rfapi_cfg->routemap_export_bgp) {
274 struct bgp_path_info info;
275 route_map_result_t ret;
276
277 memset(&info, 0, sizeof(info));
278 info.peer = peer;
279 info.attr = &hattr;
280 ret = route_map_apply(bgp->rfapi_cfg->routemap_export_bgp,
281 prefix, &info);
282 if (ret == RMAP_DENYMATCH) {
283 bgp_attr_flush(&hattr);
284 return;
285 }
286 }
287
288 iattr = bgp_attr_intern(&hattr);
289 bgp_attr_flush(&hattr);
290
291 /*
292 * Rule: disallow route-map alteration of next-hop, because it
293 * would make it too difficult to keep track of the correspondence
294 * between VPN routes and unicast routes.
295 */
296 rfapiUnicastNexthop2Prefix(afi, iattr, &post_routemap_nexthop);
297
298 if (!prefix_same(&ce_nexthop, &post_routemap_nexthop)) {
299 vnc_zlog_debug_verbose(
300 "%s: route-map modification of nexthop not allowed, skipping",
301 __func__);
302 bgp_attr_unintern(&iattr);
303 return;
304 }
305
306 bgp_update(peer, prefix, 0, /* addpath_id */
307 iattr, /* bgp_update copies this attr */
308 afi, SAFI_UNICAST, ZEBRA_ROUTE_VNC_DIRECT,
309 BGP_ROUTE_REDISTRIBUTE, NULL, /* RD not used for unicast */
310 NULL, 0, /* tag not used for unicast */
311 0, NULL); /* EVPN not used */
312 bgp_attr_unintern(&iattr);
313 }
314
315
316 /*
317 * "Withdrawing a Route" export process
318 */
319 void vnc_direct_bgp_del_route_ce(struct bgp *bgp, struct agg_node *rn,
320 struct bgp_path_info *bpi)
321 {
322 const struct prefix *p = agg_node_get_prefix(rn);
323 afi_t afi = family2afi(p->family);
324 struct bgp_path_info *vbpi;
325 struct prefix ce_nexthop;
326
327 if (!afi) {
328 flog_err(EC_LIB_DEVELOPMENT, "%s: bad afi", __func__);
329 return;
330 }
331
332 /* check bgp redist flag for vnc direct ("vpn") routes */
333 if (!bgp->redist[afi][ZEBRA_ROUTE_VNC_DIRECT]) {
334 vnc_zlog_debug_verbose(
335 "%s: bgp redistribution of VNC direct routes is off",
336 __func__);
337 return;
338 }
339
340 if (!bgp->rfapi_cfg) {
341 vnc_zlog_debug_verbose("%s: bgp->rfapi_cfg is NULL, skipping",
342 __func__);
343 return;
344 }
345 if (!VNC_EXPORT_BGP_CE_ENABLED(bgp->rfapi_cfg)) {
346 vnc_zlog_debug_verbose(
347 "%s: export-to-bgp ce mode not enabled, skipping",
348 __func__);
349 return;
350 }
351
352 /*
353 * Extract CE
354 * This works only for IPv4 because IPv6 addresses are too big
355 * to fit in an extended community
356 */
357 if (getce(bgp, bpi->attr, &ce_nexthop)) {
358 vnc_zlog_debug_verbose("%s: EC has no encoded CE, skipping",
359 __func__);
360 return;
361 }
362
363 /*
364 * Look for other VPN routes with same prefix, same 5226 CE,
365 * same peer. If at least one is present, don't remove the
366 * route from the unicast RIB
367 */
368
369 for (vbpi = rn->info; vbpi; vbpi = vbpi->next) {
370 struct prefix ce;
371 if (bpi == vbpi)
372 continue;
373 if (bpi->peer != vbpi->peer)
374 continue;
375 if (getce(bgp, vbpi->attr, &ce))
376 continue;
377 if (prefix_same(&ce, &ce_nexthop)) {
378 vnc_zlog_debug_verbose(
379 "%s: still have a route via CE, not deleting unicast",
380 __func__);
381 return;
382 }
383 }
384
385 /*
386 * withdraw the route
387 */
388 bgp_withdraw(bpi->peer, p, 0, /* addpath_id */
389 NULL, /* attr, ignored */
390 afi, SAFI_UNICAST, ZEBRA_ROUTE_VNC_DIRECT,
391 BGP_ROUTE_REDISTRIBUTE, NULL, /* RD not used for unicast */
392 NULL, 0, NULL); /* tag not used for unicast */
393 }
394
395 static void vnc_direct_bgp_vpn_enable_ce(struct bgp *bgp, afi_t afi)
396 {
397 struct agg_node *rn;
398 struct bgp_path_info *ri;
399
400 vnc_zlog_debug_verbose("%s: entry, afi=%d", __func__, afi);
401
402 if (!bgp)
403 return;
404
405 if (!(bgp->rfapi_cfg))
406 return;
407
408 if (!VNC_EXPORT_BGP_CE_ENABLED(bgp->rfapi_cfg)) {
409 vnc_zlog_debug_verbose(
410 "%s: export of CE routes not enabled, skipping",
411 __func__);
412 return;
413 }
414
415 if (afi != AFI_IP && afi != AFI_IP6) {
416 vnc_zlog_debug_verbose("%s: bad afi: %d", __func__, afi);
417 return;
418 }
419
420 /*
421 * Go through entire ce import table and export to BGP unicast.
422 */
423 for (rn = agg_route_top(bgp->rfapi->it_ce->imported_vpn[afi]); rn;
424 rn = agg_route_next(rn)) {
425 if (!rn->info)
426 continue;
427
428 vnc_zlog_debug_verbose("%s: checking prefix %pRN", __func__,
429 rn);
430
431 for (ri = rn->info; ri; ri = ri->next) {
432
433 vnc_zlog_debug_verbose("%s: ri->sub_type: %d", __func__,
434 ri->sub_type);
435
436 if (ri->sub_type == BGP_ROUTE_NORMAL
437 || ri->sub_type == BGP_ROUTE_RFP
438 || ri->sub_type == BGP_ROUTE_STATIC) {
439
440 vnc_direct_bgp_add_route_ce(bgp, rn, ri);
441 }
442 }
443 }
444 }
445
446 static void vnc_direct_bgp_vpn_disable_ce(struct bgp *bgp, afi_t afi)
447 {
448 struct bgp_dest *dest;
449
450 vnc_zlog_debug_verbose("%s: entry, afi=%d", __func__, afi);
451
452 if (!bgp)
453 return;
454
455 if (afi != AFI_IP && afi != AFI_IP6) {
456 vnc_zlog_debug_verbose("%s: bad afi: %d", __func__, afi);
457 return;
458 }
459
460 /*
461 * Go through the entire BGP unicast table and remove routes that
462 * originated from us
463 */
464 for (dest = bgp_table_top(bgp->rib[afi][SAFI_UNICAST]); dest;
465 dest = bgp_route_next(dest)) {
466
467 struct bgp_path_info *ri;
468 struct bgp_path_info *next;
469
470 for (ri = bgp_dest_get_bgp_path_info(dest), next = NULL; ri;
471 ri = next) {
472
473 next = ri->next;
474
475 if (ri->type == ZEBRA_ROUTE_VNC_DIRECT
476 && ri->sub_type == BGP_ROUTE_REDISTRIBUTE) {
477
478 bgp_withdraw(
479 ri->peer, bgp_dest_get_prefix(dest),
480 0, /* addpath_id */
481 NULL, /* ignored */
482 AFI_IP, SAFI_UNICAST,
483 ZEBRA_ROUTE_VNC_DIRECT,
484 BGP_ROUTE_REDISTRIBUTE,
485 NULL, /* RD not used for unicast */
486 NULL, 0,
487 NULL); /* tag not used for unicast */
488 }
489 }
490 }
491 }
492
493 /***********************************************************************
494 * Export methods that set nexthop to CE (from 5226 roo EC) END
495 ***********************************************************************/
496
497 /***********************************************************************
498 * Export methods that proxy nexthop BEGIN
499 ***********************************************************************/
500
501 static struct ecommunity *vnc_route_origin_ecom(struct agg_node *rn)
502 {
503 struct ecommunity *new;
504 struct bgp_path_info *bpi;
505
506 if (!rn->info)
507 return NULL;
508
509 new = ecommunity_new();
510
511 for (bpi = rn->info; bpi; bpi = bpi->next) {
512
513 struct ecommunity_val roec;
514
515 switch (BGP_MP_NEXTHOP_FAMILY(bpi->attr->mp_nexthop_len)) {
516 case AF_INET:
517 memset(&roec, 0, sizeof(roec));
518 roec.val[0] = 0x01;
519 roec.val[1] = 0x03;
520 memcpy(roec.val + 2,
521 &bpi->attr->mp_nexthop_global_in.s_addr, 4);
522 roec.val[6] = 0;
523 roec.val[7] = 0;
524 ecommunity_add_val(new, &roec, false, false);
525 break;
526 case AF_INET6:
527 /* No support for IPv6 addresses in extended communities
528 */
529 break;
530 }
531 }
532
533 if (!new->size) {
534 ecommunity_free(&new);
535 new = NULL;
536 }
537
538 return new;
539 }
540
541 static struct ecommunity *vnc_route_origin_ecom_single(struct in_addr *origin)
542 {
543 struct ecommunity *new;
544 struct ecommunity_val roec;
545
546 memset(&roec, 0, sizeof(roec));
547 roec.val[0] = 0x01;
548 roec.val[1] = 0x03;
549 memcpy(roec.val + 2, &origin->s_addr, 4);
550 roec.val[6] = 0;
551 roec.val[7] = 0;
552
553 new = ecommunity_new();
554 ecommunity_add_val(new, &roec, false, false);
555
556 if (!new->size) {
557 ecommunity_free(&new);
558 new = NULL;
559 }
560
561 return new;
562 }
563
564
565 /*
566 * New memory allocation approach: make a ghost attr that
567 * has non-interned parts for the modifications. ghost attr
568 * memory is allocated by caller.
569 */
570 static int
571 encap_attr_export(struct attr *new, struct attr *orig,
572 struct prefix *new_nexthop,
573 struct agg_node *rn) /* for VN addrs for ecom list */
574 /* if rn is 0, use route's nexthop */
575 {
576 struct prefix orig_nexthop;
577 struct prefix *use_nexthop;
578 static struct ecommunity *ecom_ro;
579
580 if (new_nexthop) {
581 use_nexthop = new_nexthop;
582 } else {
583 use_nexthop = &orig_nexthop;
584 orig_nexthop.family =
585 BGP_MP_NEXTHOP_FAMILY(orig->mp_nexthop_len);
586 if (orig_nexthop.family == AF_INET) {
587 orig_nexthop.prefixlen = IPV4_MAX_BITLEN;
588 orig_nexthop.u.prefix4 = orig->mp_nexthop_global_in;
589 } else if (orig_nexthop.family == AF_INET6) {
590 orig_nexthop.prefixlen = IPV6_MAX_BITLEN;
591 orig_nexthop.u.prefix6 = orig->mp_nexthop_global;
592 } else {
593 return -1; /* FAIL - can't compute nexthop */
594 }
595 }
596
597
598 /*
599 * Make "new" a ghost attr copy of "orig"
600 */
601 memset(new, 0, sizeof(struct attr));
602 *new = *orig;
603
604 /*
605 * Set nexthop
606 */
607 switch (use_nexthop->family) {
608 case AF_INET:
609 new->nexthop = use_nexthop->u.prefix4;
610 new->mp_nexthop_len = BGP_ATTR_NHLEN_IPV4; /* bytes */
611 new->flag |= ATTR_FLAG_BIT(BGP_ATTR_NEXT_HOP);
612 break;
613
614 case AF_INET6:
615 new->mp_nexthop_global = use_nexthop->u.prefix6;
616 new->mp_nexthop_len = BGP_ATTR_NHLEN_IPV6_GLOBAL; /* bytes */
617 break;
618
619 default:
620 assert(0);
621 break;
622 }
623
624 if (rn) {
625 ecom_ro = vnc_route_origin_ecom(rn);
626 } else {
627 /* TBD use lcom for IPv6 */
628 ecom_ro = vnc_route_origin_ecom_single(&use_nexthop->u.prefix4);
629 }
630 if (bgp_attr_get_ecommunity(new)) {
631 if (ecom_ro)
632 bgp_attr_set_ecommunity(
633 new,
634 ecommunity_merge(ecom_ro,
635 bgp_attr_get_ecommunity(new)));
636 } else {
637 bgp_attr_set_ecommunity(new, ecom_ro);
638 }
639
640 /*
641 * Set MED
642 *
643 * Note that it will be deleted when BGP sends to any eBGP
644 * peer unless PEER_FLAG_MED_UNCHANGED is set:
645 *
646 * neighbor NEIGHBOR attribute-unchanged med
647 */
648 if (!CHECK_FLAG(new->flag, BGP_ATTR_MULTI_EXIT_DISC)) {
649 if (CHECK_FLAG(new->flag, BGP_ATTR_LOCAL_PREF)) {
650 if (new->local_pref > 255)
651 new->med = 0;
652 else
653 new->med = 255 - new->local_pref;
654 } else {
655 new->med = 255; /* shouldn't happen */
656 }
657 new->flag |= ATTR_FLAG_BIT(BGP_ATTR_MULTI_EXIT_DISC);
658 }
659
660 /*
661 * "new" is now a ghost attr:
662 * - it owns an "extra" struct
663 * - it owns any non-interned parts
664 * - any references to interned parts are not counted
665 *
666 * Caller should, after using the attr, call:
667 * - bgp_attr_flush() to free non-interned parts
668 */
669
670 return 0;
671 }
672
673 /*
674 * "Adding a Route" export process
675 */
676 void vnc_direct_bgp_add_prefix(struct bgp *bgp,
677 struct rfapi_import_table *import_table,
678 struct agg_node *rn)
679 {
680 struct attr attr = {0};
681 struct listnode *node, *nnode;
682 struct rfapi_rfg_name *rfgn;
683 const struct prefix *p = agg_node_get_prefix(rn);
684 afi_t afi = family2afi(p->family);
685
686 if (!afi) {
687 flog_err(EC_LIB_DEVELOPMENT, "%s: can't get afi of route node",
688 __func__);
689 return;
690 }
691
692 /* check bgp redist flag for vnc direct ("vpn") routes */
693 if (!bgp->redist[afi][ZEBRA_ROUTE_VNC_DIRECT]) {
694 vnc_zlog_debug_verbose(
695 "%s: bgp redistribution of VNC direct routes is off",
696 __func__);
697 return;
698 }
699
700 if (!bgp->rfapi_cfg) {
701 vnc_zlog_debug_verbose("%s: bgp->rfapi_cfg is NULL, skipping",
702 __func__);
703 return;
704 }
705
706 if (!VNC_EXPORT_BGP_GRP_ENABLED(bgp->rfapi_cfg)) {
707 vnc_zlog_debug_verbose(
708 "%s: export-to-bgp group mode not enabled, skipping",
709 __func__);
710 return;
711 }
712
713 if (!listcount(bgp->rfapi_cfg->rfg_export_direct_bgp_l)) {
714 vnc_zlog_debug_verbose(
715 "%s: no bgp-direct export nve group, skipping",
716 __func__);
717 return;
718 }
719
720 bgp_attr_default_set(&attr, bgp, BGP_ORIGIN_INCOMPLETE);
721 /* TBD set some configured med, see add_vnc_route() */
722
723 vnc_zlog_debug_verbose(
724 "%s: looping over nve-groups in direct-bgp export list",
725 __func__);
726
727 for (ALL_LIST_ELEMENTS(bgp->rfapi_cfg->rfg_export_direct_bgp_l, node,
728 nnode, rfgn)) {
729
730 struct listnode *ln;
731
732 /*
733 * If nve group is not defined yet, skip it
734 */
735 if (!rfgn->rfg)
736 continue;
737
738 /*
739 * If the nve group uses a different import table, skip it
740 */
741 if (import_table != rfgn->rfg->rfapi_import_table)
742 continue;
743
744 /*
745 * if no NVEs currently associated with this group, skip it
746 */
747 if (rfgn->rfg->type != RFAPI_GROUP_CFG_VRF && !rfgn->rfg->nves)
748 continue;
749
750 /*
751 * per-nve-group prefix list check
752 */
753 if (rfgn->rfg->plist_export_bgp[afi]) {
754 if (prefix_list_apply(rfgn->rfg->plist_export_bgp[afi],
755 p)
756 == PREFIX_DENY)
757
758 continue;
759 }
760
761 if (rfgn->rfg->type == RFAPI_GROUP_CFG_VRF) {
762 vnc_direct_add_rn_group_rd(bgp, rfgn->rfg, rn, &attr,
763 afi, rfgn->rfg->rfd);
764 /*
765 * yuck!
766 * - but consistent with rest of function
767 */
768 continue;
769 }
770 /*
771 * For each NVE that is assigned to the export nve group,
772 * generate
773 * a route with that NVE as its next hop
774 */
775 for (ln = listhead(rfgn->rfg->nves); ln;
776 ln = listnextnode(ln)) {
777 vnc_direct_add_rn_group_rd(bgp, rfgn->rfg, rn, &attr,
778 afi, listgetdata(ln));
779 }
780 }
781
782 aspath_unintern(&attr.aspath);
783 }
784
785 /*
786 * "Withdrawing a Route" export process
787 */
788 void vnc_direct_bgp_del_prefix(struct bgp *bgp,
789 struct rfapi_import_table *import_table,
790 struct agg_node *rn)
791 {
792 struct listnode *node, *nnode;
793 struct rfapi_rfg_name *rfgn;
794 const struct prefix *p = agg_node_get_prefix(rn);
795 afi_t afi = family2afi(p->family);
796
797 if (!afi) {
798 flog_err(EC_LIB_DEVELOPMENT, "%s: can't get afi route node",
799 __func__);
800 return;
801 }
802
803 /* check bgp redist flag for vnc direct ("vpn") routes */
804 if (!bgp->redist[afi][ZEBRA_ROUTE_VNC_DIRECT]) {
805 vnc_zlog_debug_verbose(
806 "%s: bgp redistribution of VNC direct routes is off",
807 __func__);
808 return;
809 }
810
811 if (!bgp->rfapi_cfg) {
812 vnc_zlog_debug_verbose("%s: bgp->rfapi_cfg is NULL, skipping",
813 __func__);
814 return;
815 }
816
817 if (!VNC_EXPORT_BGP_GRP_ENABLED(bgp->rfapi_cfg)) {
818 vnc_zlog_debug_verbose(
819 "%s: export-to-bgp group mode not enabled, skipping",
820 __func__);
821 return;
822 }
823
824 if (!listcount(bgp->rfapi_cfg->rfg_export_direct_bgp_l)) {
825 vnc_zlog_debug_verbose(
826 "%s: no bgp-direct export nve group, skipping",
827 __func__);
828 return;
829 }
830
831 for (ALL_LIST_ELEMENTS(bgp->rfapi_cfg->rfg_export_direct_bgp_l, node,
832 nnode, rfgn)) {
833
834 struct listnode *ln;
835
836 /*
837 * If nve group is not defined yet, skip it
838 */
839 if (!rfgn->rfg)
840 continue;
841
842 /*
843 * if no NVEs currently associated with this group, skip it
844 */
845 if (rfgn->rfg->type != RFAPI_GROUP_CFG_VRF && !rfgn->rfg->nves)
846 continue;
847
848 /*
849 * If the nve group uses a different import table,
850 * skip it
851 */
852 if (import_table != rfgn->rfg->rfapi_import_table)
853 continue;
854
855 if (rfgn->rfg->type == RFAPI_GROUP_CFG_VRF) {
856 struct prefix nhp;
857 struct rfapi_descriptor *irfd;
858
859 irfd = rfgn->rfg->rfd;
860
861 if (rfapiRaddr2Qprefix(&irfd->vn_addr, &nhp))
862 continue;
863
864 bgp_withdraw(irfd->peer, p, /* prefix */
865 0, /* addpath_id */
866 NULL, /* attr, ignored */
867 afi, SAFI_UNICAST, ZEBRA_ROUTE_VNC_DIRECT,
868 BGP_ROUTE_REDISTRIBUTE,
869 NULL, /* RD not used for unicast */
870 NULL, 0,
871 NULL); /* tag not used for unicast */
872 /*
873 * yuck!
874 * - but consistent with rest of function
875 */
876 continue;
877 }
878 /*
879 * For each NVE that is assigned to the export nve group,
880 * generate
881 * a route with that NVE as its next hop
882 */
883 for (ln = listhead(rfgn->rfg->nves); ln;
884 ln = listnextnode(ln)) {
885
886 struct prefix nhp;
887 struct rfapi_descriptor *irfd;
888
889 irfd = listgetdata(ln);
890
891 if (rfapiRaddr2Qprefix(&irfd->vn_addr, &nhp))
892 continue;
893
894 bgp_withdraw(irfd->peer, p, /* prefix */
895 0, /* addpath_id */
896 NULL, /* attr, ignored */
897 afi, SAFI_UNICAST, ZEBRA_ROUTE_VNC_DIRECT,
898 BGP_ROUTE_REDISTRIBUTE,
899 NULL, /* RD not used for unicast */
900 NULL, 0,
901 NULL); /* tag not used for unicast */
902 }
903 }
904 }
905
906 void vnc_direct_bgp_add_nve(struct bgp *bgp, struct rfapi_descriptor *rfd)
907 {
908 struct listnode *node, *nnode;
909 struct rfapi_rfg_name *rfgn;
910 struct rfapi_nve_group_cfg *rfg = rfd->rfg;
911 afi_t afi = family2afi(rfd->vn_addr.addr_family);
912
913 if (!afi) {
914 flog_err(EC_LIB_DEVELOPMENT, "%s: can't get afi of nve vn addr",
915 __func__);
916 return;
917 }
918
919 if (!bgp)
920 return;
921 if (!bgp->rfapi_cfg) {
922 vnc_zlog_debug_verbose("%s: bgp->rfapi_cfg is NULL, skipping",
923 __func__);
924 return;
925 }
926 if (!VNC_EXPORT_BGP_GRP_ENABLED(bgp->rfapi_cfg)) {
927 vnc_zlog_debug_verbose(
928 "%s: export-to-bgp group mode not enabled, skipping",
929 __func__);
930 return;
931 }
932
933 if (!bgp->redist[afi][ZEBRA_ROUTE_VNC_DIRECT]) {
934 vnc_zlog_debug_verbose(
935 "%s: bgp redistribution of VNC direct routes is off",
936 __func__);
937 return;
938 }
939
940 /*
941 * Loop over the list of NVE-Groups configured for
942 * exporting to direct-bgp and see if this new NVE's
943 * group is among them.
944 */
945 for (ALL_LIST_ELEMENTS(bgp->rfapi_cfg->rfg_export_direct_bgp_l, node,
946 nnode, rfgn)) {
947
948 /*
949 * Yes, this NVE's group is configured for export to direct-bgp
950 */
951 if (rfgn->rfg == rfg) {
952
953 struct agg_table *rt = NULL;
954 struct agg_node *rn;
955 struct attr attr = {0};
956 struct rfapi_import_table *import_table;
957
958
959 import_table = rfg->rfapi_import_table;
960
961 if (afi == AFI_IP || afi == AFI_IP6) {
962 rt = import_table->imported_vpn[afi];
963 } else {
964 flog_err(EC_LIB_DEVELOPMENT, "%s: bad afi %d",
965 __func__, afi);
966 return;
967 }
968
969 bgp_attr_default_set(&attr, bgp, BGP_ORIGIN_INCOMPLETE);
970 /* TBD set some configured med, see add_vnc_route() */
971
972 /*
973 * Walk the NVE-Group's VNC Import table
974 */
975 for (rn = agg_route_top(rt); rn;
976 rn = agg_route_next(rn)) {
977
978 if (rn->info) {
979
980 struct prefix nhp;
981 struct rfapi_descriptor *irfd = rfd;
982 struct attr hattr;
983 struct attr *iattr;
984 struct bgp_path_info info;
985 const struct prefix *p =
986 agg_node_get_prefix(rn);
987
988 if (rfapiRaddr2Qprefix(&irfd->vn_addr,
989 &nhp))
990 continue;
991
992 /*
993 * per-nve-group prefix list check
994 */
995 if (rfgn->rfg->plist_export_bgp[afi]) {
996 if (prefix_list_apply(
997 rfgn->rfg->plist_export_bgp
998 [afi],
999 p)
1000 == PREFIX_DENY)
1001
1002 continue;
1003 }
1004
1005
1006 /*
1007 * Construct new attribute set with
1008 * NVE's VN addr as
1009 * nexthop and without Tunnel Encap attr
1010 */
1011 if (encap_attr_export(&hattr, &attr,
1012 &nhp, rn))
1013 continue;
1014
1015 if (rfgn->rfg->routemap_export_bgp) {
1016 route_map_result_t ret;
1017 info.peer = irfd->peer;
1018 info.attr = &hattr;
1019 ret = route_map_apply(
1020 rfgn->rfg
1021 ->routemap_export_bgp,
1022 p, &info);
1023 if (ret == RMAP_DENYMATCH) {
1024 bgp_attr_flush(&hattr);
1025 continue;
1026 }
1027 }
1028
1029 iattr = bgp_attr_intern(&hattr);
1030 bgp_attr_flush(&hattr);
1031 bgp_update(
1032 irfd->peer, p, /* prefix */
1033 0, /* addpath_id */
1034 iattr, /* bgp_update copies
1035 it */
1036 afi, SAFI_UNICAST,
1037 ZEBRA_ROUTE_VNC_DIRECT,
1038 BGP_ROUTE_REDISTRIBUTE, NULL,
1039 /* RD not used for unicast */
1040 NULL,
1041 /* tag not used for unicast */
1042 0, 0, NULL); /* EVPN not used */
1043
1044 bgp_attr_unintern(&iattr);
1045 }
1046 }
1047
1048 aspath_unintern(&attr.aspath);
1049 }
1050 }
1051 }
1052
1053
1054 void vnc_direct_bgp_del_nve(struct bgp *bgp, struct rfapi_descriptor *rfd)
1055 {
1056 struct listnode *node, *nnode;
1057 struct rfapi_rfg_name *rfgn;
1058 struct rfapi_nve_group_cfg *rfg = rfd->rfg;
1059 afi_t afi = family2afi(rfd->vn_addr.addr_family);
1060
1061 if (!afi) {
1062 flog_err(EC_LIB_DEVELOPMENT, "%s: can't get afi of nve vn addr",
1063 __func__);
1064 return;
1065 }
1066
1067 if (!bgp)
1068 return;
1069 if (!bgp->rfapi_cfg) {
1070 vnc_zlog_debug_verbose("%s: bgp->rfapi_cfg is NULL, skipping",
1071 __func__);
1072 return;
1073 }
1074 if (!VNC_EXPORT_BGP_GRP_ENABLED(bgp->rfapi_cfg)) {
1075 vnc_zlog_debug_verbose(
1076 "%s: export-to-bgp group mode not enabled, skipping",
1077 __func__);
1078 return;
1079 }
1080
1081 if (!bgp->redist[afi][ZEBRA_ROUTE_VNC_DIRECT]) {
1082 vnc_zlog_debug_verbose(
1083 "%s: bgp redistribution of VNC direct routes is off",
1084 __func__);
1085 return;
1086 }
1087
1088 /*
1089 * Loop over the list of NVE-Groups configured for
1090 * exporting to direct-bgp and see if this new NVE's
1091 * group is among them.
1092 */
1093 for (ALL_LIST_ELEMENTS(bgp->rfapi_cfg->rfg_export_direct_bgp_l, node,
1094 nnode, rfgn)) {
1095
1096 /*
1097 * Yes, this NVE's group is configured for export to direct-bgp
1098 */
1099 if (rfg && rfgn->rfg == rfg) {
1100
1101 struct agg_table *rt = NULL;
1102 struct agg_node *rn;
1103 struct rfapi_import_table *import_table;
1104
1105 import_table = rfg->rfapi_import_table;
1106
1107 if (afi == AFI_IP || afi == AFI_IP6) {
1108 rt = import_table->imported_vpn[afi];
1109 } else {
1110 flog_err(EC_LIB_DEVELOPMENT, "%s: bad afi %d",
1111 __func__, afi);
1112 return;
1113 }
1114
1115 /*
1116 * Walk the NVE-Group's VNC Import table
1117 */
1118 for (rn = agg_route_top(rt); rn;
1119 rn = agg_route_next(rn)) {
1120
1121 if (rn->info) {
1122 const struct prefix *p =
1123 agg_node_get_prefix(rn);
1124 struct prefix nhp;
1125 struct rfapi_descriptor *irfd = rfd;
1126
1127 if (rfapiRaddr2Qprefix(&irfd->vn_addr,
1128 &nhp))
1129 continue;
1130
1131 bgp_withdraw(irfd->peer, p, /* prefix */
1132 0, /* addpath_id */
1133 NULL, /* attr, ignored */
1134 afi, SAFI_UNICAST,
1135 ZEBRA_ROUTE_VNC_DIRECT,
1136 BGP_ROUTE_REDISTRIBUTE,
1137 NULL, /* RD not used for
1138 unicast */
1139 NULL, 0, NULL); /* tag not
1140 used for
1141 unicast */
1142 }
1143 }
1144 }
1145 }
1146 }
1147
1148 static void vnc_direct_add_rn_group_rd(struct bgp *bgp,
1149 struct rfapi_nve_group_cfg *rfg,
1150 struct agg_node *rn, struct attr *attr,
1151 afi_t afi, struct rfapi_descriptor *irfd)
1152 {
1153 struct prefix nhp;
1154 struct bgp_path_info info;
1155 struct attr hattr;
1156 struct attr *iattr;
1157 const struct prefix *p = agg_node_get_prefix(rn);
1158
1159 if (irfd == NULL && rfg->type != RFAPI_GROUP_CFG_VRF) {
1160 /* need new rfapi_handle, for peer strcture
1161 * -- based on vnc_add_vrf_prefi */
1162 assert(rfg->rfd == NULL);
1163
1164 if (!rfg->rt_export_list || !rfg->rfapi_import_table) {
1165 vnc_zlog_debug_verbose(
1166 "%s: VRF \"%s\" is missing RT import/export configuration.",
1167 __func__, rfg->name);
1168 return;
1169 }
1170 if (!rfg->rd.prefixlen) {
1171 vnc_zlog_debug_verbose(
1172 "%s: VRF \"%s\" is missing RD configuration.",
1173 __func__, rfg->name);
1174 return;
1175 }
1176 if (rfg->label > MPLS_LABEL_MAX) {
1177 vnc_zlog_debug_verbose(
1178 "%s: VRF \"%s\" is missing default label configuration.",
1179 __func__, rfg->name);
1180 return;
1181 }
1182
1183 irfd = XCALLOC(MTYPE_RFAPI_DESC,
1184 sizeof(struct rfapi_descriptor));
1185 irfd->bgp = bgp;
1186 rfg->rfd = irfd;
1187 /*
1188 * leave most fields empty as will get from (dynamic) config
1189 * when needed
1190 */
1191 irfd->default_tunneltype_option.type = BGP_ENCAP_TYPE_MPLS;
1192 irfd->cookie = rfg;
1193 if (rfg->vn_prefix.family
1194 && !CHECK_FLAG(rfg->flags, RFAPI_RFG_VPN_NH_SELF)) {
1195 rfapiQprefix2Raddr(&rfg->vn_prefix, &irfd->vn_addr);
1196 } else {
1197 memset(&irfd->vn_addr, 0, sizeof(struct rfapi_ip_addr));
1198 irfd->vn_addr.addr_family = AF_INET;
1199 irfd->vn_addr.addr.v4 = bgp->router_id;
1200 }
1201 irfd->un_addr = irfd->vn_addr; /* sigh, need something in UN for
1202 lookups */
1203 vnc_zlog_debug_verbose("%s: Opening RFD for VRF %s", __func__,
1204 rfg->name);
1205 rfapi_init_and_open(bgp, irfd, rfg);
1206 }
1207
1208 if (irfd == NULL || rfapiRaddr2Qprefix(&irfd->vn_addr, &nhp))
1209 return;
1210
1211 /*
1212 * Construct new attribute set with NVE's VN
1213 * addr as
1214 * nexthop and without Tunnel Encap attr
1215 */
1216 if (encap_attr_export(&hattr, attr, &nhp, rn))
1217 return;
1218
1219 if (VNC_DEBUG(EXPORT_BGP_DIRECT_ADD)) {
1220 vnc_zlog_debug_any("%s: attr follows", __func__);
1221 rfapiPrintAttrPtrs(NULL, attr);
1222 vnc_zlog_debug_any("%s: hattr follows", __func__);
1223 rfapiPrintAttrPtrs(NULL, &hattr);
1224 }
1225
1226 if (rfg->routemap_export_bgp) {
1227 route_map_result_t ret;
1228
1229 info.peer = irfd->peer;
1230 info.attr = &hattr;
1231 ret = route_map_apply(rfg->routemap_export_bgp, p, &info);
1232 if (ret == RMAP_DENYMATCH) {
1233 bgp_attr_flush(&hattr);
1234 vnc_zlog_debug_verbose(
1235 "%s: route map says DENY, so not calling bgp_update",
1236 __func__);
1237 return;
1238 }
1239 }
1240
1241 if (VNC_DEBUG(EXPORT_BGP_DIRECT_ADD)) {
1242 vnc_zlog_debug_any("%s: hattr after route_map_apply:",
1243 __func__);
1244 rfapiPrintAttrPtrs(NULL, &hattr);
1245 }
1246 iattr = bgp_attr_intern(&hattr);
1247 bgp_attr_flush(&hattr);
1248
1249 bgp_update(irfd->peer, p, /* prefix */
1250 0, /* addpath_id */
1251 iattr, /* bgp_update copies it */
1252 afi, SAFI_UNICAST, ZEBRA_ROUTE_VNC_DIRECT,
1253 BGP_ROUTE_REDISTRIBUTE, NULL, /* RD not used for unicast */
1254 NULL, /* tag not used for unicast */
1255 0, 0, NULL); /* EVPN not used */
1256
1257 bgp_attr_unintern(&iattr);
1258
1259 return;
1260 }
1261
1262 /*
1263 * Caller is responsible for ensuring that the specified nve-group
1264 * is actually part of the list of exported nve groups.
1265 */
1266 static void vnc_direct_bgp_add_group_afi(struct bgp *bgp,
1267 struct rfapi_nve_group_cfg *rfg,
1268 afi_t afi)
1269 {
1270 struct agg_table *rt = NULL;
1271 struct agg_node *rn;
1272 struct attr attr = {0};
1273 struct rfapi_import_table *import_table;
1274
1275 vnc_zlog_debug_verbose("%s: entry", __func__);
1276
1277 import_table = rfg->rfapi_import_table;
1278 if (!import_table) {
1279 vnc_zlog_debug_verbose(
1280 "%s: import table not defined, returning", __func__);
1281 return;
1282 }
1283
1284 if (afi == AFI_IP || afi == AFI_IP6) {
1285 rt = import_table->imported_vpn[afi];
1286 } else {
1287 flog_err(EC_LIB_DEVELOPMENT, "%s: bad afi %d", __func__, afi);
1288 return;
1289 }
1290
1291 if (!rfg->nves && rfg->type != RFAPI_GROUP_CFG_VRF) {
1292 vnc_zlog_debug_verbose("%s: no NVEs in this group", __func__);
1293 return;
1294 }
1295
1296 bgp_attr_default_set(&attr, bgp, BGP_ORIGIN_INCOMPLETE);
1297 /* TBD set some configured med, see add_vnc_route() */
1298
1299 /*
1300 * Walk the NVE-Group's VNC Import table
1301 */
1302 for (rn = agg_route_top(rt); rn; rn = agg_route_next(rn)) {
1303
1304 if (rn->info) {
1305 const struct prefix *p = agg_node_get_prefix(rn);
1306 struct listnode *ln;
1307
1308 /*
1309 * per-nve-group prefix list check
1310 */
1311 if (rfg->plist_export_bgp[afi]) {
1312 if (prefix_list_apply(
1313 rfg->plist_export_bgp[afi], p)
1314 == PREFIX_DENY)
1315
1316 continue;
1317 }
1318 if (rfg->type == RFAPI_GROUP_CFG_VRF) {
1319 vnc_direct_add_rn_group_rd(bgp, rfg, rn, &attr,
1320 afi, rfg->rfd);
1321 /*
1322 * yuck!
1323 * - but consistent with rest of function
1324 */
1325 continue;
1326 }
1327 /*
1328 * For each NVE that is assigned to the export nve
1329 * group, generate
1330 * a route with that NVE as its next hop
1331 */
1332 for (ln = listhead(rfg->nves); ln;
1333 ln = listnextnode(ln)) {
1334 vnc_direct_add_rn_group_rd(bgp, rfg, rn, &attr,
1335 afi,
1336 listgetdata(ln));
1337 }
1338 }
1339 }
1340
1341 aspath_unintern(&attr.aspath);
1342 }
1343
1344
1345 /*
1346 * Caller is responsible for ensuring that the specified nve-group
1347 * is actually part of the list of exported nve groups.
1348 */
1349 void vnc_direct_bgp_add_group(struct bgp *bgp, struct rfapi_nve_group_cfg *rfg)
1350 {
1351 vnc_direct_bgp_add_group_afi(bgp, rfg, AFI_IP);
1352 vnc_direct_bgp_add_group_afi(bgp, rfg, AFI_IP6);
1353 }
1354
1355 static void vnc_direct_del_rn_group_rd(struct bgp *bgp,
1356 struct rfapi_nve_group_cfg *rfg,
1357 struct agg_node *rn, afi_t afi,
1358 struct rfapi_descriptor *irfd)
1359 {
1360 if (irfd == NULL)
1361 return;
1362
1363 bgp_withdraw(irfd->peer, agg_node_get_prefix(rn), /* prefix */
1364 0, /* addpath_id */
1365 NULL, /* attr, ignored */
1366 afi, SAFI_UNICAST, ZEBRA_ROUTE_VNC_DIRECT,
1367 BGP_ROUTE_REDISTRIBUTE, NULL, /* RD not used for unicast */
1368 NULL, 0, NULL); /* tag not used for unicast */
1369 return;
1370 }
1371
1372 /*
1373 * Caller is responsible for ensuring that the specified nve-group
1374 * was actually part of the list of exported nve groups.
1375 */
1376 static void vnc_direct_bgp_del_group_afi(struct bgp *bgp,
1377 struct rfapi_nve_group_cfg *rfg,
1378 afi_t afi)
1379 {
1380 struct agg_table *rt = NULL;
1381 struct agg_node *rn;
1382 struct rfapi_import_table *import_table;
1383
1384 vnc_zlog_debug_verbose("%s: entry", __func__);
1385
1386 import_table = rfg->rfapi_import_table;
1387 if (!import_table) {
1388 vnc_zlog_debug_verbose(
1389 "%s: import table not defined, returning", __func__);
1390 return;
1391 }
1392
1393 rt = import_table->imported_vpn[afi];
1394
1395 if (!rfg->nves && rfg->type != RFAPI_GROUP_CFG_VRF) {
1396 vnc_zlog_debug_verbose("%s: no NVEs in this group", __func__);
1397 return;
1398 }
1399
1400 /*
1401 * Walk the NVE-Group's VNC Import table
1402 */
1403 for (rn = agg_route_top(rt); rn; rn = agg_route_next(rn))
1404 if (rn->info) {
1405 if (rfg->type == RFAPI_GROUP_CFG_VRF)
1406 vnc_direct_del_rn_group_rd(bgp, rfg, rn, afi,
1407 rfg->rfd);
1408 else {
1409 struct listnode *ln;
1410
1411 /*
1412 * For each NVE that is assigned to the export
1413 * nve
1414 * group, generate
1415 * a route with that NVE as its next hop
1416 */
1417 for (ln = listhead(rfg->nves); ln;
1418 ln = listnextnode(ln))
1419 vnc_direct_del_rn_group_rd(
1420 bgp, rfg, rn, afi,
1421 listgetdata(ln));
1422 }
1423 }
1424 }
1425
1426 /*
1427 * Caller is responsible for ensuring that the specified nve-group
1428 * was actually part of the list of exported nve groups.
1429 */
1430 void vnc_direct_bgp_del_group(struct bgp *bgp, struct rfapi_nve_group_cfg *rfg)
1431 {
1432 vnc_direct_bgp_del_group_afi(bgp, rfg, AFI_IP);
1433 vnc_direct_bgp_del_group_afi(bgp, rfg, AFI_IP6);
1434 }
1435
1436 void vnc_direct_bgp_reexport_group_afi(struct bgp *bgp,
1437 struct rfapi_nve_group_cfg *rfg,
1438 afi_t afi)
1439 {
1440 struct listnode *node;
1441 struct rfapi_rfg_name *rfgn;
1442
1443 if (VNC_EXPORT_BGP_GRP_ENABLED(bgp->rfapi_cfg)) {
1444 /*
1445 * look in the list of currently-exported groups
1446 */
1447 for (ALL_LIST_ELEMENTS_RO(
1448 bgp->rfapi_cfg->rfg_export_direct_bgp_l, node,
1449 rfgn)) {
1450
1451 if (rfgn->rfg == rfg) {
1452 /*
1453 * If it matches, reexport it
1454 */
1455 vnc_direct_bgp_del_group_afi(bgp, rfg, afi);
1456 vnc_direct_bgp_add_group_afi(bgp, rfg, afi);
1457 break;
1458 }
1459 }
1460 }
1461 }
1462
1463
1464 static void vnc_direct_bgp_unexport_table(afi_t afi, struct agg_table *rt,
1465 struct list *nve_list)
1466 {
1467 if (nve_list) {
1468
1469 struct agg_node *rn;
1470
1471 for (rn = agg_route_top(rt); rn; rn = agg_route_next(rn)) {
1472
1473 if (rn->info) {
1474
1475 struct listnode *hln;
1476 struct rfapi_descriptor *irfd;
1477
1478 for (ALL_LIST_ELEMENTS_RO(nve_list, hln,
1479 irfd)) {
1480
1481 bgp_withdraw(irfd->peer,
1482 agg_node_get_prefix(rn),
1483 0, /* addpath_id */
1484 NULL, /* attr, ignored */
1485 afi, SAFI_UNICAST,
1486 ZEBRA_ROUTE_VNC_DIRECT,
1487 BGP_ROUTE_REDISTRIBUTE,
1488 NULL, /* RD not used for
1489 unicast */
1490 NULL, 0, NULL); /* tag not
1491 used for
1492 unicast,
1493 EVPN
1494 neither */
1495 }
1496 }
1497 }
1498 }
1499 }
1500
1501 static void import_table_to_nve_list_direct_bgp(struct bgp *bgp,
1502 struct rfapi_import_table *it,
1503 struct list **nves,
1504 uint8_t family)
1505 {
1506 struct listnode *node;
1507 struct rfapi_rfg_name *rfgn;
1508
1509 /*
1510 * Loop over the list of NVE-Groups configured for
1511 * exporting to direct-bgp.
1512 *
1513 * Build a list of NVEs that use this import table
1514 */
1515 *nves = NULL;
1516 for (ALL_LIST_ELEMENTS_RO(bgp->rfapi_cfg->rfg_export_direct_bgp_l, node,
1517 rfgn)) {
1518
1519 /*
1520 * If this NVE-Group's import table matches the current one
1521 */
1522 if (rfgn->rfg && rfgn->rfg->rfapi_import_table == it) {
1523 if (rfgn->rfg->nves)
1524 nve_group_to_nve_list(rfgn->rfg, nves, family);
1525 else if (rfgn->rfg->rfd
1526 && rfgn->rfg->type == RFAPI_GROUP_CFG_VRF) {
1527 if (!*nves)
1528 *nves = list_new();
1529 listnode_add(*nves, rfgn->rfg->rfd);
1530 }
1531 }
1532 }
1533 }
1534
1535 void vnc_direct_bgp_vpn_enable(struct bgp *bgp, afi_t afi)
1536 {
1537 struct listnode *rfgn;
1538 struct rfapi_nve_group_cfg *rfg;
1539
1540 if (!bgp)
1541 return;
1542
1543 if (!VNC_EXPORT_BGP_GRP_ENABLED(bgp->rfapi_cfg)) {
1544 vnc_zlog_debug_verbose(
1545 "%s: export-to-bgp group mode not enabled, skipping",
1546 __func__);
1547 return;
1548 }
1549
1550 if (afi != AFI_IP && afi != AFI_IP6) {
1551 vnc_zlog_debug_verbose("%s: bad afi: %d", __func__, afi);
1552 return;
1553 }
1554
1555 /*
1556 * Policy is applied per-nve-group, so we need to iterate
1557 * over the groups to add everything.
1558 */
1559 for (ALL_LIST_ELEMENTS_RO(bgp->rfapi_cfg->nve_groups_sequential, rfgn,
1560 rfg)) {
1561
1562 /*
1563 * contains policy management
1564 */
1565 vnc_direct_bgp_add_group_afi(bgp, rfg, afi);
1566 }
1567 }
1568
1569
1570 void vnc_direct_bgp_vpn_disable(struct bgp *bgp, afi_t afi)
1571 {
1572 struct rfapi_import_table *it;
1573 uint8_t family = afi2family(afi);
1574
1575 vnc_zlog_debug_verbose("%s: entry, afi=%d", __func__, afi);
1576
1577 if (!bgp)
1578 return;
1579
1580 if (!bgp->rfapi) {
1581 vnc_zlog_debug_verbose("%s: rfapi not initialized", __func__);
1582 return;
1583 }
1584
1585 if (!family || (afi != AFI_IP && afi != AFI_IP6)) {
1586 vnc_zlog_debug_verbose("%s: bad afi: %d", __func__, afi);
1587 return;
1588 }
1589
1590 for (it = bgp->rfapi->imports; it; it = it->next) {
1591
1592 struct list *nve_list = NULL;
1593
1594 import_table_to_nve_list_direct_bgp(bgp, it, &nve_list, family);
1595
1596 if (nve_list) {
1597 vnc_direct_bgp_unexport_table(
1598 afi, it->imported_vpn[afi], nve_list);
1599 list_delete(&nve_list);
1600 }
1601 }
1602 }
1603
1604
1605 /***********************************************************************
1606 * Export methods that proxy nexthop END
1607 ***********************************************************************/
1608
1609
1610 /***********************************************************************
1611 * Export methods that preserve original nexthop BEGIN
1612 * rh = "registering nve"
1613 ***********************************************************************/
1614
1615
1616 /*
1617 * "Adding a Route" export process
1618 * TBD do we need to check bpi->type and bpi->sub_type here, or does
1619 * caller do it?
1620 */
1621 void vnc_direct_bgp_rh_add_route(struct bgp *bgp, afi_t afi,
1622 const struct prefix *prefix, struct peer *peer,
1623 struct attr *attr)
1624 {
1625 struct vnc_export_info *eti;
1626 struct attr hattr;
1627 struct rfapi_cfg *hc;
1628 struct attr *iattr;
1629
1630 if (!afi) {
1631 flog_err(EC_LIB_DEVELOPMENT, "%s: can't get afi of route node",
1632 __func__);
1633 return;
1634 }
1635
1636 /* check bgp redist flag for vnc direct ("vpn") routes */
1637 if (!bgp->redist[afi][ZEBRA_ROUTE_VNC_DIRECT]) {
1638 vnc_zlog_debug_verbose(
1639 "%s: bgp redistribution of VNC direct routes is off",
1640 __func__);
1641 return;
1642 }
1643
1644 if (!(hc = bgp->rfapi_cfg)) {
1645 vnc_zlog_debug_verbose("%s: bgp->rfapi_cfg is NULL, skipping",
1646 __func__);
1647 return;
1648 }
1649
1650 if (!VNC_EXPORT_BGP_RH_ENABLED(bgp->rfapi_cfg)) {
1651 vnc_zlog_debug_verbose(
1652 "%s: export-to-bgp RH mode not enabled, skipping",
1653 __func__);
1654 return;
1655 }
1656
1657 /*
1658 * prefix list check
1659 */
1660 if (hc->plist_export_bgp[afi]) {
1661 if (prefix_list_apply(hc->plist_export_bgp[afi], prefix)
1662 == PREFIX_DENY)
1663 return;
1664 }
1665
1666 /*
1667 * Construct new attribute set with NVE's VN addr as
1668 * nexthop and without Tunnel Encap attr
1669 */
1670 if (encap_attr_export(&hattr, attr, NULL, NULL))
1671 return;
1672 if (hc->routemap_export_bgp) {
1673 struct bgp_path_info info;
1674 route_map_result_t ret;
1675
1676 memset(&info, 0, sizeof(info));
1677 info.peer = peer;
1678 info.attr = &hattr;
1679 ret = route_map_apply(hc->routemap_export_bgp, prefix, &info);
1680 if (ret == RMAP_DENYMATCH) {
1681 bgp_attr_flush(&hattr);
1682 return;
1683 }
1684 }
1685
1686 iattr = bgp_attr_intern(&hattr);
1687 bgp_attr_flush(&hattr);
1688
1689 /*
1690 * record route information that we will need to expire
1691 * this route
1692 */
1693 eti = vnc_eti_get(bgp, EXPORT_TYPE_BGP, prefix, peer,
1694 ZEBRA_ROUTE_VNC_DIRECT_RH, BGP_ROUTE_REDISTRIBUTE);
1695 rfapiGetVncLifetime(attr, &eti->lifetime);
1696 eti->lifetime = rfapiGetHolddownFromLifetime(eti->lifetime);
1697
1698 /*
1699 * export expiration timer is already running on
1700 * this route: cancel it
1701 */
1702 THREAD_OFF(eti->timer);
1703
1704 bgp_update(peer, prefix, /* prefix */
1705 0, /* addpath_id */
1706 iattr, /* bgp_update copies this attr */
1707 afi, SAFI_UNICAST, ZEBRA_ROUTE_VNC_DIRECT_RH,
1708 BGP_ROUTE_REDISTRIBUTE, NULL, /* RD not used for unicast */
1709 NULL, /* tag not used for unicast, EVPN neither */
1710 0, 0, NULL); /* EVPN not used */
1711 bgp_attr_unintern(&iattr);
1712 }
1713
1714 static void vncExportWithdrawTimer(struct thread *t)
1715 {
1716 struct vnc_export_info *eti = THREAD_ARG(t);
1717 const struct prefix *p = agg_node_get_prefix(eti->node);
1718
1719 /*
1720 * withdraw the route
1721 */
1722 bgp_withdraw(eti->peer, p, 0, /* addpath_id */
1723 NULL, /* attr, ignored */
1724 family2afi(p->family), SAFI_UNICAST, eti->type,
1725 eti->subtype, NULL, /* RD not used for unicast */
1726 NULL, 0,
1727 NULL); /* tag not used for unicast, EVPN neither */
1728
1729 /*
1730 * Free the eti
1731 */
1732 vnc_eti_delete(eti);
1733 }
1734
1735 /*
1736 * "Withdrawing a Route" export process
1737 * TBD do we need to check bpi->type and bpi->sub_type here, or does
1738 * caller do it?
1739 */
1740 void vnc_direct_bgp_rh_del_route(struct bgp *bgp, afi_t afi,
1741 const struct prefix *prefix, struct peer *peer)
1742 {
1743 struct vnc_export_info *eti;
1744
1745 if (!afi) {
1746 flog_err(EC_LIB_DEVELOPMENT, "%s: can't get afi route node",
1747 __func__);
1748 return;
1749 }
1750
1751 /* check bgp redist flag for vnc direct ("vpn") routes */
1752 if (!bgp->redist[afi][ZEBRA_ROUTE_VNC_DIRECT]) {
1753 vnc_zlog_debug_verbose(
1754 "%s: bgp redistribution of VNC direct routes is off",
1755 __func__);
1756 return;
1757 }
1758
1759 if (!bgp->rfapi_cfg) {
1760 vnc_zlog_debug_verbose("%s: bgp->rfapi_cfg is NULL, skipping",
1761 __func__);
1762 return;
1763 }
1764 if (!VNC_EXPORT_BGP_RH_ENABLED(bgp->rfapi_cfg)) {
1765 vnc_zlog_debug_verbose(
1766 "%s: export-to-bgp group mode not enabled, skipping",
1767 __func__);
1768 return;
1769 }
1770
1771 eti = vnc_eti_get(bgp, EXPORT_TYPE_BGP, prefix, peer,
1772 ZEBRA_ROUTE_VNC_DIRECT_RH, BGP_ROUTE_REDISTRIBUTE);
1773
1774 if (!eti->timer && eti->lifetime <= INT32_MAX) {
1775 eti->timer = NULL;
1776 thread_add_timer(bm->master, vncExportWithdrawTimer, eti,
1777 eti->lifetime, &eti->timer);
1778 vnc_zlog_debug_verbose(
1779 "%s: set expiration timer for %u seconds", __func__,
1780 eti->lifetime);
1781 }
1782 }
1783
1784
1785 void vnc_direct_bgp_rh_vpn_enable(struct bgp *bgp, afi_t afi)
1786 {
1787 struct prefix_rd prd;
1788 struct bgp_dest *pdest;
1789 struct rfapi_cfg *hc;
1790
1791 vnc_zlog_debug_verbose("%s: entry, afi=%d", __func__, afi);
1792
1793 if (!bgp)
1794 return;
1795
1796 if (!(hc = bgp->rfapi_cfg))
1797 return;
1798
1799 if (!VNC_EXPORT_BGP_RH_ENABLED(bgp->rfapi_cfg)) {
1800 vnc_zlog_debug_verbose(
1801 "%s: export of RH routes not enabled, skipping",
1802 __func__);
1803 return;
1804 }
1805
1806 if (afi != AFI_IP && afi != AFI_IP6) {
1807 vnc_zlog_debug_verbose("%s: bad afi: %d", __func__, afi);
1808 return;
1809 }
1810
1811 /*
1812 * Go through the entire BGP VPN table and export to BGP unicast.
1813 */
1814
1815 vnc_zlog_debug_verbose("%s: starting RD loop", __func__);
1816
1817 /* Loop over all the RDs */
1818 for (pdest = bgp_table_top(bgp->rib[afi][SAFI_MPLS_VPN]); pdest;
1819 pdest = bgp_route_next(pdest)) {
1820
1821 struct bgp_table *table;
1822 struct bgp_dest *dest;
1823 struct bgp_path_info *ri;
1824 const struct prefix *pdest_p = bgp_dest_get_prefix(pdest);
1825
1826 memset(&prd, 0, sizeof(prd));
1827 prd.family = AF_UNSPEC;
1828 prd.prefixlen = 64;
1829 memcpy(prd.val, pdest_p->u.val, 8);
1830
1831 /* This is the per-RD table of prefixes */
1832 table = bgp_dest_get_bgp_table_info(pdest);
1833
1834 if (!table)
1835 continue;
1836
1837 for (dest = bgp_table_top(table); dest;
1838 dest = bgp_route_next(dest)) {
1839 const struct prefix *dest_p;
1840
1841 /*
1842 * skip prefix list check if no routes here
1843 */
1844 if (!bgp_dest_has_bgp_path_info_data(dest))
1845 continue;
1846
1847 vnc_zlog_debug_verbose("%s: checking prefix %pBD",
1848 __func__, dest);
1849
1850 dest_p = bgp_dest_get_prefix(dest);
1851
1852 /*
1853 * prefix list check
1854 */
1855 if (hc->plist_export_bgp[afi]) {
1856 if (prefix_list_apply(hc->plist_export_bgp[afi],
1857 dest_p)
1858 == PREFIX_DENY) {
1859
1860 vnc_zlog_debug_verbose(
1861 "%s: prefix list says DENY",
1862 __func__);
1863 continue;
1864 }
1865 }
1866
1867 for (ri = bgp_dest_get_bgp_path_info(dest); ri;
1868 ri = ri->next) {
1869
1870 vnc_zlog_debug_verbose("%s: ri->sub_type: %d",
1871 __func__, ri->sub_type);
1872
1873 if (ri->sub_type == BGP_ROUTE_NORMAL
1874 || ri->sub_type == BGP_ROUTE_RFP) {
1875
1876 struct vnc_export_info *eti;
1877 struct attr hattr;
1878 struct attr *iattr;
1879
1880 /*
1881 * Construct new attribute set with
1882 * NVE's VN addr as
1883 * nexthop and without Tunnel Encap attr
1884 */
1885 if (encap_attr_export(&hattr, ri->attr,
1886 NULL, NULL)) {
1887 vnc_zlog_debug_verbose(
1888 "%s: encap_attr_export failed",
1889 __func__);
1890 continue;
1891 }
1892
1893 if (hc->routemap_export_bgp) {
1894 struct bgp_path_info info;
1895 route_map_result_t ret;
1896
1897 memset(&info, 0, sizeof(info));
1898 info.peer = ri->peer;
1899 info.attr = &hattr;
1900 ret = route_map_apply(
1901 hc->routemap_export_bgp,
1902 dest_p, &info);
1903 if (ret == RMAP_DENYMATCH) {
1904 bgp_attr_flush(&hattr);
1905 vnc_zlog_debug_verbose(
1906 "%s: route map says DENY",
1907 __func__);
1908 continue;
1909 }
1910 }
1911
1912 iattr = bgp_attr_intern(&hattr);
1913 bgp_attr_flush(&hattr);
1914
1915 /*
1916 * record route information that we will
1917 * need to expire
1918 * this route
1919 */
1920 eti = vnc_eti_get(
1921 bgp, EXPORT_TYPE_BGP, dest_p,
1922 ri->peer,
1923 ZEBRA_ROUTE_VNC_DIRECT_RH,
1924 BGP_ROUTE_REDISTRIBUTE);
1925 rfapiGetVncLifetime(ri->attr,
1926 &eti->lifetime);
1927
1928 /*
1929 * export expiration timer is
1930 * already running on
1931 * this route: cancel it
1932 */
1933 THREAD_OFF(eti->timer);
1934
1935 vnc_zlog_debug_verbose(
1936 "%s: calling bgp_update",
1937 __func__);
1938
1939 bgp_update(
1940 ri->peer, dest_p, /* prefix */
1941 0, /* addpath_id */
1942 iattr, /* bgp_update copies
1943 it */
1944 AFI_IP, SAFI_UNICAST,
1945 ZEBRA_ROUTE_VNC_DIRECT_RH,
1946 BGP_ROUTE_REDISTRIBUTE, NULL,
1947 /* RD not used for unicast */
1948 NULL,
1949 /* tag not used for unicast,
1950 or EVPN */
1951 0, 0, NULL); /* EVPN not used */
1952
1953 bgp_attr_unintern(&iattr);
1954 }
1955 }
1956 }
1957 }
1958 }
1959
1960 void vnc_direct_bgp_rh_vpn_disable(struct bgp *bgp, afi_t afi)
1961 {
1962 struct bgp_dest *dest;
1963
1964 vnc_zlog_debug_verbose("%s: entry, afi=%d", __func__, afi);
1965
1966 if (!bgp)
1967 return;
1968
1969 if (afi != AFI_IP && afi != AFI_IP6) {
1970 vnc_zlog_debug_verbose("%s: bad afi: %d", __func__, afi);
1971 return;
1972 }
1973
1974 /*
1975 * Go through the entire BGP unicast table and remove routes that
1976 * originated from us
1977 */
1978 for (dest = bgp_table_top(bgp->rib[afi][SAFI_UNICAST]); dest;
1979 dest = bgp_route_next(dest)) {
1980 const struct prefix *dest_p = bgp_dest_get_prefix(dest);
1981 struct bgp_path_info *ri;
1982 struct bgp_path_info *next;
1983
1984 for (ri = bgp_dest_get_bgp_path_info(dest), next = NULL; ri;
1985 ri = next) {
1986
1987 next = ri->next;
1988
1989 if (ri->type == ZEBRA_ROUTE_VNC_DIRECT_RH
1990 && ri->sub_type == BGP_ROUTE_REDISTRIBUTE) {
1991
1992 struct vnc_export_info *eti;
1993
1994 /*
1995 * Delete routes immediately (no timer)
1996 */
1997 eti = vnc_eti_checktimer(
1998 bgp, EXPORT_TYPE_BGP, dest_p, ri->peer,
1999 ZEBRA_ROUTE_VNC_DIRECT_RH,
2000 BGP_ROUTE_REDISTRIBUTE);
2001 if (eti) {
2002 THREAD_OFF(eti->timer);
2003 vnc_eti_delete(eti);
2004 }
2005
2006 bgp_withdraw(ri->peer, dest_p, /* prefix */
2007 0, /* addpath_id */
2008 NULL, /* ignored */
2009 AFI_IP, SAFI_UNICAST,
2010 ZEBRA_ROUTE_VNC_DIRECT_RH,
2011 BGP_ROUTE_REDISTRIBUTE,
2012 NULL, /* RD not used for unicast */
2013 NULL, 0, NULL); /* tag not used for
2014 unicast, EVPN
2015 neither */
2016 }
2017 }
2018 }
2019 }
2020
2021 void vnc_direct_bgp_rh_reexport(struct bgp *bgp, afi_t afi)
2022 {
2023 if (VNC_EXPORT_BGP_RH_ENABLED(bgp->rfapi_cfg)) {
2024 vnc_direct_bgp_rh_vpn_disable(bgp, afi);
2025 vnc_direct_bgp_rh_vpn_enable(bgp, afi);
2026 }
2027 }
2028
2029 /***********************************************************************
2030 * Generic Export methods
2031 ***********************************************************************/
2032
2033 /*
2034 * Assumes the correct mode bits are already turned on. Thus it
2035 * is OK to call this function from, e.g., bgp_redistribute_set()
2036 * without caring if export is enabled or not
2037 */
2038 void vnc_export_bgp_enable(struct bgp *bgp, afi_t afi)
2039 {
2040 if (!bgp->rfapi_cfg)
2041 return;
2042
2043 switch (bgp->rfapi_cfg->flags & BGP_VNC_CONFIG_EXPORT_BGP_MODE_BITS) {
2044 case BGP_VNC_CONFIG_EXPORT_BGP_MODE_NONE:
2045 break;
2046
2047 case BGP_VNC_CONFIG_EXPORT_BGP_MODE_GRP:
2048 vnc_direct_bgp_vpn_enable(bgp, afi);
2049 break;
2050
2051 case BGP_VNC_CONFIG_EXPORT_BGP_MODE_RH:
2052 vnc_direct_bgp_rh_vpn_enable(bgp, afi);
2053 break;
2054
2055 case BGP_VNC_CONFIG_EXPORT_BGP_MODE_CE:
2056 vnc_direct_bgp_vpn_enable_ce(bgp, afi);
2057 break;
2058 }
2059 }
2060
2061 void vnc_export_bgp_disable(struct bgp *bgp, afi_t afi)
2062 {
2063 if (!bgp->rfapi_cfg)
2064 return;
2065
2066 switch (bgp->rfapi_cfg->flags & BGP_VNC_CONFIG_EXPORT_BGP_MODE_BITS) {
2067 case BGP_VNC_CONFIG_EXPORT_BGP_MODE_NONE:
2068 break;
2069
2070 case BGP_VNC_CONFIG_EXPORT_BGP_MODE_GRP:
2071 vnc_direct_bgp_vpn_disable(bgp, afi);
2072 break;
2073
2074 case BGP_VNC_CONFIG_EXPORT_BGP_MODE_RH:
2075 vnc_direct_bgp_rh_vpn_disable(bgp, afi);
2076 break;
2077
2078 case BGP_VNC_CONFIG_EXPORT_BGP_MODE_CE:
2079 vnc_direct_bgp_vpn_disable_ce(bgp, afi);
2080 break;
2081 }
2082 }
2083
2084 void vnc_export_bgp_prechange(struct bgp *bgp)
2085 {
2086 vnc_export_bgp_disable(bgp, AFI_IP);
2087 vnc_export_bgp_disable(bgp, AFI_IP6);
2088 }
2089
2090 void vnc_export_bgp_postchange(struct bgp *bgp)
2091 {
2092 vnc_export_bgp_enable(bgp, AFI_IP);
2093 vnc_export_bgp_enable(bgp, AFI_IP6);
2094 }
2095
2096 void vnc_direct_bgp_reexport(struct bgp *bgp, afi_t afi)
2097 {
2098 vnc_export_bgp_disable(bgp, afi);
2099 vnc_export_bgp_enable(bgp, afi);
2100 }