vxlan-and-evpn.adoc

   1
   2 ////
   3
   4 This is currently not included, because
   5 - it requires ifupdown2
   6 - routing needs more documentation
   7
   8 ////
   9
  10
  11 VXLAN layer2 with vlan unware linux bridges
  12 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  13
  14 VXLAN is an overlay network to carry Ethernet traffic over an existing IP network
  15 while accommodating a very large number of tenants. It is defined in RFC 7348.
  16 Each overlay network is known as a VXLAN Segment and identified by a unique
  17 24-bit segment ID called a VXLAN Network Identifier (VNI).
  18
  19 For BUM traffic (broadcast / unknown unicast traffic, multicast),
  20 we have 3 differents vxlan setup modes : multicast, unicast, bgp-evpn
  21
  22 image::images/vxlan-l2-vlanunaware.svg["vxlan l2 bridge vlan unaware",align="center"]
  23
  24 multicast mode
  25 ^^^^^^^^^^^^^^
  26
  27 This scenario relies in head end replication, meaning that end host in case
  28 of not having any entry for the destination MAC address will send out an ARP
  29 to other devices / VTEPs in the VXLAN network.
  30 This is done by sending the request to the VXLAN multicast group,
  31 remote VTEPs will get the packet and answer accordingly direct to the originating VTEP.
  32
  33
  34 * node1
  35
  36 ----
  37 auto eno1
  38 iface eno1 inet manual
  39
  40 auto vmbr0
  41 iface vmbr0 inet static
  42         address 192.168.0.1
  43         netmask  255.255.255.0
  44         bridge_ports eno1
  45         bridge_stp off
  46         bridge_fd 0
  47
  48 auto vxlan2
  49 iface vxlan2 inet manual
  50         vxlan-svcnodeip 225.20.1.1
  51         vxlan-physdev eno1
  52
  53 auto vmbr2
  54 iface vmbr2 inet manual
  55         bridge_ports vxlan2
  56         bridge_stp off
  57         bridge_fd 0
  58
  59 auto vxlan3
  60 iface vxlan3 inet manual
  61         vxlan-svcnodeip 225.20.1.1
  62         vxlan-physdev eno1
  63
  64 auto vmbr3
  65 iface vmbr3 inet manual
  66         bridge_ports vxlan3
  67         bridge_stp off
  68         bridge_fd 0
  69 ----
  70
  71
  72 * node2
  73
  74 ----
  75 auto eno1
  76 iface eno1 inet manual
  77
  78 auto vmbr0
  79 iface vmbr0 inet static
  80         address 192.168.0.2
  81         netmask  255.255.255.0
  82         bridge_ports eno1
  83         bridge_stp off
  84         bridge_fd 0
  85
  86 auto vxlan2
  87 iface vxlan2 inet manual
  88         vxlan-svcnodeip 225.20.1.1
  89         vxlan-physdev eno1
  90
  91 auto vmbr2
  92 iface vmbr2 inet manual
  93         bridge_ports vxlan2
  94         bridge_stp off
  95         bridge_fd 0
  96
  97
  98 auto vxlan3
  99 iface vxlan3 inet manual
 100         vxlan-svcnodeip 225.20.1.1
 101         vxlan-physdev eno1
 102
 103 auto vmbr3
 104 iface vmbr3 inet manual
 105         bridge_ports vxlan3
 106         bridge_stp off
 107         bridge_fd 0
 108 ----
 109
 110
 111 * node3
 112
 113 ----
 114 auto eno1
 115 iface eno1 inet manual
 116
 117 auto vmbr0
 118 iface vmbr0 inet static
 119         address 192.168.0.3
 120         netmask  255.255.255.0
 121         bridge_ports eno1
 122         bridge_stp off
 123         bridge_fd 0
 124
 125 auto vxlan2
 126 iface vxlan2 inet manual
 127         vxlan-svcnodeip 225.20.1.1
 128         vxlan-physdev eno1
 129
 130 auto vmbr2
 131 iface vmbr2 inet manual
 132         bridge_ports vxlan2
 133         bridge_stp off
 134         bridge_fd 0
 135
 136
 137 auto vxlan3
 138 iface vxlan3 inet manual
 139         vxlan-svcnodeip 225.20.1.1
 140         vxlan-physdev eno1
 141
 142 auto vmbr3
 143 iface vmbr3 inet manual
 144         bridge_ports vxlan3
 145         bridge_stp off
 146         bridge_fd 0
 147 ----
 148
 149
 150 unicast mode
 151 ^^^^^^^^^^^^
 152
 153 We can replace multicast by head-end replication of BUM frames to a statically configured lists of remote VTEPs.
 154 The VXLAN is defined without a remote multicast group.
 155 Instead, all the remote VTEPs are associated with the all-zero address:
 156 a BUM frame will be duplicated to all these destinations.
 157 The VXLAN device will still learn remote addresses automatically using source-address learning.
 158
 159 * node1
 160
 161 ----
 162 auto eno1
 163 iface eno1 inet manual
 164
 165 auto vmbr0
 166 iface vmbr0 inet static
 167         address 192.168.0.1
 168         netmask  255.255.255.0
 169         bridge_ports eno1
 170         bridge_stp off
 171         bridge_fd 0
 172
 173
 174 auto vxlan2
 175 iface vxlan2 inet manual
 176         vxlan_remoteip 192.168.0.2
 177         vxlan_remoteip 192.168.0.3
 178
 179
 180 auto vmbr2
 181 iface vmbr2 inet manual
 182         bridge_ports vxlan2
 183         bridge_stp off
 184         bridge_fd 0
 185
 186
 187 auto vxlan3
 188 iface vxlan2 inet manual
 189         vxlan_remoteip 192.168.0.2
 190         vxlan_remoteip 192.168.0.3
 191
 192
 193 auto vmbr3
 194 iface vmbr3 inet manual
 195         bridge_ports vxlan3
 196         bridge_stp off
 197         bridge_fd 0
 198 ----
 199
 200
 201 * node2
 202
 203 ----
 204 auto eno1
 205 iface eno1 inet manual
 206
 207 auto vmbr0
 208 iface vmbr0 inet static
 209         address 192.168.0.2
 210         netmask  255.255.255.0
 211         bridge_ports eno1
 212         bridge_stp off
 213         bridge_fd 0
 214
 215 auto vxlan2
 216 iface vxlan2 inet manual
 217         vxlan_remoteip 192.168.0.1
 218         vxlan_remoteip 192.168.0.3
 219
 220
 221
 222 auto vmbr2
 223 iface vmbr2 inet manual
 224         bridge_ports vxlan2
 225         bridge_stp off
 226         bridge_fd 0
 227
 228 auto vxlan3
 229 iface vxlan2 inet manual
 230         vxlan_remoteip 192.168.0.1
 231         vxlan_remoteip 192.168.0.3
 232
 233
 234 auto vmbr3
 235 iface vmbr3 inet manual
 236         bridge_ports vxlan3
 237         bridge_stp off
 238         bridge_fd 0
 239 ----
 240
 241
 242 * node3
 243
 244 ----
 245 auto eno1
 246 iface eno1 inet manual
 247
 248 auto vmbr0
 249 iface vmbr0 inet static
 250         address 192.168.0.3
 251         netmask  255.255.255.0
 252         bridge_ports eno1
 253         bridge_stp off
 254         bridge_fd 0
 255
 256 auto vxlan2
 257 iface vxlan2 inet manual
 258         vxlan_remoteip 192.168.0.2
 259         vxlan_remoteip 192.168.0.3
 260
 261
 262
 263 auto vmbr2
 264 iface vmbr2 inet manual
 265         bridge_ports vxlan2
 266         bridge_stp off
 267         bridge_fd 0
 268
 269 auto vxlan3
 270 iface vxlan2 inet manual
 271         vxlan_remoteip 192.168.0.2
 272         vxlan_remoteip 192.168.0.3
 273
 274
 275 auto vmbr3
 276 iface vmbr3 inet manual
 277         bridge_ports vxlan3
 278         bridge_stp off
 279         bridge_fd 0
 280 ----
 281
 282
 283 bgp-evpn
 284 ^^^^^^^^
 285
 286 VTEPs use control plane learning/distribution via BGP for remote MAC addresses instead of data plane learning.
 287 VTEPs have the ability to suppress ARP flooding over VXLAN tunnels.
 288
 289 The control plane used here is FRR, a bgp routing software.
 290 Each node in the proxmox cluster peer with each others nodes.
 291 For bigger networks, or multiple proxmox clusters,
 292 it's possible to use external bgp route reflector servers.
 293
 294 * node1
 295
 296 ----
 297 auto eno1
 298 iface eno1 inet manual
 299
 300 auto vmbr0
 301 iface vmbr0 inet static
 302         address 192.168.0.1
 303         netmask 255.255.255.0
 304         bridge_ports eno1
 305         bridge_stp off
 306         bridge_fd 0
 307
 308 auto vxlan2
 309 iface vxlan2 inet manual
 310         vxlan-local-tunnelip 192.168.0.1
 311         bridge-learning off
 312         bridge-arp-nd-suppress on
 313         bridge-unicast-flood off
 314         bridge-multicast-flood off
 315
 316
 317 auto vmbr2
 318 iface vmbr2 inet manual
 319         bridge_ports vxlan2
 320         bridge_stp off
 321         bridge_fd 0
 322
 323
 324 auto vxlan3
 325 iface vxlan3 inet manual
 326         vxlan-local-tunnelip 192.168.0.1
 327         bridge-learning off
 328         bridge-arp-nd-suppress on
 329         bridge-unicast-flood off
 330         bridge-multicast-flood off
 331
 332
 333 auto vmbr3
 334 iface vmbr3 inet manual
 335         bridge_ports vxlan3
 336         bridge_stp off
 337         bridge_fd 0
 338 ----
 339
 340
 341 /etc/frr/frr.conf
 342
 343 ----
 344 router bgp 1234
 345  no bgp default ipv4-unicast
 346  coalesce-time 1000
 347  neighbor 192.168.0.2 remote-as 1234
 348  neighbor 192.168.0.3 remote-as 1234
 349  !
 350  address-family l2vpn evpn
 351   neighbor 192.168.0.2 activate
 352   neighbor 192.168.0.3 activate
 353   advertise-all-vni
 354  exit-address-family
 355 !
 356 line vty
 357 !
 358 ----
 359
 360
 361 * node2
 362
 363 ----
 364 auto eno1
 365 iface eno1 inet manual
 366
 367 auto vmbr0
 368 iface vmbr0 inet static
 369         address 192.168.0.2
 370         netmask 255.255.255.0
 371         bridge_ports eno1
 372         bridge_stp off
 373         bridge_fd 0
 374
 375 auto vxlan2
 376 iface vxlan2 inet manual
 377         vxlan-local-tunnelip 192.168.0.2
 378         bridge-learning off
 379         bridge-arp-nd-suppress on
 380         bridge-unicast-flood off
 381         bridge-multicast-flood off
 382
 383
 384 auto vmbr2
 385 iface vmbr2 inet manual
 386         bridge_ports vxlan2
 387         bridge_stp off
 388         bridge_fd 0
 389
 390 auto vxlan3
 391 iface vxlan3 inet manual
 392         vxlan-local-tunnelip 192.168.0.2
 393         bridge-learning off
 394         bridge-arp-nd-suppress on
 395         bridge-unicast-flood off
 396         bridge-multicast-flood off
 397
 398
 399 auto vmbr3
 400 iface vmbr3 inet manual
 401         bridge_ports vxlan3
 402         bridge_stp off
 403         bridge_fd 0
 404 ----
 405
 406
 407 /etc/frr/frr.conf
 408
 409 ----
 410 router bgp 1234
 411  no bgp default ipv4-unicast
 412  coalesce-time 1000
 413  neighbor 192.168.0.1 remote-as 1234
 414  neighbor 192.168.0.3 remote-as 1234
 415  !
 416  address-family l2vpn evpn
 417   neighbor 192.168.0.1 activate
 418   neighbor 192.168.0.3 activate
 419   advertise-all-vni
 420  exit-address-family
 421 !
 422 line vty
 423 !
 424 ----
 425
 426
 427 * node3
 428
 429 ----
 430 auto eno1
 431 iface eno1 inet manual
 432
 433 auto vmbr0
 434 iface vmbr0 inet static
 435         address 192.168.0.2
 436         netmask 255.255.255.0
 437         bridge_ports eno1
 438         bridge_stp off
 439         bridge_fd 0
 440
 441 auto vxlan2
 442 iface vxlan2 inet manual
 443         vxlan-local-tunnelip 192.168.0.3
 444         bridge-learning off
 445         bridge-arp-nd-suppress on
 446         bridge-unicast-flood off
 447         bridge-multicast-flood off
 448
 449
 450 auto vmbr2
 451 iface vmbr2 inet manual
 452         bridge_ports vxlan2
 453         bridge_stp off
 454         bridge_fd 0
 455
 456 auto vxlan3
 457 iface vxlan3 inet manual
 458         vxlan-local-tunnelip 192.168.0.3
 459         bridge-learning off
 460         bridge-arp-nd-suppress on
 461         bridge-unicast-flood off
 462         bridge-multicast-flood off
 463
 464
 465 auto vmbr3
 466 iface vmbr3 inet manual
 467         bridge_ports vxlan3
 468         bridge_stp off
 469         bridge_fd 0
 470 ----
 471
 472
 473 /etc/frr/frr.conf
 474
 475
 476 ----
 477 router bgp 1234
 478  no bgp default ipv4-unicast
 479  coalesce-time 1000
 480  neighbor 192.168.0.1 remote-as 1234
 481  neighbor 192.168.0.2 remote-as 1234
 482  !
 483  address-family l2vpn evpn
 484   neighbor 192.168.0.1 activate
 485   neighbor 192.168.0.2 activate
 486   advertise-all-vni
 487  exit-address-family
 488 !
 489 line vty
 490 !
 491 ----
 492
 493 VXLAN layer3 routing with anycast gateway
 494 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 495
 496 With this need, each vmbr bridge will be the gateway for the vm.
 497 Same vmbr on different node, will have same ip address and same mac address,
 498 to have working vm live migration and no network disruption.
 499
 500 VXLAN layer3 routing only work with FRR and non-aware bridge.
 501 (vlan aware bridge support is buggy currently).
 502
 503
 504 sysctl.conf tuning
 505
 506 ----
 507 #enable routing
 508 net.ipv4.ip_forward=1
 509 net.ipv6.conf.all.forwarding=1
 510 #disable reverse path filtering
 511 net.ipv4.conf.default.rp_filter=0
 512 net.ipv4.conf.all.rp_filter=0
 513 #allow frr to work with vrf
 514 net.ipv4.tcp_l3mdev_accept=1
 515 ----
 516
 517 asymmetric model
 518 ^^^^^^^^^^^^^^^^
 519
 520 This is the simplest mode. To get it work, all vxlan need to be defined on all nodes.
 521
 522 The asymmetric model allows routing and bridging on the VXLAN tunnel ingress,
 523 but only bridging on the egress.
 524 This results in bi-directional VXLAN traffic traveling on different VNIs
 525 in each direction (always the destination VNI) across the routed infrastructure.
 526
 527 image::images/vxlan-l3-asymmetric.svg["vxlan l3 asymmetric",align="center"]
 528
 529
 530 * node1
 531
 532 ----
 533 auto eno1
 534 iface eno1 inet manual
 535
 536 auto vmbr0
 537 iface vmbr0 inet static
 538          address 192.168.0.1
 539          netmask  255.255.255.0
 540          bridge_ports eno1
 541          bridge_stp off
 542          bridge_fd 0
 543
 544 auto vxlan2
 545 iface vxlan2 inet manual
 546         vxlan-local-tunnelip 192.168.0.1
 547         bridge-learning off
 548         bridge-arp-nd-suppress on
 549         bridge-unicast-flood off
 550         bridge-multicast-flood off
 551
 552
 553 auto vmbr2
 554 iface vmbr2 inet static
 555         address 10.0.2.254
 556         netmask 255.255.255.0
 557         hwaddress 44:39:39:FF:40:94
 558         bridge_ports vxlan2
 559         bridge_stp off
 560         bridge_fd 0
 561
 562
 563 auto vxlan3
 564 iface vxlan3 inet manual
 565         vxlan-local-tunnelip 192.168.0.1
 566         bridge-learning off
 567         bridge-arp-nd-suppress on
 568         bridge-unicast-flood off
 569         bridge-multicast-flood off
 570
 571
 572 auto vmbr3
 573 iface vmbr3 inet static
 574         address 10.0.3.254
 575         netmask 255.255.255.0
 576         hwaddress 44:39:39:FF:40:94
 577         bridge_ports vxlan3
 578         bridge_stp off
 579         bridge_fd 0
 580 ----
 581
 582
 583 frr.conf
 584
 585 ----
 586 router bgp 1234
 587  bgp router-id 192.168.0.1
 588  no bgp default ipv4-unicast
 589  coalesce-time 1000
 590  neighbor 192.168.0.2 remote-as 1234
 591  neighbor 192.168.0.3 remote-as 1234
 592  !
 593  address-family l2vpn evpn
 594   neighbor 192.168.0.2 activate
 595   neighbor 192.168.0.3 activate
 596   advertise-all-vni
 597  exit-address-family
 598 !
 599 line vty
 600 !
 601 ----
 602
 603
 604 * node2
 605
 606 ----
 607 auto eno1
 608 iface eno1 inet manual
 609
 610 auto vmbr0
 611 iface vmbr0 inet static
 612          address 192.168.0.2
 613          netmask  255.255.255.0
 614          bridge_ports eno1
 615          bridge_stp off
 616          bridge_fd 0
 617
 618 auto vxlan2
 619 iface vxlan2 inet manual
 620         vxlan-local-tunnelip 192.168.0.2
 621         bridge-learning off
 622         bridge-arp-nd-suppress on
 623         bridge-unicast-flood off
 624         bridge-multicast-flood off
 625
 626
 627 auto vmbr2
 628 iface vmbr2 inet static
 629         address 10.0.2.254
 630         netmask 255.255.255.0
 631         hwaddress 44:39:39:FF:40:94
 632         bridge_ports vxlan2
 633         bridge_stp off
 634         bridge_fd 0
 635
 636
 637 auto vxlan3
 638 iface vxlan3 inet manual
 639         vxlan-local-tunnelip 192.168.0.2
 640         bridge-learning off
 641         bridge-arp-nd-suppress on
 642         bridge-unicast-flood off
 643         bridge-multicast-flood off
 644
 645
 646 auto vmbr3
 647 iface vmbr3 inet static
 648         address 10.0.3.254
 649         netmask 255.255.255.0
 650         hwaddress 44:39:39:FF:40:94
 651         bridge_ports vxlan3
 652         bridge_stp off
 653         bridge_fd 0
 654 ----
 655
 656
 657 frr.conf
 658
 659 ----
 660 router bgp 1234
 661  bgp router-id 192.168.0.2
 662  no bgp default ipv4-unicast
 663  coalesce-time 1000
 664  neighbor 192.168.0.1 remote-as 1234
 665  neighbor 192.168.0.3 remote-as 1234
 666  !
 667  address-family l2vpn evpn
 668   neighbor 192.168.0.1 activate
 669   neighbor 192.168.0.3 activate
 670   advertise-all-vni
 671  exit-address-family
 672 !
 673 line vty
 674 !
 675 ----
 676
 677
 678 * node3
 679
 680 ----
 681 auto eno1
 682 iface eno1 inet manual
 683
 684 auto vmbr0
 685 iface vmbr0 inet static
 686          address 192.168.0.3
 687          netmask  255.255.255.0
 688          bridge_ports eno1
 689          bridge_stp off
 690          bridge_fd 0
 691
 692 auto vxlan2
 693 iface vxlan2 inet manual
 694         vxlan-local-tunnelip 192.168.0.3
 695         bridge-learning off
 696         bridge-arp-nd-suppress on
 697         bridge-unicast-flood off
 698         bridge-multicast-flood off
 699
 700
 701 auto vmbr2
 702 iface vmbr2 inet static
 703         address 10.0.2.254
 704         netmask 255.255.255.0
 705         hwaddress 44:39:39:FF:40:94
 706         bridge_ports vxlan2
 707         bridge_stp off
 708         bridge_fd 0
 709
 710
 711 auto vxlan3
 712 iface vxlan3 inet manual
 713         vxlan-local-tunnelip 192.168.0.3
 714         bridge-learning off
 715         bridge-arp-nd-suppress on
 716         bridge-unicast-flood off
 717         bridge-multicast-flood off
 718
 719
 720 auto vmbr3
 721 iface vmbr3 inet static
 722         address 10.0.3.254
 723         netmask 255.255.255.0
 724         hwaddress 44:39:39:FF:40:94
 725         bridge_ports vxlan3
 726         bridge_stp off
 727         bridge_fd 0
 728 ----
 729
 730
 731 frr.conf
 732
 733 ----
 734 router bgp 1234
 735  bgp router-id 192.168.0.3
 736  no bgp default ipv4-unicast
 737  coalesce-time 1000
 738  neighbor 192.168.0.1 remote-as 1234
 739  neighbor 192.168.0.2 remote-as 1234
 740  !
 741  address-family l2vpn evpn
 742   neighbor 192.168.0.1 activate
 743   neighbor 192.168.0.2 activate
 744   advertise-all-vni
 745  exit-address-family
 746 !
 747 line vty
 748 !
 749 ----
 750
 751
 752 symmetric model
 753 ^^^^^^^^^^^^^^^
 754
 755 With this model, you don't need to have all vxlan on all nodes.
 756 This model will also be needed to route traffic to an external router.
 757
 758 The symmetric model routes and bridges on both the ingress and the egress leafs.
 759 This results in bi-directional traffic being able to travel on the same VNI, hence the symmetric name.
 760 However, a new specialty transit VNI is used for all routed VXLAN traffic, called the L3VNI.
 761 All traffic that needs to be routed will be routed onto the L3VNI, tunneled across the layer 3 Infrastructure,
 762 routed off the L3VNI to the appropriate VLAN and ultimately bridged to the destination.
 763
 764 A vrf is needed for the L3VNI, so all vmbr bridge need to be in the vrf if they want to be able to reach each others.
 765
 766 image::images/vxlan-l3-symmetric.svg["vxlan l3 symmetric",align="center"]
 767
 768
 769 * node1
 770
 771 ----
 772 auto vrf1
 773 iface vrf1
 774     vrf-table auto
 775
 776 auto eno1
 777 iface eno1 inet manual
 778
 779 auto vmbr0
 780 iface vmbr0 inet static
 781          address 192.168.0.1
 782          netmask  255.255.255.0
 783          bridge_ports eno1
 784          bridge_stp off
 785          bridge_fd 0
 786
 787 auto vxlan2
 788 iface vxlan2 inet manual
 789         vxlan-local-tunnelip 192.168.0.1
 790         bridge-learning off
 791         bridge-arp-nd-suppress on
 792         bridge-unicast-flood off
 793         bridge-multicast-flood off
 794
 795 auto vmbr2
 796 iface vmbr2 inet static
 797         bridge_ports vxlan2
 798         bridge_stp off
 799         bridge_fd 0
 800         address 10.0.2.254
 801         netmask 255.255.255.0
 802         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
 803         vrf vrf1
 804
 805 auto vxlan3
 806 iface vxlan3 inet manual
 807         vxlan-local-tunnelip 192.168.0.1
 808         bridge-learning off
 809         bridge-arp-nd-suppress on
 810         bridge-unicast-flood off
 811         bridge-multicast-flood off
 812
 813 auto vmbr3
 814 iface vmbr3 inet static
 815         bridge_ports vxlan3
 816         bridge_stp off
 817         bridge_fd 0
 818         address 10.0.3.254
 819         netmask 255.255.255.0
 820         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
 821         vrf vrf1
 822
 823 #interconnect vxlan-vfr l3vni
 824 auto vxlan4000
 825 iface vxlan4000 inet manual
 826         vxlan-local-tunnelip 192.168.0.1
 827         bridge-learning off
 828         bridge-arp-nd-suppress on
 829         bridge-unicast-flood off
 830         bridge-multicast-flood off
 831
 832
 833 auto vmbr4000
 834 iface vmbr4000 inet manual
 835         bridge_ports vxlan4000
 836         bridge_stp off
 837         bridge_fd 0
 838         hwaddress 44:39:39:FF:40:90  #must be different on each node
 839         vrf vrf1
 840 ----
 841
 842 frr.conf
 843
 844 ----
 845 vrf vrf1
 846  vni 4000
 847 !
 848 router bgp 1234
 849  bgp router-id 192.168.0.1
 850  no bgp default ipv4-unicast
 851  coalesce-time 1000
 852  neighbor 192.168.0.2 remote-as 1234
 853  neighbor 192.168.0.3 remote-as 1234
 854  !
 855  address-family l2vpn evpn
 856   neighbor 192.168.0.2 activate
 857   neighbor 192.168.0.3 activate
 858   advertise-all-vni
 859  exit-address-family
 860 !
 861 router bgp 1234 vrf vrf1
 862 !
 863  bgp router-id 192.168.0.1
 864  !
 865  address-family ipv4 unicast
 866   redistribute connected
 867  exit-address-family
 868  !
 869  address-family l2vpn evpn
 870   advertise ipv4 unicast
 871  exit-address-family
 872 !
 873 line vty
 874 !
 875 ----
 876
 877
 878 * node2
 879
 880 ----
 881 auto vrf1
 882 iface vrf1
 883     vrf-table auto
 884
 885 auto eno1
 886 iface eno1 inet manual
 887
 888 auto vmbr0
 889 iface vmbr0 inet static
 890          address 192.168.0.2
 891          netmask  255.255.255.0
 892          bridge_ports eno1
 893          bridge_stp off
 894          bridge_fd 0
 895
 896 auto vxlan2
 897 iface vxlan2 inet manual
 898         vxlan-local-tunnelip 192.168.0.2
 899         bridge-learning off
 900         bridge-arp-nd-suppress on
 901         bridge-unicast-flood off
 902         bridge-multicast-flood off
 903
 904 auto vmbr2
 905 iface vmbr2 inet static
 906         bridge_ports vxlan2
 907         bridge_stp off
 908         bridge_fd 0
 909         address 10.0.2.254
 910         netmask 255.255.255.0
 911         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
 912         vrf vrf1
 913
 914 auto vxlan3
 915 iface vxlan3 inet manual
 916         vxlan-local-tunnelip 192.168.0.2
 917         bridge-learning off
 918         bridge-arp-nd-suppress on
 919         bridge-unicast-flood off
 920         bridge-multicast-flood off
 921
 922 auto vmbr3
 923 iface vmbr3 inet static
 924         bridge_ports vxlan3
 925         bridge_stp off
 926         bridge_fd 0
 927         address 10.0.3.254
 928         netmask 255.255.255.0
 929         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
 930         vrf vrf1
 931
 932 #interconnect vxlan-vfr l3vni
 933 auto vxlan4000
 934 iface vxlan4000 inet manual
 935         vxlan-local-tunnelip 192.168.0.2
 936         bridge-learning off
 937         bridge-arp-nd-suppress on
 938         bridge-unicast-flood off
 939         bridge-multicast-flood off
 940
 941
 942 auto vmbr4000
 943 iface vmbr4000 inet manual
 944         bridge_ports vxlan4000
 945         bridge_stp off
 946         bridge_fd 0
 947         hwaddress 44:39:39:FF:40:91  #must be different on each node
 948         vrf vrf1
 949 ----
 950
 951
 952 frr.conf
 953
 954 ----
 955 vrf vrf1
 956  vni 4000
 957 !
 958 router bgp 1234
 959  bgp router-id 192.168.0.2
 960  no bgp default ipv4-unicast
 961  coalesce-time 1000
 962  neighbor 192.168.0.1 remote-as 1234
 963  neighbor 192.168.0.3 remote-as 1234
 964  !
 965  address-family l2vpn evpn
 966   neighbor 192.168.0.1 activate
 967   neighbor 192.168.0.3 activate
 968   advertise-all-vni
 969  exit-address-family
 970 !
 971 router bgp 1234 vrf vrf1
 972 !
 973  bgp router-id 192.168.0.2
 974  !
 975  address-family ipv4 unicast
 976   redistribute connected
 977  exit-address-family
 978  !
 979  address-family l2vpn evpn
 980   advertise ipv4 unicast
 981  exit-address-family
 982 !
 983 line vty
 984 !
 985 ----
 986
 987
 988 * node3
 989
 990 ----
 991 auto vrf1
 992 iface vrf1
 993     vrf-table auto
 994
 995 auto eno1
 996 iface eno1 inet manual
 997
 998 auto vmbr0
 999 iface vmbr0 inet static
1000          address 192.168.0.3
1001          netmask  255.255.255.0
1002          bridge_ports eno1
1003          bridge_stp off
1004          bridge_fd 0
1005
1006 auto vxlan2
1007 iface vxlan2 inet manual
1008         vxlan-local-tunnelip 192.168.0.3
1009         bridge-learning off
1010         bridge-arp-nd-suppress on
1011         bridge-unicast-flood off
1012         bridge-multicast-flood off
1013
1014 auto vmbr2
1015 iface vmbr2 inet static
1016         bridge_ports vxlan2
1017         bridge_stp off
1018         bridge_fd 0
1019         address 10.0.2.254
1020         netmask 255.255.255.0
1021         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
1022         vrf vrf1
1023
1024 auto vxlan3
1025 iface vxlan3 inet manual
1026         vxlan-local-tunnelip 192.168.0.3
1027         bridge-learning off
1028         bridge-arp-nd-suppress on
1029         bridge-unicast-flood off
1030         bridge-multicast-flood off
1031
1032 auto vmbr3
1033 iface vmbr3 inet static
1034         bridge_ports vxlan3
1035         bridge_stp off
1036         bridge_fd 0
1037         address 10.0.3.254
1038         netmask 255.255.255.0
1039         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
1040         vrf vrf1
1041
1042 #interconnect vxlan-vfr l3vni
1043 auto vxlan4000
1044 iface vxlan4000 inet manual
1045         vxlan-local-tunnelip 192.168.0.3
1046         bridge-learning off
1047         bridge-arp-nd-suppress on
1048         bridge-unicast-flood off
1049         bridge-multicast-flood off
1050
1051
1052 auto vmbr4000
1053 iface vmbr4000 inet manual
1054         bridge_ports vxlan4000
1055         bridge_stp off
1056         bridge_fd 0
1057         hwaddress 44:39:39:FF:40:92  #must be different on each node
1058         vrf vrf1
1059 ----
1060
1061
1062 frr.conf
1063
1064 ----
1065 vrf vrf1
1066  vni 4000
1067 !
1068 router bgp 1234
1069  bgp router-id 192.168.0.3
1070  no bgp default ipv4-unicast
1071  coalesce-time 1000
1072  neighbor 192.168.0.1 remote-as 1234
1073  neighbor 192.168.0.2 remote-as 1234
1074  !
1075  address-family l2vpn evpn
1076   neighbor 192.168.0.1 activate
1077   neighbor 192.168.0.2 activate
1078   advertise-all-vni
1079  exit-address-family
1080 !
1081 router bgp 1234 vrf vrf1
1082 !
1083  bgp router-id 192.168.0.3
1084  !
1085  address-family ipv4 unicast
1086   redistribute connected
1087  exit-address-family
1088  !
1089  address-family l2vpn evpn
1090   advertise ipv4 unicast
1091  exit-address-family
1092 !
1093 line vty
1094 !
1095 ----