vxlan-and-evpn.adoc

   1
   2 ////
   3
   4 This is currently not included, because
   5 - it requires ifupdown2
   6 - routing needs more documentation
   7
   8 ////
   9
  10
  11 VXLAN layer2 with vlan unware linux bridges
  12 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  13
  14 VXLAN is an overlay network to carry Ethernet traffic over an existing IP network
  15 while accommodating a very large number of tenants. It is defined in RFC 7348.
  16 Each overlay network is known as a VXLAN Segment and identified by a unique
  17 24-bit segment ID called a VXLAN Network Identifier (VNI).
  18
  19 For BUM traffic (broadcast / unknown unicast traffic, multicast),
  20 we have 3 differents vxlan setup modes : multicast, unicast, bgp-evpn
  21
  22 image::images/vxlan-l2-vlanunaware.svg["vxlan l2 bridge vlan unaware",align="center"]
  23
  24 multicast mode
  25 ^^^^^^^^^^^^^^
  26
  27 This scenario relies in head end replication, meaning that end host in case
  28 of not having any entry for the destination MAC address will send out an ARP
  29 to other devices / VTEPs in the VXLAN network.
  30 This is done by sending the request to the VXLAN multicast group,
  31 remote VTEPs will get the packet and answer accordingly direct to the originating VTEP.
  32
  33
  34 * node1
  35
  36 ----
  37 auto eno1
  38 iface eno1 inet manual
  39
  40 auto vmbr0
  41 iface vmbr0 inet static
  42         address 192.168.0.1
  43         netmask  255.255.255.0
  44         bridge_ports eno1
  45         bridge_stp off
  46         bridge_fd 0
  47
  48 auto vxlan2
  49 iface vxlan2 inet manual
  50         vxlan-svcnodeip 225.20.1.1
  51         vxlan-physdev eno1
  52
  53 auto vmbr2
  54 iface vmbr2 inet manual
  55         bridge_ports vxlan2
  56         bridge_stp off
  57         bridge_fd 0
  58
  59 auto vxlan3
  60 iface vxlan3 inet manual
  61         vxlan-svcnodeip 225.20.1.1
  62         vxlan-physdev eno1
  63
  64 auto vmbr3
  65 iface vmbr3 inet manual
  66         bridge_ports vxlan3
  67         bridge_stp off
  68         bridge_fd 0
  69 ----
  70
  71
  72 * node2
  73
  74 ----
  75 auto eno1
  76 iface eno1 inet manual
  77
  78 auto vmbr0
  79 iface vmbr0 inet static
  80         address 192.168.0.2
  81         netmask  255.255.255.0
  82         bridge_ports eno1
  83         bridge_stp off
  84         bridge_fd 0
  85
  86 auto vxlan2
  87 iface vxlan2 inet manual
  88         vxlan-svcnodeip 225.20.1.1
  89         vxlan-physdev eno1
  90
  91 auto vmbr2
  92 iface vmbr2 inet manual
  93         bridge_ports vxlan2
  94         bridge_stp off
  95         bridge_fd 0
  96
  97
  98 auto vxlan3
  99 iface vxlan3 inet manual
 100         vxlan-svcnodeip 225.20.1.1
 101         vxlan-physdev eno1
 102
 103 auto vmbr3
 104 iface vmbr3 inet manual
 105         bridge_ports vxlan3
 106         bridge_stp off
 107         bridge_fd 0
 108 ----
 109
 110
 111 * node3
 112
 113 ----
 114 auto eno1
 115 iface eno1 inet manual
 116
 117 auto vmbr0
 118 iface vmbr0 inet static
 119         address 192.168.0.3
 120         netmask  255.255.255.0
 121         bridge_ports eno1
 122         bridge_stp off
 123         bridge_fd 0
 124
 125 auto vxlan2
 126 iface vxlan2 inet manual
 127         vxlan-svcnodeip 225.20.1.1
 128         vxlan-physdev eno1
 129
 130 auto vmbr2
 131 iface vmbr2 inet manual
 132         bridge_ports vxlan2
 133         bridge_stp off
 134         bridge_fd 0
 135
 136
 137 auto vxlan3
 138 iface vxlan3 inet manual
 139         vxlan-svcnodeip 225.20.1.1
 140         vxlan-physdev eno1
 141
 142 auto vmbr3
 143 iface vmbr3 inet manual
 144         bridge_ports vxlan3
 145         bridge_stp off
 146         bridge_fd 0
 147 ----
 148
 149
 150 unicast mode
 151 ^^^^^^^^^^^^
 152
 153 We can replace multicast by head-end replication of BUM frames to a statically configured lists of remote VTEPs.
 154 The VXLAN is defined without a remote multicast group.
 155 Instead, all the remote VTEPs are associated with the all-zero address:
 156 a BUM frame will be duplicated to all these destinations.
 157 The VXLAN device will still learn remote addresses automatically using source-address learning.
 158
 159 * node1
 160
 161 ----
 162 auto eno1
 163 iface eno1 inet manual
 164
 165 auto vmbr0
 166 iface vmbr0 inet static
 167         address 192.168.0.1
 168         netmask  255.255.255.0
 169         bridge_ports eno1
 170         bridge_stp off
 171         bridge_fd 0
 172
 173
 174 auto vxlan2
 175 iface vxlan2 inet manual
 176         vxlan_remoteip 192.168.0.2
 177         vxlan_remoteip 192.168.0.3
 178
 179
 180 auto vmbr2
 181 iface vmbr2 inet manual
 182         bridge_ports vxlan2
 183         bridge_stp off
 184         bridge_fd 0
 185
 186
 187 auto vxlan3
 188 iface vxlan2 inet manual
 189         vxlan_remoteip 192.168.0.2
 190         vxlan_remoteip 192.168.0.3
 191
 192
 193 auto vmbr3
 194 iface vmbr3 inet manual
 195         bridge_ports vxlan3
 196         bridge_stp off
 197         bridge_fd 0
 198 ----
 199
 200
 201 * node2
 202
 203 ----
 204 auto eno1
 205 iface eno1 inet manual
 206
 207 auto vmbr0
 208 iface vmbr0 inet static
 209         address 192.168.0.2
 210         netmask  255.255.255.0
 211         bridge_ports eno1
 212         bridge_stp off
 213         bridge_fd 0
 214
 215 auto vxlan2
 216 iface vxlan2 inet manual
 217         vxlan_remoteip 192.168.0.1
 218         vxlan_remoteip 192.168.0.3
 219
 220
 221
 222 auto vmbr2
 223 iface vmbr2 inet manual
 224         bridge_ports vxlan2
 225         bridge_stp off
 226         bridge_fd 0
 227
 228 auto vxlan3
 229 iface vxlan2 inet manual
 230         vxlan_remoteip 192.168.0.1
 231         vxlan_remoteip 192.168.0.3
 232
 233
 234 auto vmbr3
 235 iface vmbr3 inet manual
 236         bridge_ports vxlan3
 237         bridge_stp off
 238         bridge_fd 0
 239 ----
 240
 241
 242 * node3
 243
 244 ----
 245 auto eno1
 246 iface eno1 inet manual
 247
 248 auto vmbr0
 249 iface vmbr0 inet static
 250         address 192.168.0.3
 251         netmask  255.255.255.0
 252         bridge_ports eno1
 253         bridge_stp off
 254         bridge_fd 0
 255
 256 auto vxlan2
 257 iface vxlan2 inet manual
 258         vxlan_remoteip 192.168.0.2
 259         vxlan_remoteip 192.168.0.3
 260
 261
 262
 263 auto vmbr2
 264 iface vmbr2 inet manual
 265         bridge_ports vxlan2
 266         bridge_stp off
 267         bridge_fd 0
 268
 269 auto vxlan3
 270 iface vxlan2 inet manual
 271         vxlan_remoteip 192.168.0.2
 272         vxlan_remoteip 192.168.0.3
 273
 274
 275 auto vmbr3
 276 iface vmbr3 inet manual
 277         bridge_ports vxlan3
 278         bridge_stp off
 279         bridge_fd 0
 280 ----
 281
 282
 283 bgp-evpn
 284 ^^^^^^^^
 285
 286 VTEPs use control plane learning/distribution via BGP for remote MAC addresses instead of data plane learning.
 287 VTEPs have the ability to suppress ARP flooding over VXLAN tunnels.
 288
 289 The control plane used here is FRR, a bgp routing software.
 290 Each node in the proxmox cluster peer with each others nodes.
 291 For bigger networks, or multiple proxmox clusters,
 292 it's possible to use external bgp route reflector servers.
 293
 294 * node1
 295
 296 ----
 297 auto eno1
 298 iface eno1 inet manual
 299
 300 auto vmbr0
 301 iface vmbr0 inet static
 302         address 192.168.0.1
 303         netmask 255.255.255.0
 304         bridge_ports eno1
 305         bridge_stp off
 306         bridge_fd 0
 307
 308 auto vxlan2
 309 iface vxlan2 inet manual
 310         vxlan-local-tunnelip 192.168.0.1
 311         bridge-learning off
 312         bridge-arp-nd-suppress on
 313         bridge-unicast-flood off
 314         bridge-multicast-flood off
 315
 316
 317 auto vmbr2
 318 iface vmbr2 inet manual
 319         bridge_ports vxlan2
 320         bridge_stp off
 321         bridge_fd 0
 322
 323
 324 auto vxlan3
 325 iface vxlan3 inet manual
 326         vxlan-local-tunnelip 192.168.0.1
 327         bridge-learning off
 328         bridge-arp-nd-suppress on
 329         bridge-unicast-flood off
 330         bridge-multicast-flood off
 331
 332
 333 auto vmbr3
 334 iface vmbr3 inet manual
 335         bridge_ports vxlan3
 336         bridge_stp off
 337         bridge_fd 0
 338 ----
 339
 340
 341 /etc/frr/frr.conf
 342
 343 ----
 344 router bgp 1234
 345  no bgp default ipv4-unicast
 346  coalesce-time 1000
 347  neighbor 192.168.0.2 remote-as 1234
 348  neighbor 192.168.0.3 remote-as 1234
 349  !
 350  address-family l2vpn evpn
 351   neighbor 192.168.0.2 activate
 352   neighbor 192.168.0.3 activate
 353   advertise-all-vni
 354  exit-address-family
 355 !
 356 line vty
 357 !
 358 ----
 359
 360
 361 * node2
 362
 363 ----
 364 auto eno1
 365 iface eno1 inet manual
 366
 367 auto vmbr0
 368 iface vmbr0 inet static
 369         address 192.168.0.2
 370         netmask 255.255.255.0
 371         bridge_ports eno1
 372         bridge_stp off
 373         bridge_fd 0
 374
 375 auto vxlan2
 376 iface vxlan2 inet manual
 377         vxlan-local-tunnelip 192.168.0.2
 378         bridge-learning off
 379         bridge-arp-nd-suppress on
 380         bridge-unicast-flood off
 381         bridge-multicast-flood off
 382
 383
 384 auto vmbr2
 385 iface vmbr2 inet manual
 386         bridge_ports vxlan2
 387         bridge_stp off
 388         bridge_fd 0
 389
 390 auto vxlan3
 391 iface vxlan3 inet manual
 392         vxlan-local-tunnelip 192.168.0.2
 393         bridge-learning off
 394         bridge-arp-nd-suppress on
 395         bridge-unicast-flood off
 396         bridge-multicast-flood off
 397
 398
 399 auto vmbr3
 400 iface vmbr3 inet manual
 401         bridge_ports vxlan3
 402         bridge_stp off
 403         bridge_fd 0
 404 ----
 405
 406
 407 /etc/frr/frr.conf
 408
 409 ----
 410 router bgp 1234
 411  no bgp default ipv4-unicast
 412  coalesce-time 1000
 413  neighbor 192.168.0.1 remote-as 1234
 414  neighbor 192.168.0.3 remote-as 1234
 415  !
 416  address-family l2vpn evpn
 417   neighbor 192.168.0.1 activate
 418   neighbor 192.168.0.3 activate
 419   advertise-all-vni
 420  exit-address-family
 421 !
 422 line vty
 423 !
 424 ----
 425
 426
 427 * node3
 428
 429 ----
 430 auto eno1
 431 iface eno1 inet manual
 432
 433 auto vmbr0
 434 iface vmbr0 inet static
 435         address 192.168.0.2
 436         netmask 255.255.255.0
 437         bridge_ports eno1
 438         bridge_stp off
 439         bridge_fd 0
 440
 441 auto vxlan2
 442 iface vxlan2 inet manual
 443         vxlan-local-tunnelip 192.168.0.3
 444         bridge-learning off
 445         bridge-arp-nd-suppress on
 446         bridge-unicast-flood off
 447         bridge-multicast-flood off
 448
 449
 450 auto vmbr2
 451 iface vmbr2 inet manual
 452         bridge_ports vxlan2
 453         bridge_stp off
 454         bridge_fd 0
 455
 456 auto vxlan3
 457 iface vxlan3 inet manual
 458         vxlan-local-tunnelip 192.168.0.3
 459         bridge-learning off
 460         bridge-arp-nd-suppress on
 461         bridge-unicast-flood off
 462         bridge-multicast-flood off
 463
 464
 465 auto vmbr3
 466 iface vmbr3 inet manual
 467         bridge_ports vxlan3
 468         bridge_stp off
 469         bridge_fd 0
 470 ----
 471
 472
 473 /etc/frr/frr.conf
 474
 475
 476 ----
 477 router bgp 1234
 478  no bgp default ipv4-unicast
 479  coalesce-time 1000
 480  neighbor 192.168.0.1 remote-as 1234
 481  neighbor 192.168.0.2 remote-as 1234
 482  !
 483  address-family l2vpn evpn
 484   neighbor 192.168.0.1 activate
 485   neighbor 192.168.0.2 activate
 486   advertise-all-vni
 487  exit-address-family
 488 !
 489 line vty
 490 !
 491 ----
 492
 493
 494 VXLAN layer2 with vlan aware linux bridges
 495 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 496
 497 We use 1 vmbr bridge, each vxlan is mapped to a vlan
 498
 499 image::images/vxlan-l2-vlanaware.svg["vxlan l2 bridge vlan aware",align="center"]
 500
 501 multicast mode
 502 ^^^^^^^^^^^^^^
 503
 504 * node1
 505
 506 ----
 507 auto eno1
 508 iface eno1 inet manual
 509
 510 auto vmbr0
 511 iface vmbr0 inet static
 512         address 192.168.0.1
 513         netmask  255.255.255.0
 514         bridge_ports eno1 vxlan2 vxlan3
 515         bridge_stp off
 516         bridge_fd 0
 517         bridge_vlan_aware yes
 518
 519 auto vxlan2
 520 iface vxlan2 inet manual
 521         vxlan-svcnodeip 225.20.1.1
 522         vxlan-physdev eno1
 523         bridge-access 2
 524
 525 auto vxlan3
 526 iface vxlan3 inet manual
 527         vxlan-svcnodeip 225.20.1.1
 528         vxlan-physdev eno1
 529         bridge-access 3
 530 ----
 531
 532
 533 * node2
 534
 535 ----
 536 auto eno1
 537 iface eno1 inet manual
 538
 539 auto vmbr0
 540 iface vmbr0 inet static
 541         address 192.168.0.2
 542         netmask  255.255.255.0
 543         bridge_ports eno1 vxlan2 vxlan3
 544         bridge_stp off
 545         bridge_fd 0
 546         bridge_vlan_aware yes
 547
 548 auto vxlan2
 549 iface vxlan2 inet manual
 550         vxlan-svcnodeip 225.20.1.1
 551         vxlan-physdev eno1
 552         bridge-access 2
 553
 554 auto vxlan3
 555 iface vxlan3 inet manual
 556         vxlan-svcnodeip 225.20.1.1
 557         vxlan-physdev eno1
 558         bridge-access 3
 559 ----
 560
 561
 562 * node3
 563
 564 ----
 565 auto eno1
 566 iface eno1 inet manual
 567
 568 auto vmbr0
 569 iface vmbr0 inet static
 570         address 192.168.0.3
 571         netmask 255.255.255.0
 572         bridge_ports eno1 vxlan2 vxlan3
 573         bridge_stp off
 574         bridge_fd 0
 575         bridge_vlan_aware yes
 576
 577 auto vxlan2
 578 iface vxlan2 inet manual
 579         vxlan-svcnodeip 225.20.1.1
 580         vxlan-physdev eno1
 581         bridge-access 2
 582
 583 auto vxlan3
 584 iface vxlan3 inet manual
 585         vxlan-svcnodeip 225.20.1.1
 586         vxlan-physdev eno1
 587         bridge-access 3
 588 ----
 589
 590
 591 unicast mode
 592 ^^^^^^^^^^^^
 593
 594 * node1
 595
 596 ----
 597 auto eno1
 598 iface eno1 inet manual
 599
 600 auto vmbr0
 601 iface vmbr0 inet static
 602         address 192.168.0.1
 603         netmask 255.255.255.0
 604         bridge_ports eno1 vxlan2 vxlan3
 605         bridge_stp off
 606         bridge_fd 0
 607         bridge_vlan_aware yes
 608
 609 auto vxlan2
 610 iface vxlan2 inet manual
 611         vxlan_remoteip 192.168.0.2
 612         vxlan_remoteip 192.168.0.3
 613         bridge-access 2
 614
 615 auto vxlan3
 616 iface vxlan3 inet manual
 617         vxlan_remoteip 192.168.0.2
 618         vxlan_remoteip 192.168.0.3
 619         bridge-access 3
 620 ----
 621
 622
 623 * node2
 624
 625 ----
 626 auto eno1
 627 iface eno1 inet manual
 628
 629 auto vmbr0
 630 iface vmbr0 inet static
 631         address 192.168.0.2
 632         netmask 255.255.255.0
 633         bridge_ports eno1 vxlan2 vxlan3
 634         bridge_stp off
 635         bridge_fd 0
 636         bridge_vlan_aware yes
 637
 638 auto vxlan2
 639 iface vxlan2 inet manual
 640         vxlan_remoteip 192.168.0.1
 641         vxlan_remoteip 192.168.0.3
 642         bridge-access 2
 643
 644 auto vxlan3
 645 iface vxlan3 inet manual
 646         vxlan_remoteip 192.168.0.1
 647         vxlan_remoteip 192.168.0.3
 648         bridge-access 3
 649 ----
 650
 651
 652 * node3
 653
 654 ----
 655 auto eno1
 656 iface eno1 inet manual
 657
 658 auto vmbr0
 659 iface vmbr0 inet static
 660         address 192.168.0.3
 661         netmask 255.255.255.0
 662         bridge_ports eno1 vxlan2 vxlan3
 663         bridge_stp off
 664         bridge_fd 0
 665         bridge_vlan_aware yes
 666
 667 auto vxlan2
 668 iface vxlan2 inet manual
 669         vxlan_remoteip 192.168.0.2
 670         vxlan_remoteip 192.168.0.3
 671         bridge-access 2
 672
 673 auto vxlan3
 674 iface vxlan3 inet manual
 675         vxlan_remoteip 192.168.0.2
 676         vxlan_remoteip 192.168.0.3
 677         bridge-access 3
 678 ----
 679
 680
 681 bgp-evpn
 682 ^^^^^^^^
 683
 684 Note: currently FRR is working only with 1 vlan aware bridge
 685
 686 * node1
 687
 688
 689 ----
 690 auto eno1
 691 iface eno1 inet manual
 692
 693 auto vmbr0
 694 iface vmbr0 inet static
 695         address 192.168.0.1
 696         netmask 255.255.255.0
 697         bridge_ports eno1 vxlan2 vxlan3
 698         bridge_stp off
 699         bridge_fd 0
 700         bridge_vlan_aware yes
 701
 702 auto vxlan0
 703 iface vxlan0 inet manual
 704         vxlan-local-tunnelip 192.168.0.1
 705         bridge-learning off
 706         bridge-arp-nd-suppress on
 707         bridge-unicast-flood off
 708         bridge-multicast-flood off
 709         bridge-access 2
 710
 711
 712 auto vxlan3
 713 iface vxlan3 inet manual
 714         vxlan-local-tunnelip 192.168.0.1
 715         bridge-learning off
 716         bridge-arp-nd-suppress on
 717         bridge-unicast-flood off
 718         bridge-multicast-flood off
 719         bridge-access 3
 720 ----
 721
 722
 723 /etc/frr/frr.conf
 724
 725 ----
 726 router bgp 1234
 727  no bgp default ipv4-unicast
 728  coalesce-time 1000
 729  neighbor 192.168.0.2 remote-as 1234
 730  neighbor 192.168.0.3 remote-as 1234
 731  !
 732  address-family l2vpn evpn
 733   neighbor 192.168.0.2 activate
 734   neighbor 192.168.0.3 activate
 735   advertise-all-vni
 736  exit-address-family
 737 !
 738 line vty
 739 !
 740 ----
 741
 742
 743 * node2
 744
 745 ----
 746 auto eno1
 747 iface eno1 inet manual
 748
 749 auto vmbr0
 750 iface vmbr0 inet static
 751         address 192.168.0.2
 752         netmask 255.255.255.0
 753         bridge_ports eno1 vxlan2 vxlan3
 754         bridge_stp off
 755         bridge_fd 0
 756         bridge_vlan_aware yes
 757
 758 auto vxlan0
 759 iface vxlan0 inet manual
 760         vxlan-local-tunnelip 192.168.0.2
 761         bridge-learning off
 762         bridge-arp-nd-suppress on
 763         bridge-unicast-flood off
 764         bridge-multicast-flood off
 765         bridge-access 2
 766
 767
 768 auto vxlan3
 769 iface vxlan3 inet manual
 770         vxlan-local-tunnelip 192.168.0.2
 771         bridge-learning off
 772         bridge-arp-nd-suppress on
 773         bridge-unicast-flood off
 774         bridge-multicast-flood off
 775         bridge-access 3
 776 ----
 777
 778
 779 /etc/frr/frr.conf
 780
 781 ----
 782 router bgp 1234
 783  no bgp default ipv4-unicast
 784  coalesce-time 1000
 785  neighbor 192.168.0.1 remote-as 1234
 786  neighbor 192.168.0.3 remote-as 1234
 787  !
 788  address-family l2vpn evpn
 789   neighbor 192.168.0.1 activate
 790   neighbor 192.168.0.3 activate
 791   advertise-all-vni
 792  exit-address-family
 793 !
 794 line vty
 795 !
 796 ----
 797
 798
 799 * node3
 800
 801 ----
 802 auto eno1
 803 iface eno1 inet manual
 804
 805 auto vmbr0
 806 iface vmbr0 inet static
 807         address 192.168.0.3
 808         netmask 255.255.255.0
 809         bridge_ports eno1 vxlan2 vxlan3
 810         bridge_stp off
 811         bridge_fd 0
 812         bridge_vlan_aware yes
 813
 814 auto vxlan0
 815 iface vxlan0 inet manual
 816         vxlan-local-tunnelip 192.168.0.3
 817         bridge-learning off
 818         bridge-arp-nd-suppress on
 819         bridge-unicast-flood off
 820         bridge-multicast-flood off
 821         bridge-access 2
 822
 823
 824 auto vxlan3
 825 iface vxlan3 inet manual
 826         vxlan-local-tunnelip 192.168.0.3
 827         bridge-learning off
 828         bridge-arp-nd-suppress on
 829         bridge-unicast-flood off
 830         bridge-multicast-flood off
 831         bridge-access 3
 832 ----
 833
 834
 835 /etc/frr/frr.conf
 836 ----
 837 router bgp 1234
 838  no bgp default ipv4-unicast
 839  coalesce-time 1000
 840  neighbor 192.168.0.1 remote-as 1234
 841  neighbor 192.168.0.2 remote-as 1234
 842  !
 843  address-family l2vpn evpn
 844   neighbor 192.168.0.1 activate
 845   neighbor 192.168.0.2 activate
 846   advertise-all-vni
 847  exit-address-family
 848 !
 849 line vty
 850 !
 851 ----
 852
 853 VXLAN layer3 routing with anycast gateway
 854 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 855
 856 With this need, each vmbr bridge will be the gateway for the vm.
 857 Same vmbr on different node, will have same ip address and same mac address,
 858 to have working vm live migration and no network disruption.
 859
 860 VXLAN layer3 routing only work with FRR and non-aware bridge.
 861 (vlan aware bridge support is buggy currently).
 862
 863
 864 sysctl.conf tuning
 865
 866 ----
 867 #enable routing
 868 net.ipv4.ip_forward=1
 869 net.ipv6.conf.all.forwarding=1
 870 #disable reverse path filtering
 871 net.ipv4.conf.default.rp_filter=0
 872 net.ipv4.conf.all.rp_filter=0
 873 #allow frr to work with vrf
 874 net.ipv4.tcp_l3mdev_accept=1
 875 ----
 876
 877 asymmetric model
 878 ^^^^^^^^^^^^^^^^
 879
 880 This is the simplest mode. To get it work, all vxlan need to be defined on all nodes.
 881
 882 The asymmetric model allows routing and bridging on the VXLAN tunnel ingress,
 883 but only bridging on the egress.
 884 This results in bi-directional VXLAN traffic traveling on different VNIs
 885 in each direction (always the destination VNI) across the routed infrastructure.
 886
 887 image::images/vxlan-l3-asymmetric.svg["vxlan l3 asymmetric",align="center"]
 888
 889
 890 * node1
 891
 892 ----
 893 auto eno1
 894 iface eno1 inet manual
 895
 896 auto vmbr0
 897 iface vmbr0 inet static
 898          address 192.168.0.1
 899          netmask  255.255.255.0
 900          bridge_ports eno1
 901          bridge_stp off
 902          bridge_fd 0
 903
 904 auto vxlan2
 905 iface vxlan2 inet manual
 906         vxlan-local-tunnelip 192.168.0.1
 907         bridge-learning off
 908         bridge-arp-nd-suppress on
 909         bridge-unicast-flood off
 910         bridge-multicast-flood off
 911
 912
 913 auto vmbr2
 914 iface vmbr2 inet static
 915         address 10.0.2.254
 916         netmask 255.255.255.0
 917         hwaddress 44:39:39:FF:40:94
 918         bridge_ports vxlan2
 919         bridge_stp off
 920         bridge_fd 0
 921
 922
 923 auto vxlan3
 924 iface vxlan3 inet manual
 925         vxlan-local-tunnelip 192.168.0.1
 926         bridge-learning off
 927         bridge-arp-nd-suppress on
 928         bridge-unicast-flood off
 929         bridge-multicast-flood off
 930
 931
 932 auto vmbr3
 933 iface vmbr3 inet static
 934         address 10.0.3.254
 935         netmask 255.255.255.0
 936         hwaddress 44:39:39:FF:40:94
 937         bridge_ports vxlan3
 938         bridge_stp off
 939         bridge_fd 0
 940 ----
 941
 942
 943 frr.conf
 944
 945 ----
 946 router bgp 1234
 947  bgp router-id 192.168.0.1
 948  no bgp default ipv4-unicast
 949  coalesce-time 1000
 950  neighbor 192.168.0.2 remote-as 1234
 951  neighbor 192.168.0.3 remote-as 1234
 952  !
 953  address-family l2vpn evpn
 954   neighbor 192.168.0.2 activate
 955   neighbor 192.168.0.3 activate
 956   advertise-all-vni
 957  exit-address-family
 958 !
 959 line vty
 960 !
 961 ----
 962
 963
 964 * node2
 965
 966 ----
 967 auto eno1
 968 iface eno1 inet manual
 969
 970 auto vmbr0
 971 iface vmbr0 inet static
 972          address 192.168.0.2
 973          netmask  255.255.255.0
 974          bridge_ports eno1
 975          bridge_stp off
 976          bridge_fd 0
 977
 978 auto vxlan2
 979 iface vxlan2 inet manual
 980         vxlan-local-tunnelip 192.168.0.2
 981         bridge-learning off
 982         bridge-arp-nd-suppress on
 983         bridge-unicast-flood off
 984         bridge-multicast-flood off
 985
 986
 987 auto vmbr2
 988 iface vmbr2 inet static
 989         address 10.0.2.254
 990         netmask 255.255.255.0
 991         hwaddress 44:39:39:FF:40:94
 992         bridge_ports vxlan2
 993         bridge_stp off
 994         bridge_fd 0
 995
 996
 997 auto vxlan3
 998 iface vxlan3 inet manual
 999         vxlan-local-tunnelip 192.168.0.2
1000         bridge-learning off
1001         bridge-arp-nd-suppress on
1002         bridge-unicast-flood off
1003         bridge-multicast-flood off
1004
1005
1006 auto vmbr3
1007 iface vmbr3 inet static
1008         address 10.0.3.254
1009         netmask 255.255.255.0
1010         hwaddress 44:39:39:FF:40:94
1011         bridge_ports vxlan3
1012         bridge_stp off
1013         bridge_fd 0
1014 ----
1015
1016
1017 frr.conf
1018
1019 ----
1020 router bgp 1234
1021  bgp router-id 192.168.0.2
1022  no bgp default ipv4-unicast
1023  coalesce-time 1000
1024  neighbor 192.168.0.1 remote-as 1234
1025  neighbor 192.168.0.3 remote-as 1234
1026  !
1027  address-family l2vpn evpn
1028   neighbor 192.168.0.1 activate
1029   neighbor 192.168.0.3 activate
1030   advertise-all-vni
1031  exit-address-family
1032 !
1033 line vty
1034 !
1035 ----
1036
1037
1038 * node3
1039
1040 ----
1041 auto eno1
1042 iface eno1 inet manual
1043
1044 auto vmbr0
1045 iface vmbr0 inet static
1046          address 192.168.0.3
1047          netmask  255.255.255.0
1048          bridge_ports eno1
1049          bridge_stp off
1050          bridge_fd 0
1051
1052 auto vxlan2
1053 iface vxlan2 inet manual
1054         vxlan-local-tunnelip 192.168.0.3
1055         bridge-learning off
1056         bridge-arp-nd-suppress on
1057         bridge-unicast-flood off
1058         bridge-multicast-flood off
1059
1060
1061 auto vmbr2
1062 iface vmbr2 inet static
1063         address 10.0.2.254
1064         netmask 255.255.255.0
1065         hwaddress 44:39:39:FF:40:94
1066         bridge_ports vxlan2
1067         bridge_stp off
1068         bridge_fd 0
1069
1070
1071 auto vxlan3
1072 iface vxlan3 inet manual
1073         vxlan-local-tunnelip 192.168.0.3
1074         bridge-learning off
1075         bridge-arp-nd-suppress on
1076         bridge-unicast-flood off
1077         bridge-multicast-flood off
1078
1079
1080 auto vmbr3
1081 iface vmbr3 inet static
1082         address 10.0.3.254
1083         netmask 255.255.255.0
1084         hwaddress 44:39:39:FF:40:94
1085         bridge_ports vxlan3
1086         bridge_stp off
1087         bridge_fd 0
1088 ----
1089
1090
1091 frr.conf
1092
1093 ----
1094 router bgp 1234
1095  bgp router-id 192.168.0.3
1096  no bgp default ipv4-unicast
1097  coalesce-time 1000
1098  neighbor 192.168.0.1 remote-as 1234
1099  neighbor 192.168.0.2 remote-as 1234
1100  !
1101  address-family l2vpn evpn
1102   neighbor 192.168.0.1 activate
1103   neighbor 192.168.0.2 activate
1104   advertise-all-vni
1105  exit-address-family
1106 !
1107 line vty
1108 !
1109 ----
1110
1111
1112 symmetric model
1113 ^^^^^^^^^^^^^^^
1114
1115 With this model, you don't need to have all vxlan on all nodes.
1116 This model will also be needed to route traffic to an external router.
1117
1118 The symmetric model routes and bridges on both the ingress and the egress leafs.
1119 This results in bi-directional traffic being able to travel on the same VNI, hence the symmetric name.
1120 However, a new specialty transit VNI is used for all routed VXLAN traffic, called the L3VNI.
1121 All traffic that needs to be routed will be routed onto the L3VNI, tunneled across the layer 3 Infrastructure,
1122 routed off the L3VNI to the appropriate VLAN and ultimately bridged to the destination.
1123
1124 A vrf is needed for the L3VNI, so all vmbr bridge need to be in the vrf if they want to be able to reach each others.
1125
1126 image::images/vxlan-l3-symmetric.svg["vxlan l3 symmetric",align="center"]
1127
1128
1129 * node1
1130
1131 ----
1132 auto vrf1
1133 iface vrf1
1134     vrf-table auto
1135
1136 auto eno1
1137 iface eno1 inet manual
1138
1139 auto vmbr0
1140 iface vmbr0 inet static
1141          address 192.168.0.1
1142          netmask  255.255.255.0
1143          bridge_ports eno1
1144          bridge_stp off
1145          bridge_fd 0
1146
1147 auto vxlan2
1148 iface vxlan2 inet manual
1149         vxlan-local-tunnelip 192.168.0.1
1150         bridge-learning off
1151         bridge-arp-nd-suppress on
1152         bridge-unicast-flood off
1153         bridge-multicast-flood off
1154
1155 auto vmbr2
1156 iface vmbr2 inet static
1157         bridge_ports vxlan2
1158         bridge_stp off
1159         bridge_fd 0
1160         address 10.0.2.254
1161         netmask 255.255.255.0
1162         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
1163         vrf vrf1
1164
1165 auto vxlan3
1166 iface vxlan3 inet manual
1167         vxlan-local-tunnelip 192.168.0.1
1168         bridge-learning off
1169         bridge-arp-nd-suppress on
1170         bridge-unicast-flood off
1171         bridge-multicast-flood off
1172
1173 auto vmbr3
1174 iface vmbr3 inet static
1175         bridge_ports vxlan3
1176         bridge_stp off
1177         bridge_fd 0
1178         address 10.0.3.254
1179         netmask 255.255.255.0
1180         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
1181         vrf vrf1
1182
1183 #interconnect vxlan-vfr l3vni
1184 auto vxlan4000
1185 iface vxlan4000 inet manual
1186         vxlan-local-tunnelip 192.168.0.1
1187         bridge-learning off
1188         bridge-arp-nd-suppress on
1189         bridge-unicast-flood off
1190         bridge-multicast-flood off
1191
1192
1193 auto vmbr4000
1194 iface vmbr4000 inet manual
1195         bridge_ports vxlan4000
1196         bridge_stp off
1197         bridge_fd 0
1198         hwaddress 44:39:39:FF:40:90  #must be different on each node
1199         vrf vrf1
1200 ----
1201
1202 frr.conf
1203
1204 ----
1205 vrf vrf1
1206  vni 4000
1207 !
1208 router bgp 1234
1209  bgp router-id 192.168.0.1
1210  no bgp default ipv4-unicast
1211  coalesce-time 1000
1212  neighbor 192.168.0.2 remote-as 1234
1213  neighbor 192.168.0.3 remote-as 1234
1214  !
1215  address-family l2vpn evpn
1216   neighbor 192.168.0.2 activate
1217   neighbor 192.168.0.3 activate
1218   advertise-all-vni
1219  exit-address-family
1220 !
1221 router bgp 1234 vrf vrf1
1222 !
1223  bgp router-id 192.168.0.1
1224  !
1225  address-family ipv4 unicast
1226   redistribute connected
1227  exit-address-family
1228  !
1229  address-family l2vpn evpn
1230   advertise ipv4 unicast
1231  exit-address-family
1232 !
1233 line vty
1234 !
1235 ----
1236
1237
1238 * node2
1239
1240 ----
1241 auto vrf1
1242 iface vrf1
1243     vrf-table auto
1244
1245 auto eno1
1246 iface eno1 inet manual
1247
1248 auto vmbr0
1249 iface vmbr0 inet static
1250          address 192.168.0.2
1251          netmask  255.255.255.0
1252          bridge_ports eno1
1253          bridge_stp off
1254          bridge_fd 0
1255
1256 auto vxlan2
1257 iface vxlan2 inet manual
1258         vxlan-local-tunnelip 192.168.0.2
1259         bridge-learning off
1260         bridge-arp-nd-suppress on
1261         bridge-unicast-flood off
1262         bridge-multicast-flood off
1263
1264 auto vmbr2
1265 iface vmbr2 inet static
1266         bridge_ports vxlan2
1267         bridge_stp off
1268         bridge_fd 0
1269         address 10.0.2.254
1270         netmask 255.255.255.0
1271         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
1272         vrf vrf1
1273
1274 auto vxlan3
1275 iface vxlan3 inet manual
1276         vxlan-local-tunnelip 192.168.0.2
1277         bridge-learning off
1278         bridge-arp-nd-suppress on
1279         bridge-unicast-flood off
1280         bridge-multicast-flood off
1281
1282 auto vmbr3
1283 iface vmbr3 inet static
1284         bridge_ports vxlan3
1285         bridge_stp off
1286         bridge_fd 0
1287         address 10.0.3.254
1288         netmask 255.255.255.0
1289         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
1290         vrf vrf1
1291
1292 #interconnect vxlan-vfr l3vni
1293 auto vxlan4000
1294 iface vxlan4000 inet manual
1295         vxlan-local-tunnelip 192.168.0.2
1296         bridge-learning off
1297         bridge-arp-nd-suppress on
1298         bridge-unicast-flood off
1299         bridge-multicast-flood off
1300
1301
1302 auto vmbr4000
1303 iface vmbr4000 inet manual
1304         bridge_ports vxlan4000
1305         bridge_stp off
1306         bridge_fd 0
1307         hwaddress 44:39:39:FF:40:91  #must be different on each node
1308         vrf vrf1
1309 ----
1310
1311
1312 frr.conf
1313
1314 ----
1315 vrf vrf1
1316  vni 4000
1317 !
1318 router bgp 1234
1319  bgp router-id 192.168.0.2
1320  no bgp default ipv4-unicast
1321  coalesce-time 1000
1322  neighbor 192.168.0.1 remote-as 1234
1323  neighbor 192.168.0.3 remote-as 1234
1324  !
1325  address-family l2vpn evpn
1326   neighbor 192.168.0.1 activate
1327   neighbor 192.168.0.3 activate
1328   advertise-all-vni
1329  exit-address-family
1330 !
1331 router bgp 1234 vrf vrf1
1332 !
1333  bgp router-id 192.168.0.2
1334  !
1335  address-family ipv4 unicast
1336   redistribute connected
1337  exit-address-family
1338  !
1339  address-family l2vpn evpn
1340   advertise ipv4 unicast
1341  exit-address-family
1342 !
1343 line vty
1344 !
1345 ----
1346
1347
1348 * node3
1349
1350 ----
1351 auto vrf1
1352 iface vrf1
1353     vrf-table auto
1354
1355 auto eno1
1356 iface eno1 inet manual
1357
1358 auto vmbr0
1359 iface vmbr0 inet static
1360          address 192.168.0.3
1361          netmask  255.255.255.0
1362          bridge_ports eno1
1363          bridge_stp off
1364          bridge_fd 0
1365
1366 auto vxlan2
1367 iface vxlan2 inet manual
1368         vxlan-local-tunnelip 192.168.0.3
1369         bridge-learning off
1370         bridge-arp-nd-suppress on
1371         bridge-unicast-flood off
1372         bridge-multicast-flood off
1373
1374 auto vmbr2
1375 iface vmbr2 inet static
1376         bridge_ports vxlan2
1377         bridge_stp off
1378         bridge_fd 0
1379         address 10.0.2.254
1380         netmask 255.255.255.0
1381         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2
1382         vrf vrf1
1383
1384 auto vxlan3
1385 iface vxlan3 inet manual
1386         vxlan-local-tunnelip 192.168.0.3
1387         bridge-learning off
1388         bridge-arp-nd-suppress on
1389         bridge-unicast-flood off
1390         bridge-multicast-flood off
1391
1392 auto vmbr3
1393 iface vmbr3 inet static
1394         bridge_ports vxlan3
1395         bridge_stp off
1396         bridge_fd 0
1397         address 10.0.3.254
1398         netmask 255.255.255.0
1399         hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3
1400         vrf vrf1
1401
1402 #interconnect vxlan-vfr l3vni
1403 auto vxlan4000
1404 iface vxlan4000 inet manual
1405         vxlan-local-tunnelip 192.168.0.3
1406         bridge-learning off
1407         bridge-arp-nd-suppress on
1408         bridge-unicast-flood off
1409         bridge-multicast-flood off
1410
1411
1412 auto vmbr4000
1413 iface vmbr4000 inet manual
1414         bridge_ports vxlan4000
1415         bridge_stp off
1416         bridge_fd 0
1417         hwaddress 44:39:39:FF:40:92  #must be different on each node
1418         vrf vrf1
1419 ----
1420
1421
1422 frr.conf
1423
1424 ----
1425 vrf vrf1
1426  vni 4000
1427 !
1428 router bgp 1234
1429  bgp router-id 192.168.0.3
1430  no bgp default ipv4-unicast
1431  coalesce-time 1000
1432  neighbor 192.168.0.1 remote-as 1234
1433  neighbor 192.168.0.2 remote-as 1234
1434  !
1435  address-family l2vpn evpn
1436   neighbor 192.168.0.1 activate
1437   neighbor 192.168.0.2 activate
1438   advertise-all-vni
1439  exit-address-family
1440 !
1441 router bgp 1234 vrf vrf1
1442 !
1443  bgp router-id 192.168.0.3
1444  !
1445  address-family ipv4 unicast
1446   redistribute connected
1447  exit-address-family
1448  !
1449  address-family l2vpn evpn
1450   advertise ipv4 unicast
1451  exit-address-family
1452 !
1453 line vty
1454 !
1455 ----