X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=vxlan-and-evpn.adoc;h=12bcea90f64e9490e0edd42b76d6457ad3463a9d;hb=9a08108970ee1fd321806776f1ede67bb9a05cc1;hp=8e563822eeb8573cf3c71bcc55bf10510cd8250f;hpb=8adeb0eb177b936b03a6cf5430967cb841edef8f;p=pve-docs.git diff --git a/vxlan-and-evpn.adoc b/vxlan-and-evpn.adoc index 8e56382..12bcea9 100644 --- a/vxlan-and-evpn.adoc +++ b/vxlan-and-evpn.adoc @@ -20,7 +20,7 @@ VXLAN encapsulation add 50bytes overhead, so you need to increase mtu on your ho physical interfaces to 1550 at minimum. (or decrease mtu inside your vms to 1450) For BUM traffic (broadcast / unknown unicast traffic, multicast), -we have 3 differents vxlan setup modes : multicast, unicast, bgp-evpn +we have 3 different VXLAN setup modes : multicast, unicast, bgp-evpn image::images/vxlan-l2-vlanunaware.svg["vxlan l2 bridge vlan unaware",align="center"] @@ -542,15 +542,6 @@ in each direction (always the destination VNI) across the routed infrastructure. image::images/vxlan-l3-asymmetric.svg["vxlan l3 asymmetric",align="center"] - -sysctl.conf tuning - ----- -#enable routing -net.ipv4.ip_forward=1 -net.ipv6.conf.all.forwarding=1 ----- - * node1 ---- @@ -584,7 +575,9 @@ iface vmbr2 inet static bridge_ports vxlan2 bridge_stp off bridge_fd 0 - + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -604,6 +597,9 @@ iface vmbr3 inet static bridge_ports vxlan3 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on + arp-accept on ---- @@ -661,6 +657,9 @@ iface vmbr2 inet static bridge_ports vxlan2 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 @@ -681,6 +680,9 @@ iface vmbr3 inet static bridge_ports vxlan3 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on + arp-accept on ---- @@ -738,7 +740,9 @@ iface vmbr2 inet static bridge_ports vxlan2 bridge_stp off bridge_fd 0 - + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -749,7 +753,6 @@ iface vxlan3 inet manual bridge-unicast-flood off bridge-multicast-flood off - auto vmbr3 iface vmbr3 inet static address 10.0.3.254 @@ -758,6 +761,9 @@ iface vmbr3 inet static bridge_ports vxlan3 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on + arp-accept on ---- @@ -798,14 +804,6 @@ A vrf is needed for the L3VNI, so all vmbr bridge need to be in the vrf if they image::images/vxlan-l3-symmetric.svg["vxlan l3 symmetric",align="center"] -sysctl.conf tuning - ----- -#enable routing -net.ipv4.ip_forward=1 -net.ipv6.conf.all.forwarding=1 ----- - * node1 ---- @@ -843,6 +841,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -862,6 +863,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -944,6 +948,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -963,6 +970,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1046,6 +1056,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1065,6 +1078,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1110,8 +1126,8 @@ line vty ! ---- -VXLAN layer3 routing with anycast gateway + routing to outside with external router -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +VXLAN layer3 routing with anycast gateway + routing to outside with external router with static default gw +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Routing to outside need the symmetric model. 1 gateway node @@ -1139,6 +1155,8 @@ iface vmbr0 inet static bridge_ports eno1 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on auto vxlan2 iface vxlan2 inet manual @@ -1158,6 +1176,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1177,6 +1198,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1215,6 +1239,10 @@ router bgp 1234 import vrf vrf1 exit-address-family ! + address-family ipv6 unicast + import vrf vrf1 + exit-address-family + ! address-family l2vpn evpn neighbor 192.168.0.2 activate neighbor 192.168.0.3 activate @@ -1223,8 +1251,17 @@ router bgp 1234 ! router bgp 1234 vrf vrf1 ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + exit-address-family + ! address-family l2vpn evpn default-originate ipv4 + default-originate ipv6 exit-address-family ! line vty @@ -1269,6 +1306,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1288,6 +1328,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1371,6 +1414,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1390,6 +1436,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1444,6 +1493,16 @@ The external router have ecmp routes to all proxmox nodes.(balancing). If the router send the packet to a wrong node (vm is not on this node), this node will route through vxlan the packet to final destination. +If you have multiple gateway nodes, disable rp_filter as packet could incoming in a 1 node, and outgoing +to another node. + +sysctl.conf tuning +----- +net.ipv4.conf.default.rp_filter=0 +net.ipv4.conf.all.rp_filter=0 +----- + + *node1 ---- @@ -1463,6 +1522,8 @@ iface vmbr0 inet static bridge_ports eno1 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on auto vxlan2 iface vxlan2 inet manual @@ -1482,6 +1543,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1501,6 +1565,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1539,6 +1606,10 @@ router bgp 1234 import vrf vrf1 exit-address-family ! + address-family ipv6 unicast + import vrf vrf1 + exit-address-family + ! address-family l2vpn evpn neighbor 192.168.0.2 activate neighbor 192.168.0.3 activate @@ -1547,8 +1618,17 @@ router bgp 1234 ! router bgp 1234 vrf vrf1 ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + exit-address-family + ! address-family l2vpn evpn default-originate ipv4 + default-originate ipv6 exit-address-family ! line vty @@ -1575,6 +1655,8 @@ iface vmbr0 inet static bridge_ports eno1 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on auto vxlan2 iface vxlan2 inet manual @@ -1594,6 +1676,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1613,6 +1698,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1652,14 +1740,27 @@ router bgp 1234 import vrf vrf1 exit-address-family ! + address-family ipv6 unicast + import vrf vrf1 + exit-address-family + ! address-family l2vpn evpn neighbor 192.168.0.1 activate neighbor 192.168.0.3 activate advertise-all-vni exit-address-family ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + exit-address-family + ! address-family l2vpn evpn default-originate ipv4 + default-originate ipv6 exit-address-family ! line vty @@ -1686,6 +1787,8 @@ iface vmbr0 inet static bridge_ports eno1 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on auto vxlan2 iface vxlan2 inet manual @@ -1705,6 +1808,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1724,6 +1830,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1763,6 +1872,10 @@ router bgp 1234 import vrf vrf1 exit-address-family ! + address-family ipv6 unicast + import vrf vrf1 + exit-address-family + ! address-family l2vpn evpn neighbor 192.168.0.1 activate neighbor 192.168.0.2 activate @@ -1771,8 +1884,17 @@ router bgp 1234 ! router bgp 1234 vrf vrf1 ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + exit-address-family + ! address-family l2vpn evpn default-originate ipv4 + default-originate ipv6 exit-address-family ! line vty @@ -1825,6 +1947,91 @@ iface vmbr0 inet static ---- + +gateway node(s) with a upstream bgp router +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Setup is almost the same than with a static gateway, but we'll connect to an upstream bgp router. + +example with node1 as gateway (192.168.0.1) for evpn-bgp, and an upstream bgp router (running frr too) 192.168.0.254. + +* node1 + +frr.conf +---- +vrf vrf1 + vni 4000 + exit-vrf +! +router bgp 1234 + bgp router-id 192.168.0.1 + no bgp default ipv4-unicast + coalesce-time 1000 + neighbor 192.168.0.2 remote-as 1234 + neighbor 192.168.0.3 remote-as 1234 + neighbor 192.168.0.254 remote-as external + ! + address-family ipv4 unicast + import vrf vrf1 + neighbor 192.168.0.254 activate + exit-address-family + ! + address-family ipv6 unicast + import vrf vrf1 + neighbor 192.168.0.254 activate + exit-address-family + ! + address-family l2vpn evpn + neighbor 192.168.0.1 activate + neighbor 192.168.0.2 activate + neighbor 192.168.0.254 activate + advertise-all-vni + exit-address-family +! +router bgp 1234 vrf vrf1 +! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + exit-address-family + ! + address-family l2vpn evpn + default-originate ipv4 + default-originate ipv6 + exit-address-family +! +line vty +! +---- + +* bgp router + +frr.conf +---- +ip prefix-list NO32 seq 10 permit 0.0.0.0/0 ge 8 le 24 +ip prefix-list NO32 seq 20 deny any +! +router bgp 25253 + bgp router-id 192.168.0.254 + bgp bestpath as-path multipath-relax + neighbor 192.168.0.1 remote-as external + neighbor 192.168.0.1 capability extended-nexthop + ! + address-family ipv4 unicast + neighbor 192.168.0.1 default-originate + neighbor 192.168.0.1 prefix-list NO32 in #don't import /32 route from evpn + exit-address-family + ! + address-family ipv6 unicast + neighbor 192.168.0.1 default-originate + neighbor 192.168.0.1 prefix-list NO32 in #don't import /32 route from evpn + exit-address-family + ! +! +--- + Route Reflectors ^^^^^^^^^^^^^^^^ If you have a lot of proxmox nodes, or multiple proxmox clusters, you may want @@ -1891,10 +2098,6 @@ router bgp 1234 neighbor 192.168.0.200 remote-as 1234 neighbor 192.168.0.201 remote-as 1234 ! - address-family ipv4 unicast - import vrf vrf1 - exit-address-family - ! address-family l2vpn evpn neighbor 192.168.0.200 activate neighbor 192.168.0.201 activate @@ -1902,5 +2105,3 @@ router bgp 1234 exit-address-family ! ---- - -#TODO : Documentation with bgp upstream router.