X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=vxlan-and-evpn.adoc;h=ec1bc0701ffd17a6f848c94108c5761cde5af020;hb=2f48e8867eddaa9adb4262f65105f2bb1ffce7e5;hp=8e563822eeb8573cf3c71bcc55bf10510cd8250f;hpb=8adeb0eb177b936b03a6cf5430967cb841edef8f;p=pve-docs.git diff --git a/vxlan-and-evpn.adoc b/vxlan-and-evpn.adoc index 8e56382..ec1bc07 100644 --- a/vxlan-and-evpn.adoc +++ b/vxlan-and-evpn.adoc @@ -542,15 +542,6 @@ in each direction (always the destination VNI) across the routed infrastructure. image::images/vxlan-l3-asymmetric.svg["vxlan l3 asymmetric",align="center"] - -sysctl.conf tuning - ----- -#enable routing -net.ipv4.ip_forward=1 -net.ipv6.conf.all.forwarding=1 ----- - * node1 ---- @@ -584,7 +575,9 @@ iface vmbr2 inet static bridge_ports vxlan2 bridge_stp off bridge_fd 0 - + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -604,6 +597,9 @@ iface vmbr3 inet static bridge_ports vxlan3 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on + arp-accept on ---- @@ -661,6 +657,9 @@ iface vmbr2 inet static bridge_ports vxlan2 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 @@ -681,6 +680,9 @@ iface vmbr3 inet static bridge_ports vxlan3 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on + arp-accept on ---- @@ -738,7 +740,9 @@ iface vmbr2 inet static bridge_ports vxlan2 bridge_stp off bridge_fd 0 - + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -749,7 +753,6 @@ iface vxlan3 inet manual bridge-unicast-flood off bridge-multicast-flood off - auto vmbr3 iface vmbr3 inet static address 10.0.3.254 @@ -758,6 +761,9 @@ iface vmbr3 inet static bridge_ports vxlan3 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on + arp-accept on ---- @@ -798,14 +804,6 @@ A vrf is needed for the L3VNI, so all vmbr bridge need to be in the vrf if they image::images/vxlan-l3-symmetric.svg["vxlan l3 symmetric",align="center"] -sysctl.conf tuning - ----- -#enable routing -net.ipv4.ip_forward=1 -net.ipv6.conf.all.forwarding=1 ----- - * node1 ---- @@ -843,6 +841,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -862,6 +863,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -944,6 +948,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -963,6 +970,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1046,6 +1056,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1065,6 +1078,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1110,8 +1126,8 @@ line vty ! ---- -VXLAN layer3 routing with anycast gateway + routing to outside with external router -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +VXLAN layer3 routing with anycast gateway + routing to outside with external router with static default gw +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Routing to outside need the symmetric model. 1 gateway node @@ -1139,6 +1155,8 @@ iface vmbr0 inet static bridge_ports eno1 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on auto vxlan2 iface vxlan2 inet manual @@ -1158,6 +1176,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1177,6 +1198,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1215,6 +1239,10 @@ router bgp 1234 import vrf vrf1 exit-address-family ! + address-family ipv6 unicast + import vrf vrf1 + exit-address-family + ! address-family l2vpn evpn neighbor 192.168.0.2 activate neighbor 192.168.0.3 activate @@ -1223,8 +1251,17 @@ router bgp 1234 ! router bgp 1234 vrf vrf1 ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + exit-address-family + ! address-family l2vpn evpn default-originate ipv4 + default-originate ipv6 exit-address-family ! line vty @@ -1269,6 +1306,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1288,6 +1328,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1371,6 +1414,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1390,6 +1436,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1444,6 +1493,16 @@ The external router have ecmp routes to all proxmox nodes.(balancing). If the router send the packet to a wrong node (vm is not on this node), this node will route through vxlan the packet to final destination. +If you have multiple gateway nodes, disable rp_filter as packet could incoming in a 1 node, and outgoing +to another node. + +sysctl.conf tuning +----- +net.ipv4.conf.default.rp_filter=0 +net.ipv4.conf.all.rp_filter=0 +----- + + *node1 ---- @@ -1463,6 +1522,8 @@ iface vmbr0 inet static bridge_ports eno1 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on auto vxlan2 iface vxlan2 inet manual @@ -1482,6 +1543,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1501,6 +1565,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1539,6 +1606,10 @@ router bgp 1234 import vrf vrf1 exit-address-family ! + address-family ipv6 unicast + import vrf vrf1 + exit-address-family + ! address-family l2vpn evpn neighbor 192.168.0.2 activate neighbor 192.168.0.3 activate @@ -1547,8 +1618,17 @@ router bgp 1234 ! router bgp 1234 vrf vrf1 ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + exit-address-family + ! address-family l2vpn evpn default-originate ipv4 + default-originate ipv6 exit-address-family ! line vty @@ -1575,6 +1655,8 @@ iface vmbr0 inet static bridge_ports eno1 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on auto vxlan2 iface vxlan2 inet manual @@ -1594,6 +1676,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1613,6 +1698,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1652,14 +1740,27 @@ router bgp 1234 import vrf vrf1 exit-address-family ! + address-family ipv6 unicast + import vrf vrf1 + exit-address-family + ! address-family l2vpn evpn neighbor 192.168.0.1 activate neighbor 192.168.0.3 activate advertise-all-vni exit-address-family ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + exit-address-family + ! address-family l2vpn evpn default-originate ipv4 + default-originate ipv6 exit-address-family ! line vty @@ -1686,6 +1787,8 @@ iface vmbr0 inet static bridge_ports eno1 bridge_stp off bridge_fd 0 + ip-forward on + ip6-forward on auto vxlan2 iface vxlan2 inet manual @@ -1705,6 +1808,9 @@ iface vmbr2 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr2 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on auto vxlan3 iface vxlan3 inet manual @@ -1724,6 +1830,9 @@ iface vmbr3 inet static netmask 255.255.255.0 hwaddress 44:39:39:FF:40:94 #must be same on each node vmbr3 vrf vrf1 + ip-forward on + ip6-forward on + arp-accept on #interconnect vxlan-vfr l3vni auto vxlan4000 @@ -1763,6 +1872,10 @@ router bgp 1234 import vrf vrf1 exit-address-family ! + address-family ipv6 unicast + import vrf vrf1 + exit-address-family + ! address-family l2vpn evpn neighbor 192.168.0.1 activate neighbor 192.168.0.2 activate @@ -1771,8 +1884,17 @@ router bgp 1234 ! router bgp 1234 vrf vrf1 ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + exit-address-family + ! address-family l2vpn evpn default-originate ipv4 + default-originate ipv6 exit-address-family ! line vty @@ -1825,6 +1947,91 @@ iface vmbr0 inet static ---- + +gateway node(s) with a upstream bgp router +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Setup is almost the same than with a static gateway, but we'll connect to an upstream bgp router. + +example with node1 as gateway (192.168.0.1) for evpn-bgp, and an upstream bgp router (running frr too) 192.168.0.254. + +* node1 + +frr.conf +---- +vrf vrf1 + vni 4000 + exit-vrf +! +router bgp 1234 + bgp router-id 192.168.0.1 + no bgp default ipv4-unicast + coalesce-time 1000 + neighbor 192.168.0.2 remote-as 1234 + neighbor 192.168.0.3 remote-as 1234 + neighbor 192.168.0.254 remote-as external + ! + address-family ipv4 unicast + import vrf vrf1 + neighbor 192.168.0.254 activate + exit-address-family + ! + address-family ipv6 unicast + import vrf vrf1 + neighbor 192.168.0.254 activate + exit-address-family + ! + address-family l2vpn evpn + neighbor 192.168.0.1 activate + neighbor 192.168.0.2 activate + neighbor 192.168.0.254 activate + advertise-all-vni + exit-address-family +! +router bgp 1234 vrf vrf1 +! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + exit-address-family + ! + address-family l2vpn evpn + default-originate ipv4 + default-originate ipv6 + exit-address-family +! +line vty +! +---- + +* bgp router + +frr.conf +---- +ip prefix-list NO32 seq 10 permit 0.0.0.0/0 ge 8 le 24 +ip prefix-list NO32 seq 20 deny any +! +router bgp 25253 + bgp router-id 192.168.0.254 + bgp bestpath as-path multipath-relax + neighbor 192.168.0.1 remote-as external + neighbor 192.168.0.1 capability extended-nexthop + ! + address-family ipv4 unicast + neighbor 192.168.0.1 default-originate + neighbor 192.168.0.1 prefix-list NO32 in #don't import /32 route from evpn + exit-address-family + ! + address-family ipv6 unicast + neighbor 192.168.0.1 default-originate + neighbor 192.168.0.1 prefix-list NO32 in #don't import /32 route from evpn + exit-address-family + ! +! +--- + Route Reflectors ^^^^^^^^^^^^^^^^ If you have a lot of proxmox nodes, or multiple proxmox clusters, you may want @@ -1891,10 +2098,6 @@ router bgp 1234 neighbor 192.168.0.200 remote-as 1234 neighbor 192.168.0.201 remote-as 1234 ! - address-family ipv4 unicast - import vrf vrf1 - exit-address-family - ! address-family l2vpn evpn neighbor 192.168.0.200 activate neighbor 192.168.0.201 activate @@ -1902,5 +2105,3 @@ router bgp 1234 exit-address-family ! ---- - -#TODO : Documentation with bgp upstream router.