Each overlay network is known as a VXLAN Segment and identified by a unique
24-bit segment ID called a VXLAN Network Identifier (VNI).
+VXLAN encapsulation add 50bytes overhead, so you need to increase mtu on your host
+physical interfaces to 1550 at minimum. (or decrease mtu inside your vms to 1450)
+
For BUM traffic (broadcast / unknown unicast traffic, multicast),
we have 3 differents vxlan setup modes : multicast, unicast, bgp-evpn
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-svcnodeip 225.20.1.1
vxlan-physdev eno1
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-svcnodeip 225.20.1.1
vxlan-physdev eno1
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-svcnodeip 225.20.1.1
vxlan-physdev eno1
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-svcnodeip 225.20.1.1
vxlan-physdev eno1
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-svcnodeip 225.20.1.1
vxlan-physdev eno1
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-svcnodeip 225.20.1.1
vxlan-physdev eno1
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan_remoteip 192.168.0.2
vxlan_remoteip 192.168.0.3
auto vxlan3
iface vxlan2 inet manual
+ vxlan-id 3
vxlan_remoteip 192.168.0.2
vxlan_remoteip 192.168.0.3
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan_remoteip 192.168.0.1
vxlan_remoteip 192.168.0.3
auto vxlan3
iface vxlan2 inet manual
+ vxlan-id 3
vxlan_remoteip 192.168.0.1
vxlan_remoteip 192.168.0.3
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan_remoteip 192.168.0.2
vxlan_remoteip 192.168.0.3
auto vxlan3
iface vxlan2 inet manual
+ vxlan-id 3
vxlan_remoteip 192.168.0.2
vxlan_remoteip 192.168.0.3
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
#enable routing
net.ipv4.ip_forward=1
net.ipv6.conf.all.forwarding=1
-#disable reverse path filtering
-net.ipv4.conf.default.rp_filter=0
-net.ipv4.conf.all.rp_filter=0
-#allow frr to work with vrf (kernel >4.14 bug)
-net.ipv4.tcp_l3mdev_accept=1
----
* node1
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
#interconnect vxlan-vfr l3vni
auto vxlan4000
iface vxlan4000 inet manual
+ vxlan-id 4000
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
bridge_ports vxlan4000
bridge_stp off
bridge_fd 0
- hwaddress 44:39:39:FF:40:90 #must be different on each node
vrf vrf1
----
----
vrf vrf1
vni 4000
+ exit-vrf
!
router bgp 1234
bgp router-id 192.168.0.1
advertise-all-vni
exit-address-family
!
-router bgp 1234 vrf vrf1
-!
- bgp router-id 192.168.0.1
- !
- address-family ipv4 unicast
- redistribute connected
- exit-address-family
- !
- address-family l2vpn evpn
- advertise ipv4 unicast
- exit-address-family
-!
line vty
!
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
#interconnect vxlan-vfr l3vni
auto vxlan4000
iface vxlan4000 inet manual
+ vxlan-id 4000
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
bridge_ports vxlan4000
bridge_stp off
bridge_fd 0
- hwaddress 44:39:39:FF:40:91 #must be different on each node
vrf vrf1
----
----
vrf vrf1
vni 4000
+ exit-vrf
!
router bgp 1234
bgp router-id 192.168.0.2
advertise-all-vni
exit-address-family
!
-router bgp 1234 vrf vrf1
-!
- bgp router-id 192.168.0.2
- !
- address-family ipv4 unicast
- redistribute connected
- exit-address-family
- !
- address-family l2vpn evpn
- advertise ipv4 unicast
- exit-address-family
-!
line vty
!
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
#interconnect vxlan-vfr l3vni
auto vxlan4000
iface vxlan4000 inet manual
+ vxlan-id 4000
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
bridge_ports vxlan4000
bridge_stp off
bridge_fd 0
- hwaddress 44:39:39:FF:40:92 #must be different on each node
vrf vrf1
----
----
vrf vrf1
vni 4000
+ exit-vrf
!
router bgp 1234
bgp router-id 192.168.0.3
advertise-all-vni
exit-address-family
!
-router bgp 1234 vrf vrf1
-!
- bgp router-id 192.168.0.3
- !
- address-family ipv4 unicast
- redistribute connected
- exit-address-family
- !
- address-family l2vpn evpn
- advertise ipv4 unicast
- exit-address-family
-!
line vty
!
----
1 gateway node
^^^^^^^^^^^^^^
In this example, we'll use only 1 proxmox node as exit gateway. (node1)
-This node have a simple default gw in the vrf to the external router (no bgp between router and node1)
-and announce this default gw to other proxmox nodes.
+This node announce the default gw in vrf1 (default originate) and forward to his own default gateway (192.168.0.254) (no bgp between router and node1)
*node1
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
address 192.168.0.1
netmask 255.255.255.0
+ gateway 192.168.0.254
bridge_ports eno1
bridge_stp off
bridge_fd 0
-auto eno2
-iface eno2
- address 172.16.0.1
- netmask 255.255.255.0
- vrf vrf1
- post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1
- #if you have multiple external routers, you can use ecmp balancing
- #post-up route add default nexthop via 172.16.0.253 dev eno2 vrf vrf1 nexthop via 172.16.0.254 dev eno2 vrf vrf1
-
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
#interconnect vxlan-vfr l3vni
auto vxlan4000
iface vxlan4000 inet manual
+ vxlan-id 4000
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
bridge_ports vxlan4000
bridge_stp off
bridge_fd 0
- hwaddress 44:39:39:FF:40:90 #must be different on each node
vrf vrf1
----
----
vrf vrf1
vni 4000
+ exit-vrf
!
router bgp 1234
bgp router-id 192.168.0.1
neighbor 192.168.0.2 remote-as 1234
neighbor 192.168.0.3 remote-as 1234
!
+ address-family ipv4 unicast
+ import vrf vrf1
+ exit-address-family
+ !
address-family l2vpn evpn
neighbor 192.168.0.2 activate
neighbor 192.168.0.3 activate
!
router bgp 1234 vrf vrf1
!
- bgp router-id 172.16.0.1
- !
- address-family ipv4 unicast
- redistribute connected
- redistribute kernel !announce your default gw to all nodes
- exit-address-family
- !
address-family l2vpn evpn
- advertise ipv4 unicast
+ default-originate ipv4
exit-address-family
!
line vty
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
#interconnect vxlan-vfr l3vni
auto vxlan4000
iface vxlan4000 inet manual
+ vxlan-id 4000
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
bridge_ports vxlan4000
bridge_stp off
bridge_fd 0
- hwaddress 44:39:39:FF:40:91 #must be different on each node
vrf vrf1
----
----
vrf vrf1
vni 4000
+ exit-vrf
!
router bgp 1234
bgp router-id 192.168.0.2
advertise-all-vni
exit-address-family
!
-router bgp 1234 vrf vrf1
-!
- bgp router-id 192.168.0.2
- !
- address-family ipv4 unicast
- redistribute connected
- exit-address-family
- !
- address-family l2vpn evpn
- advertise ipv4 unicast
- exit-address-family
-!
line vty
!
----
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
#interconnect vxlan-vfr l3vni
auto vxlan4000
iface vxlan4000 inet manual
+ vxlan-id 4000
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
bridge_ports vxlan4000
bridge_stp off
bridge_fd 0
- hwaddress 44:39:39:FF:40:92 #must be different on each node
vrf vrf1
----
----
vrf vrf1
vni 4000
+ exit-vrf
!
router bgp 1234
bgp router-id 192.168.0.3
advertise-all-vni
exit-address-family
!
-router bgp 1234 vrf vrf1
-!
- bgp router-id 192.168.0.3
- !
- address-family ipv4 unicast
- redistribute connected
- exit-address-family
- !
- address-family l2vpn evpn
- advertise ipv4 unicast
- exit-address-family
-!
line vty
!
----
multiple gateway nodes
^^^^^^^^^^^^^^^^^^^^^^
In this example, all nodes will be used as exit gateway. (But you can use only 2 nodes if you want)
-All nodes have a simple default gw in the vrf to the external router (no bgp between router and node1)
-and announce this default gw.
+All nodes have a a default gw to the external router (192.168.0.254) (no bgp between router and node1)
+and announce this default gw in the vrf (default originate)
The external router have ecmp routes to all proxmox nodes.(balancing).
If the router send the packet to a wrong node (vm is not on this node), this node will route through
vxlan the packet to final destination.
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
address 192.168.0.1
netmask 255.255.255.0
+ gateway 192.168.0.254
bridge_ports eno1
bridge_stp off
bridge_fd 0
-auto eno2
-iface eno2
- address 172.16.0.1
- netmask 255.255.255.0
- vrf vrf1
- post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1
- #if you have multiple external routers, you can use ecmp balancing
- #post-up route add default nexthop via 172.16.0.253 dev eno2 vrf vrf1 nexthop via 172.16.0.254 dev eno2 vrf vrf1
-
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
#interconnect vxlan-vfr l3vni
auto vxlan4000
iface vxlan4000 inet manual
+ vxlan-id 4000
vxlan-local-tunnelip 192.168.0.1
bridge-learning off
bridge-arp-nd-suppress on
bridge_ports vxlan4000
bridge_stp off
bridge_fd 0
- hwaddress 44:39:39:FF:40:90 #must be different on each node
vrf vrf1
----
----
vrf vrf1
vni 4000
+ exit-vrf
!
router bgp 1234
bgp router-id 192.168.0.1
neighbor 192.168.0.2 remote-as 1234
neighbor 192.168.0.3 remote-as 1234
!
+ address-family ipv4 unicast
+ import vrf vrf1
+ exit-address-family
+ !
address-family l2vpn evpn
neighbor 192.168.0.2 activate
neighbor 192.168.0.3 activate
!
router bgp 1234 vrf vrf1
!
- bgp router-id 172.16.0.1
- !
- address-family ipv4 unicast
- redistribute connected
- redistribute kernel !announce your default gw to all nodes
- exit-address-family
- !
address-family l2vpn evpn
- advertise ipv4 unicast
+ default-originate ipv4
exit-address-family
!
line vty
auto eno1
iface eno1 inet manual
+ mtu 1550
auto vmbr0
iface vmbr0 inet static
address 192.168.0.2
netmask 255.255.255.0
+ gateway 192.168.0.254
bridge_ports eno1
bridge_stp off
bridge_fd 0
-auto eno2
-iface eno2
- address 172.16.0.3
- netmask 255.255.255.0
- vrf vrf1
- post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1
- #if you have multiple external routers, you can use ecmp balancing
- #post-up route add default nexthop via 172.16.0.253 dev eno2 vrf vrf1 nexthop via 172.16.0.254 dev eno2 vrf vrf1
-
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
#interconnect vxlan-vfr l3vni
auto vxlan4000
iface vxlan4000 inet manual
+ vxlan-id 4000
vxlan-local-tunnelip 192.168.0.2
bridge-learning off
bridge-arp-nd-suppress on
bridge_ports vxlan4000
bridge_stp off
bridge_fd 0
- hwaddress 44:39:39:FF:40:91 #must be different on each node
vrf vrf1
----
----
vrf vrf1
vni 4000
+ exit-vrf
!
router bgp 1234
bgp router-id 192.168.0.2
neighbor 192.168.0.1 remote-as 1234
neighbor 192.168.0.3 remote-as 1234
!
+ address-family ipv4 unicast
+ import vrf vrf1
+ exit-address-family
+ !
address-family l2vpn evpn
neighbor 192.168.0.1 activate
neighbor 192.168.0.3 activate
advertise-all-vni
exit-address-family
!
-router bgp 1234 vrf vrf1
-!
- bgp router-id 172.16.0.2
- !
- address-family ipv4 unicast
- redistribute connected
- redistribute kernel !announce your default gw to all nodes
- exit-address-family
- !
address-family l2vpn evpn
- advertise ipv4 unicast
+ default-originate ipv4
exit-address-family
!
line vty
auto eno1
iface eno1 inet manual
-
+ mtu 1550
+
auto vmbr0
iface vmbr0 inet static
address 192.168.0.3
netmask 255.255.255.0
+ gateway 192.168.0.254
bridge_ports eno1
bridge_stp off
bridge_fd 0
-auto eno2
-iface eno2
- address 172.16.0.3
- netmask 255.255.255.0
- vrf vrf1
- post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1
- #if you have multiple external routers, you can use ecmp balancing
- #post-up route add default nexthop via 172.16.0.253 dev eno2 vrf vrf1 nexthop via 172.16.0.254 dev eno2 vrf vrf1
-
auto vxlan2
iface vxlan2 inet manual
+ vxlan-id 2
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
auto vxlan3
iface vxlan3 inet manual
+ vxlan-id 3
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
#interconnect vxlan-vfr l3vni
auto vxlan4000
iface vxlan4000 inet manual
+ vxlan-id 4000
vxlan-local-tunnelip 192.168.0.3
bridge-learning off
bridge-arp-nd-suppress on
bridge_ports vxlan4000
bridge_stp off
bridge_fd 0
- hwaddress 44:39:39:FF:40:92 #must be different on each node
vrf vrf1
----
----
vrf vrf1
vni 4000
+ exit-vrf
!
router bgp 1234
bgp router-id 192.168.0.3
neighbor 192.168.0.1 remote-as 1234
neighbor 192.168.0.2 remote-as 1234
!
+ address-family ipv4 unicast
+ import vrf vrf1
+ exit-address-family
+ !
address-family l2vpn evpn
neighbor 192.168.0.1 activate
neighbor 192.168.0.2 activate
!
router bgp 1234 vrf vrf1
!
- bgp router-id 172.16.0.3
- !
- address-family ipv4 unicast
- redistribute connected
- redistribute kernel !announce your default gw to all nodes
- exit-address-family
- !
address-family l2vpn evpn
- advertise ipv4 unicast
+ default-originate ipv4
exit-address-family
!
line vty
Note
^^^^
-If your external router don't support ecmp to reach multiple proxmox nodes,
+If your external router don't support ecmp static route to reach multiple proxmox nodes,
you can setup an HA floating vip on proxmox nodes with vrrp
-I this example, we will setup an floating 172.16.0.10 ip on node1 and node2.
+In this example, we will setup an floating 192.168.0.10 ip on node1 and node2.
Node1 is the primary and failover to node2 in case of failure.
+This setup need vrrpd package (apt install vrrpd).
+#TODO : It should be possible to do it with frr directly with last version.
* node1
----
-auto eno2
-iface eno2
- address 172.16.0.1
- netmask 255.255.255.0
- vrf vrf1
- post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1
- vrrp-id 1
- vrrp-priority 1
- vrrp-virtual-ip 172.16.0.10
+auto vmbr0
+iface vmbr0 inet static
+ address 192.168.0.1
+ netmask 255.255.255.0
+ gateway 192.168.0.254
+ bridge_ports eno1
+ bridge_stp off
+ bridge_fd 0
+ vrrp-id 1
+ vrrp-priority 1
+ vrrp-virtual-ip 192.168.0.10
----
* node2
----
-auto eno2
-iface eno2
- address 172.16.0.2
- netmask 255.255.255.0
- vrf vrf1
- post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1
- vrrp-id 1
- vrrp-priority 2
- vrrp-virtual-ip 172.16.0.10
+auto vmbr0
+iface vmbr0 inet static
+ address 192.168.0.2
+ netmask 255.255.255.0
+ gateway 192.168.0.254
+ bridge_ports eno1
+ bridge_stp off
+ bridge_fd 0
+ vrrp-id 1
+ vrrp-priority 2
+ vrrp-virtual-ip 192.168.0.10
----
+#TODO : Documentation with bgp upstream router.
projects / pve-docs.git / blobdiff