1 package PVE
::Network
::SDN
::Controllers
::EvpnPlugin
;
7 use PVE
::JSONSchema
qw(get_standard_option);
8 use PVE
::Tools
qw(run_command file_set_contents file_get_contents);
9 use PVE
::RESTEnvironment
qw(log_warn);
11 use PVE
::Network
::SDN
::Controllers
::Plugin
;
12 use PVE
::Network
::SDN
::Zones
::Plugin
;
15 use base
('PVE::Network::SDN::Controllers::Plugin');
25 description
=> "autonomous system number",
30 description
=> "peers address list.",
31 type
=> 'string', format
=> 'ip-list'
38 'asn' => { optional
=> 0 },
39 'peers' => { optional
=> 0 },
43 # Plugin implementation
44 sub generate_controller_config
{
45 my ($class, $plugin_config, $controller_cfg, $id, $uplinks, $config) = @_;
48 @peers = PVE
::Tools
::split_list
($plugin_config->{'peers'}) if $plugin_config->{'peers'};
50 my $local_node = PVE
::INotify
::nodename
();
52 my $asn = $plugin_config->{asn
};
56 my $bgprouter = find_bgp_controller
($local_node, $controller_cfg);
57 my $isisrouter = find_isis_controller
($local_node, $controller_cfg);
60 $ebgp = 1 if $plugin_config->{'asn'} ne $bgprouter->{asn
};
61 $loopback = $bgprouter->{loopback
} if $bgprouter->{loopback
};
62 $asn = $bgprouter->{asn
} if $bgprouter->{asn
};
63 $autortas = $plugin_config->{'asn'} if $ebgp;
64 } elsif ($isisrouter) {
65 $loopback = $isisrouter->{loopback
} if $isisrouter->{loopback
};
70 my $bgp = $config->{frr
}->{router
}->{"bgp $asn"} //= {};
72 my ($ifaceip, $interface) = PVE
::Network
::SDN
::Zones
::Plugin
::find_local_ip_interface_peers
(\
@peers, $loopback);
74 my $remoteas = $ebgp ?
"external" : $asn;
77 my @controller_config = (
78 "bgp router-id $ifaceip",
79 "no bgp hard-administrative-reset",
80 "no bgp graceful-restart notification",
81 "no bgp default ipv4-unicast",
85 push(@{$bgp->{""}}, @controller_config) if keys %{$bgp} == 0;
87 @controller_config = ();
90 push @controller_config, "neighbor VTEP peer-group";
91 push @controller_config, "neighbor VTEP remote-as $remoteas";
92 push @controller_config, "neighbor VTEP bfd";
94 push @controller_config, "neighbor VTEP ebgp-multihop 10" if $ebgp && $loopback;
95 push @controller_config, "neighbor VTEP update-source $loopback" if $loopback;
98 foreach my $address (@peers) {
99 next if $address eq $ifaceip;
100 push @controller_config, "neighbor $address peer-group VTEP";
103 push(@{$bgp->{""}}, @controller_config);
105 # address-family l2vpn
106 @controller_config = ();
107 push @controller_config, "neighbor VTEP route-map MAP_VTEP_IN in";
108 push @controller_config, "neighbor VTEP route-map MAP_VTEP_OUT out";
109 push @controller_config, "neighbor VTEP activate";
110 push @controller_config, "advertise-all-vni";
111 push @controller_config, "autort as $autortas" if $autortas;
112 push(@{$bgp->{"address-family"}->{"l2vpn evpn"}}, @controller_config);
114 my $routemap = { rule
=> undef, action
=> "permit" };
115 push(@{$config->{frr_routemap
}->{'MAP_VTEP_IN'}}, $routemap );
116 push(@{$config->{frr_routemap
}->{'MAP_VTEP_OUT'}}, $routemap );
121 sub generate_controller_zone_config
{
122 my ($class, $plugin_config, $controller, $controller_cfg, $id, $uplinks, $config) = @_;
124 my $local_node = PVE
::INotify
::nodename
();
127 my $vrfvxlan = $plugin_config->{'vrf-vxlan'};
128 my $exitnodes = $plugin_config->{'exitnodes'};
129 my $exitnodes_primary = $plugin_config->{'exitnodes-primary'};
130 my $advertisesubnets = $plugin_config->{'advertise-subnets'};
131 my $exitnodes_local_routing = $plugin_config->{'exitnodes-local-routing'};
133 $rt_import = [PVE
::Tools
::split_list
($plugin_config->{'rt-import'})] if $plugin_config->{'rt-import'};
135 my $asn = $controller->{asn
};
137 @peers = PVE
::Tools
::split_list
($controller->{'peers'}) if $controller->{'peers'};
139 my $loopback = undef;
140 my $autortas = undef;
141 my $bgprouter = find_bgp_controller
($local_node, $controller_cfg);
142 my $isisrouter = find_isis_controller
($local_node, $controller_cfg);
145 $ebgp = 1 if $controller->{'asn'} ne $bgprouter->{asn
};
146 $loopback = $bgprouter->{loopback
} if $bgprouter->{loopback
};
147 $asn = $bgprouter->{asn
} if $bgprouter->{asn
};
148 $autortas = $controller->{'asn'} if $ebgp;
149 } elsif ($isisrouter) {
150 $loopback = $isisrouter->{loopback
} if $isisrouter->{loopback
};
153 return if !$vrf || !$vrfvxlan || !$asn;
155 my ($ifaceip, $interface) = PVE
::Network
::SDN
::Zones
::Plugin
::find_local_ip_interface_peers
(\
@peers, $loopback);
156 my $is_gateway = $exitnodes->{$local_node};
159 my @controller_config = ();
160 push @controller_config, "vni $vrfvxlan";
161 #avoid to routes between nodes through the exit nodes
162 #null routes subnets of other zones
164 my $subnets = PVE
::Network
::SDN
::Vnets
::get_subnets
();
165 foreach my $subnetid (sort keys %{$subnets}) {
166 my $subnet = $subnets->{$subnetid};
167 my $cidr = $subnet->{cidr
};
168 my $zone = $subnet->{zone
};
169 push @controller_config, "ip route $cidr null0" if $zone ne $id;
173 push(@{$config->{frr
}->{vrf
}->{"$vrf"}}, @controller_config);
176 @controller_config = ();
177 push @controller_config, "bgp router-id $ifaceip";
178 push @controller_config, "no bgp hard-administrative-reset";
179 push @controller_config, "no bgp graceful-restart notification";
181 # push @controller_config, "!";
182 push(@{$config->{frr
}->{router
}->{"bgp $asn vrf $vrf"}->{""}}, @controller_config);
185 push(@{$config->{frr
}->{router
}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"l2vpn evpn"}}, "route-target import $autortas:$vrfvxlan");
186 push(@{$config->{frr
}->{router
}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"l2vpn evpn"}}, "route-target export $autortas:$vrfvxlan");
191 $config->{frr_prefix_list
}->{'only_default'}->{1} = "permit 0.0.0.0/0";
192 $config->{frr_prefix_list_v6
}->{'only_default_v6'}->{1} = "permit ::/0";
194 if (!$exitnodes_primary || $exitnodes_primary eq $local_node) {
195 #filter default route coming from other exit nodes on primary node or both nodes if no primary is defined.
196 my $routemap_config_v6 = ();
197 push @{$routemap_config_v6}, "match ip address prefix-list only_default_v6";
198 my $routemap_v6 = { rule
=> $routemap_config_v6, action
=> "deny" };
199 unshift(@{$config->{frr_routemap
}->{'MAP_VTEP_IN'}}, $routemap_v6);
201 my $routemap_config = ();
202 push @{$routemap_config}, "match ip address prefix-list only_default";
203 my $routemap = { rule
=> $routemap_config, action
=> "deny" };
204 unshift(@{$config->{frr_routemap
}->{'MAP_VTEP_IN'}}, $routemap);
206 } elsif ($exitnodes_primary ne $local_node) {
207 my $routemap_config_v6 = ();
208 push @{$routemap_config_v6}, "match ipv6 address prefix-list only_default_v6";
209 push @{$routemap_config_v6}, "set metric 200";
210 my $routemap_v6 = { rule
=> $routemap_config_v6, action
=> "permit" };
211 unshift(@{$config->{frr_routemap
}->{'MAP_VTEP_OUT'}}, $routemap_v6);
213 my $routemap_config = ();
214 push @{$routemap_config}, "match ip address prefix-list only_default";
215 push @{$routemap_config}, "set metric 200";
216 my $routemap = { rule
=> $routemap_config, action
=> "permit" };
217 unshift(@{$config->{frr_routemap
}->{'MAP_VTEP_OUT'}}, $routemap);
220 if (!$exitnodes_local_routing) {
221 @controller_config = ();
222 #import /32 routes of evpn network from vrf1 to default vrf (for packet return)
223 push @controller_config, "import vrf $vrf";
224 push(@{$config->{frr
}->{router
}->{"bgp $asn"}->{"address-family"}->{"ipv4 unicast"}}, @controller_config);
225 push(@{$config->{frr
}->{router
}->{"bgp $asn"}->{"address-family"}->{"ipv6 unicast"}}, @controller_config);
227 @controller_config = ();
228 #redistribute connected to be able to route to local vms on the gateway
229 push @controller_config, "redistribute connected";
230 push(@{$config->{frr
}->{router
}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"ipv4 unicast"}}, @controller_config);
231 push(@{$config->{frr
}->{router
}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"ipv6 unicast"}}, @controller_config);
234 @controller_config = ();
235 #add default originate to announce 0.0.0.0/0 type5 route in evpn
236 push @controller_config, "default-originate ipv4";
237 push @controller_config, "default-originate ipv6";
238 push(@{$config->{frr
}->{router
}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"l2vpn evpn"}}, @controller_config);
239 } elsif ($advertisesubnets) {
241 @controller_config = ();
242 #redistribute connected networks
243 push @controller_config, "redistribute connected";
244 push(@{$config->{frr
}->{router
}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"ipv4 unicast"}}, @controller_config);
245 push(@{$config->{frr
}->{router
}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"ipv6 unicast"}}, @controller_config);
247 @controller_config = ();
248 #advertise connected networks type5 route in evpn
249 push @controller_config, "advertise ipv4 unicast";
250 push @controller_config, "advertise ipv6 unicast";
251 push(@{$config->{frr
}->{router
}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"l2vpn evpn"}}, @controller_config);
255 @controller_config = ();
256 foreach my $rt (sort @{$rt_import}) {
257 push @controller_config, "route-target import $rt";
259 push(@{$config->{frr
}->{router
}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"l2vpn evpn"}}, @controller_config);
265 sub generate_controller_vnet_config
{
266 my ($class, $plugin_config, $controller, $zone, $zoneid, $vnetid, $config) = @_;
268 my $exitnodes = $zone->{'exitnodes'};
269 my $exitnodes_local_routing = $zone->{'exitnodes-local-routing'};
271 return if !$exitnodes_local_routing;
273 my $local_node = PVE
::INotify
::nodename
();
274 my $is_gateway = $exitnodes->{$local_node};
276 return if !$is_gateway;
278 my $subnets = PVE
::Network
::SDN
::Vnets
::get_subnets
($vnetid, 1);
279 my @controller_config = ();
280 foreach my $subnetid (sort keys %{$subnets}) {
281 my $subnet = $subnets->{$subnetid};
282 my $cidr = $subnet->{cidr
};
283 push @controller_config, "ip route $cidr 10.255.255.2 xvrf_$zoneid";
285 push(@{$config->{frr_ip_protocol
}}, @controller_config);
289 my ($class, $controllerid, $zone_cfg) = @_;
291 # verify that zone is associated to this controller
292 foreach my $id (keys %{$zone_cfg->{ids
}}) {
293 my $zone = $zone_cfg->{ids
}->{$id};
294 die "controller $controllerid is used by $id"
295 if (defined($zone->{controller
}) && $zone->{controller
} eq $controllerid);
300 my ($class, $controllerid, $controller_cfg) = @_;
302 # we can only have 1 evpn controller / 1 asn by server
304 my $controllernb = 0;
305 foreach my $id (keys %{$controller_cfg->{ids
}}) {
306 next if $id eq $controllerid;
307 my $controller = $controller_cfg->{ids
}->{$id};
308 next if $controller->{type
} ne "evpn";
310 die "only 1 global evpn controller can be defined" if $controllernb >= 1;
314 sub find_bgp_controller
{
315 my ($nodename, $controller_cfg) = @_;
318 foreach my $id (keys %{$controller_cfg->{ids
}}) {
319 my $controller = $controller_cfg->{ids
}->{$id};
320 next if $controller->{type
} ne 'bgp';
321 next if $controller->{node
} ne $nodename;
328 sub find_isis_controller
{
329 my ($nodename, $controller_cfg) = @_;
332 foreach my $id (keys %{$controller_cfg->{ids
}}) {
333 my $controller = $controller_cfg->{ids
}->{$id};
334 next if $controller->{type
} ne 'isis';
335 next if $controller->{node
} ne $nodename;
342 sub generate_frr_recurse
{
343 my ($final_config, $content, $parentkey, $level) = @_;
346 $keylist->{'address-family'} = 1;
347 $keylist->{router
} = 1;
349 my $exitkeylist = {};
350 $exitkeylist->{'address-family'} = 1;
352 my $simple_exitkeylist = {};
353 $simple_exitkeylist->{router
} = 1;
355 # FIXME: make this generic
356 my $paddinglevel = undef;
357 if ($level == 1 || $level == 2) {
358 $paddinglevel = $level - 1;
359 } elsif ($level == 3 || $level == 4) {
360 $paddinglevel = $level - 2;
364 $padding = ' ' x
($paddinglevel) if $paddinglevel;
366 if (ref $content eq 'HASH') {
367 foreach my $key (sort keys %$content) {
368 next if $key eq 'vrf';
369 if ($parentkey && defined($keylist->{$parentkey})) {
370 push @{$final_config}, $padding."!";
371 push @{$final_config}, $padding."$parentkey $key";
372 } elsif ($key ne '' && !defined($keylist->{$key})) {
373 push @{$final_config}, $padding."$key";
376 my $option = $content->{$key};
377 generate_frr_recurse
($final_config, $option, $key, $level+1);
379 push @{$final_config}, $padding."exit-$parentkey" if $parentkey && defined($exitkeylist->{$parentkey});
380 push @{$final_config}, $padding."exit" if $parentkey && defined($simple_exitkeylist->{$parentkey});
384 if (ref $content eq 'ARRAY') {
385 push @{$final_config}, map { $padding . "$_" } @$content;
389 sub generate_frr_vrf
{
390 my ($final_config, $vrfs) = @_;
396 foreach my $id (sort keys %$vrfs) {
397 my $vrf = $vrfs->{$id};
399 push @config, "vrf $id";
400 foreach my $rule (@$vrf) {
401 push @config, " $rule";
404 push @config, "exit-vrf";
407 push @{$final_config}, @config;
410 sub generate_frr_ip_protocol
{
411 my ($final_config, $ips) = @_;
416 push @{$final_config}, "!";
417 foreach my $rule (sort @$ips) {
418 push @{$final_config}, $rule;
423 sub generate_frr_interfaces
{
424 my ($final_config, $interfaces) = @_;
426 foreach my $k (sort keys %$interfaces) {
427 my $iface = $interfaces->{$k};
428 push @{$final_config}, "!";
429 push @{$final_config}, "interface $k";
430 foreach my $rule (sort @$iface) {
431 push @{$final_config}, " $rule";
436 sub generate_frr_routemap
{
437 my ($final_config, $routemaps) = @_;
439 foreach my $id (sort keys %$routemaps) {
441 my $routemap = $routemaps->{$id};
443 foreach my $seq (@$routemap) {
445 next if !defined($seq->{action
});
448 push @config, "route-map $id $seq->{action} $order";
449 my $rule = $seq->{rule
};
450 push @config, map { " $_" } @$rule;
451 push @{$final_config}, @config;
452 push @{$final_config}, "exit";
457 sub generate_frr_list
{
458 my ($final_config, $lists, $type) = @_;
462 for my $id (sort keys %$lists) {
463 my $list = $lists->{$id};
465 for my $seq (sort keys %$list) {
466 my $rule = $list->{$seq};
467 push @$config, "$type $id seq $seq $rule";
472 push @{$final_config}, "!", @$config;
476 sub generate_controller_rawconfig
{
477 my ($class, $plugin_config, $config) = @_;
479 my $nodename = PVE
::INotify
::nodename
();
481 my $final_config = [];
482 push @{$final_config}, "frr version 8.5.1";
483 push @{$final_config}, "frr defaults datacenter";
484 push @{$final_config}, "hostname $nodename";
485 push @{$final_config}, "log syslog informational";
486 push @{$final_config}, "service integrated-vtysh-config";
487 push @{$final_config}, "!";
489 if (-e
"/etc/frr/frr.conf.local") {
490 my $local_conf = file_get_contents
("/etc/frr/frr.conf.local");
491 parse_merge_frr_local_config
($config, $local_conf);
494 generate_frr_vrf
($final_config, $config->{frr
}->{vrf
});
495 generate_frr_interfaces
($final_config, $config->{frr_interfaces
});
496 generate_frr_recurse
($final_config, $config->{frr
}, undef, 0);
497 generate_frr_list
($final_config, $config->{frr_access_list
}, "access-list");
498 generate_frr_list
($final_config, $config->{frr_prefix_list
}, "ip prefix-list");
499 generate_frr_list
($final_config, $config->{frr_prefix_list_v6
}, "ipv6 prefix-list");
500 generate_frr_routemap
($final_config, $config->{frr_routemap
});
501 generate_frr_ip_protocol
($final_config, $config->{frr_ip_protocol
});
503 push @{$final_config}, "!";
504 push @{$final_config}, "line vty";
505 push @{$final_config}, "!";
507 my $rawconfig = join("\n", @{$final_config});
509 return if !$rawconfig;
513 sub parse_merge_frr_local_config
{
514 my ($config, $local_conf) = @_;
516 my $section = \
$config->{""};
518 my $routemap = undef;
519 my $routemap_config = ();
520 my $routemap_action = undef;
522 while ($local_conf =~ /^\s*(.+?)\s*$/gm) {
524 $line =~ s/^\s+|\s+$//g;
526 if ($line =~ m/^router (.+)$/) {
528 $section = \
$config->{'frr'}->{'router'}->{$router}->{""};
530 } elsif ($line =~ m/^vrf (.+)$/) {
531 $section = \
$config->{'frr'}->{'vrf'}->{$1};
533 } elsif ($line =~ m/^interface (.+)$/) {
534 $section = \
$config->{'frr_interfaces'}->{$1};
536 } elsif ($line =~ m/address-family (.+)$/) {
537 $section = \
$config->{'frr'}->{'router'}->{$router}->{'address-family'}->{$1};
539 } elsif ($line =~ m/^route-map (.+) (permit|deny) (\d+)/) {
541 $routemap_config = ();
542 $routemap_action = $2;
543 $section = \
$config->{'frr_routemap'}->{$routemap};
545 } elsif ($line =~ m/^access-list (.+) seq (\d+) (.+)$/) {
546 $config->{'frr_access_list'}->{$1}->{$2} = $3;
548 } elsif ($line =~ m/^ip prefix-list (.+) seq (\d+) (.*)$/) {
549 $config->{'frr_prefix_list'}->{$1}->{$2} = $3;
551 } elsif ($line =~ m/^ipv6 prefix-list (.+) seq (\d+) (.*)$/) {
552 $config->{'frr_prefix_list_v6'}->{$1}->{$2} = $3;
554 } elsif($line =~ m/^exit-address-family$/) {
556 } elsif($line =~ m/^exit$/) {
558 $section = \
$config->{''};
561 push(@{$$section}, { rule
=> $routemap_config, action
=> $routemap_action });
562 $section = \
$config->{''};
564 $routemap_action = undef;
565 $routemap_config = ();
568 } elsif($line =~ m/!/) {
574 push(@{$routemap_config}, $line);
576 push(@{$$section}, $line);
581 sub write_controller_config
{
582 my ($class, $plugin_config, $config) = @_;
584 my $rawconfig = $class->generate_controller_rawconfig($plugin_config, $config);
585 return if !$rawconfig;
586 return if !-d
"/etc/frr";
588 file_set_contents
("/etc/frr/frr.conf", $rawconfig);
591 sub reload_controller
{
594 my $conf_file = "/etc/frr/frr.conf";
595 my $bin_path = "/usr/lib/frr/frr-reload.py";
598 log_warn
("missing $bin_path. Please install frr-pythontools package");
604 if ($line =~ /ERROR:/) {
609 if (-e
$conf_file && -e
$bin_path) {
611 run_command
([$bin_path, '--stdout', '--reload', $conf_file], outfunc
=> {}, errfunc
=> $err);
614 warn "frr reload command fail. Restarting frr.";
615 eval { run_command
(['systemctl', 'restart', 'frr']); };