]>
Commit | Line | Data |
---|---|---|
1 | package PVE::Network; | |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | use PVE::Tools qw(run_command); | |
6 | use PVE::ProcFSTools; | |
7 | use PVE::INotify; | |
8 | use File::Basename; | |
9 | ||
10 | # host network related utility functions | |
11 | ||
12 | sub setup_tc_rate_limit { | |
13 | my ($iface, $rate, $burst, $debug) = @_; | |
14 | ||
15 | system("/sbin/tc class del dev $iface parent 1: classid 1:1 >/dev/null 2>&1"); | |
16 | system("/sbin/tc filter del dev $iface parent ffff: protocol ip prio 50 estimator 1sec 8sec >/dev/null 2>&1"); | |
17 | system("/sbin/tc qdisc del dev $iface ingress >/dev/null 2>&1"); | |
18 | system("/sbin/tc qdisc del dev $iface root >/dev/null 2>&1"); | |
19 | ||
20 | return if !$rate; | |
21 | ||
22 | run_command("/sbin/tc qdisc add dev $iface handle ffff: ingress"); | |
23 | ||
24 | # this does not work wit virtio - don't know why (setting "mtu 64kb" does not help) | |
25 | #run_command("/sbin/tc filter add dev $iface parent ffff: protocol ip prio 50 u32 match ip src 0.0.0.0/0 police rate ${rate}bps burst ${burst}b drop flowid :1"); | |
26 | # so we use avrate instead | |
27 | run_command("/sbin/tc filter add dev $iface parent ffff: " . | |
28 | "protocol ip prio 50 estimator 1sec 8sec " . | |
29 | "u32 match ip src 0.0.0.0/0 police avrate ${rate}bps drop flowid :1"); | |
30 | ||
31 | # tbf does not work for unknown reason | |
32 | #$TC qdisc add dev $DEV root tbf rate $RATE latency 100ms burst $BURST | |
33 | # so we use htb instead | |
34 | run_command("/sbin/tc qdisc add dev $iface root handle 1: htb default 1"); | |
35 | run_command("/sbin/tc class add dev $iface parent 1: classid 1:1 " . | |
36 | "htb rate ${rate}bps burst ${burst}b"); | |
37 | ||
38 | if ($debug) { | |
39 | print "DEBUG tc settings\n"; | |
40 | system("/sbin/tc qdisc ls dev $iface"); | |
41 | system("/sbin/tc class ls dev $iface"); | |
42 | system("/sbin/tc filter ls dev $iface parent ffff:"); | |
43 | } | |
44 | } | |
45 | ||
46 | sub tap_rate_limit { | |
47 | my ($iface, $rate) = @_; | |
48 | ||
49 | my $debug = 0; | |
50 | $rate = int($rate*1024*1024); | |
51 | my $burst = 1024*1024; | |
52 | ||
53 | setup_tc_rate_limit($iface, $rate, $burst, $debug); | |
54 | } | |
55 | ||
56 | my $read_bridge_mtu = sub { | |
57 | my ($bridge) = @_; | |
58 | ||
59 | my $mtu = PVE::Tools::file_read_firstline("/sys/class/net/$bridge/mtu"); | |
60 | die "bridge '$bridge' does not exist\n" if !$mtu; | |
61 | # avoid insecure dependency; | |
62 | die "unable to parse mtu value" if $mtu !~ /^(\d+)$/; | |
63 | $mtu = int($1); | |
64 | ||
65 | return $mtu; | |
66 | }; | |
67 | ||
68 | my $parse_tap_devive_name = sub { | |
69 | my ($iface, $noerr) = @_; | |
70 | ||
71 | my ($vmid, $devid); | |
72 | ||
73 | if ($iface =~ m/^tap(\d+)i(\d+)$/) { | |
74 | $vmid = $1; | |
75 | $devid = $2; | |
76 | } elsif ($iface =~ m/^veth(\d+)\.(\d+)$/) { | |
77 | $vmid = $1; | |
78 | $devid = $2; | |
79 | } else { | |
80 | return undef if $noerr; | |
81 | die "can't create firewall bridge for random interface name '$iface'\n"; | |
82 | } | |
83 | ||
84 | return ($vmid, $devid); | |
85 | }; | |
86 | ||
87 | my $compute_fwbr_names = sub { | |
88 | my ($vmid, $devid) = @_; | |
89 | ||
90 | my $fwbr = "fwbr${vmid}i${devid}"; | |
91 | # Note: the firewall use 'fwln+' to filter traffic to VMs | |
92 | my $vethfw = "fwln${vmid}i${devid}"; | |
93 | my $vethfwpeer = "fwpr${vmid}p${devid}"; | |
94 | my $ovsintport = "fwln${vmid}o${devid}"; | |
95 | ||
96 | return ($fwbr, $vethfw, $vethfwpeer, $ovsintport); | |
97 | }; | |
98 | ||
99 | my $cond_create_bridge = sub { | |
100 | my ($bridge) = @_; | |
101 | ||
102 | if (! -d "/sys/class/net/$bridge") { | |
103 | system("/sbin/brctl addbr $bridge") == 0 || | |
104 | die "can't add bridge '$bridge'\n"; | |
105 | } | |
106 | }; | |
107 | ||
108 | my $bridge_add_interface = sub { | |
109 | my ($bridge, $iface) = @_; | |
110 | ||
111 | system("/sbin/brctl addif $bridge $iface") == 0 || | |
112 | die "can't add interface 'iface' to bridge '$bridge'\n"; | |
113 | }; | |
114 | ||
115 | my $ovs_bridge_add_port = sub { | |
116 | my ($bridge, $iface, $tag, $internal) = @_; | |
117 | ||
118 | my $cmd = "/usr/bin/ovs-vsctl add-port $bridge $iface"; | |
119 | $cmd .= " tag=$tag" if $tag; | |
120 | $cmd .= " -- set Interface $iface type=internal" if $internal; | |
121 | system($cmd) == 0 || | |
122 | die "can't add ovs port '$iface'\n"; | |
123 | }; | |
124 | ||
125 | my $activate_interface = sub { | |
126 | my ($iface) = @_; | |
127 | ||
128 | system("/sbin/ip link set $iface up") == 0 || | |
129 | die "can't activate interface '$iface'\n"; | |
130 | }; | |
131 | ||
132 | sub tap_create { | |
133 | my ($iface, $bridge) = @_; | |
134 | ||
135 | die "unable to get bridge setting\n" if !$bridge; | |
136 | ||
137 | my $bridgemtu = &$read_bridge_mtu($bridge); | |
138 | ||
139 | eval { | |
140 | PVE::Tools::run_command("/sbin/ifconfig $iface 0.0.0.0 promisc up mtu $bridgemtu"); | |
141 | }; | |
142 | die "interface activation failed\n" if $@; | |
143 | } | |
144 | ||
145 | sub veth_create { | |
146 | my ($veth, $vethpeer, $bridge, $mac) = @_; | |
147 | ||
148 | die "unable to get bridge setting\n" if !$bridge; | |
149 | ||
150 | my $bridgemtu = &$read_bridge_mtu($bridge); | |
151 | ||
152 | # create veth pair | |
153 | if (! -d "/sys/class/net/$veth") { | |
154 | my $cmd = "/sbin/ip link add name $veth type veth peer name $vethpeer mtu $bridgemtu"; | |
155 | $cmd .= " addr $mac" if $mac; | |
156 | system($cmd) == 0 || die "can't create interface $veth\n"; | |
157 | } | |
158 | ||
159 | # up vethpair | |
160 | &$activate_interface($veth); | |
161 | &$activate_interface($vethpeer); | |
162 | } | |
163 | ||
164 | sub veth_delete { | |
165 | my ($veth) = @_; | |
166 | ||
167 | if (-d "/sys/class/net/$veth") { | |
168 | run_command("/sbin/ip link delete dev $veth", outfunc => sub {}, errfunc => sub {}); | |
169 | } | |
170 | ||
171 | } | |
172 | ||
173 | my $create_firewall_bridge_linux = sub { | |
174 | my ($iface, $bridge) = @_; | |
175 | ||
176 | my ($vmid, $devid) = &$parse_tap_devive_name($iface); | |
177 | my ($fwbr, $vethfw, $vethfwpeer) = &$compute_fwbr_names($vmid, $devid); | |
178 | ||
179 | &$cond_create_bridge($fwbr); | |
180 | &$activate_interface($fwbr); | |
181 | ||
182 | copy_bridge_config($bridge, $fwbr); | |
183 | veth_create($vethfw, $vethfwpeer, $bridge); | |
184 | ||
185 | &$bridge_add_interface($fwbr, $vethfw); | |
186 | &$bridge_add_interface($bridge, $vethfwpeer); | |
187 | ||
188 | return $fwbr; | |
189 | }; | |
190 | ||
191 | my $create_firewall_bridge_ovs = sub { | |
192 | my ($iface, $bridge, $tag) = @_; | |
193 | ||
194 | my ($vmid, $devid) = &$parse_tap_devive_name($iface); | |
195 | my ($fwbr, undef, undef, $ovsintport) = &$compute_fwbr_names($vmid, $devid); | |
196 | ||
197 | my $bridgemtu = &$read_bridge_mtu($bridge); | |
198 | ||
199 | &$cond_create_bridge($fwbr); | |
200 | &$activate_interface($fwbr); | |
201 | ||
202 | &$bridge_add_interface($fwbr, $iface); | |
203 | ||
204 | &$ovs_bridge_add_port($bridge, $ovsintport, $tag, 1); | |
205 | &$activate_interface($ovsintport); | |
206 | ||
207 | # set the same mtu for ovs int port | |
208 | PVE::Tools::run_command("/sbin/ifconfig $ovsintport mtu $bridgemtu"); | |
209 | ||
210 | &$bridge_add_interface($fwbr, $ovsintport); | |
211 | }; | |
212 | ||
213 | my $cleanup_firewall_bridge = sub { | |
214 | my ($iface) = @_; | |
215 | ||
216 | my ($vmid, $devid) = &$parse_tap_devive_name($iface, 1); | |
217 | return if !defined($vmid); | |
218 | my ($fwbr, $vethfw, $vethfwpeer, $ovsintport) = &$compute_fwbr_names($vmid, $devid); | |
219 | ||
220 | # cleanup old port config from any openvswitch bridge | |
221 | if (-d "/sys/class/net/$ovsintport") { | |
222 | run_command("/usr/bin/ovs-vsctl del-port $ovsintport", outfunc => sub {}, errfunc => sub {}); | |
223 | } | |
224 | ||
225 | # delete old vethfw interface | |
226 | veth_delete($vethfw); | |
227 | ||
228 | # cleanup fwbr bridge | |
229 | if (-d "/sys/class/net/$fwbr") { | |
230 | run_command("/sbin/ip link set dev $fwbr down", outfunc => sub {}, errfunc => sub {}); | |
231 | run_command("/sbin/brctl delbr $fwbr", outfunc => sub {}, errfunc => sub {}); | |
232 | } | |
233 | }; | |
234 | ||
235 | sub tap_plug { | |
236 | my ($iface, $bridge, $tag, $firewall) = @_; | |
237 | ||
238 | #cleanup old port config from any openvswitch bridge | |
239 | eval {run_command("/usr/bin/ovs-vsctl del-port $iface", outfunc => sub {}, errfunc => sub {}) }; | |
240 | ||
241 | if (-d "/sys/class/net/$bridge/bridge") { | |
242 | &$cleanup_firewall_bridge($iface); # remove stale devices | |
243 | ||
244 | my $newbridge = activate_bridge_vlan($bridge, $tag); | |
245 | copy_bridge_config($bridge, $newbridge) if $bridge ne $newbridge; | |
246 | ||
247 | $newbridge = &$create_firewall_bridge_linux($iface, $newbridge) if $firewall; | |
248 | ||
249 | &$bridge_add_interface($newbridge, $iface); | |
250 | } else { | |
251 | &$cleanup_firewall_bridge($iface); # remove stale devices | |
252 | ||
253 | if ($firewall) { | |
254 | &$create_firewall_bridge_ovs($iface, $bridge, $tag); | |
255 | } else { | |
256 | &$ovs_bridge_add_port($bridge, $iface, $tag); | |
257 | } | |
258 | } | |
259 | } | |
260 | ||
261 | sub tap_unplug { | |
262 | my ($iface) = @_; | |
263 | ||
264 | my $path= "/sys/class/net/$iface/brport/bridge"; | |
265 | if (-l $path) { | |
266 | my $bridge = basename(readlink($path)); | |
267 | #avoid insecure dependency; | |
268 | ($bridge) = $bridge =~ /(\S+)/; | |
269 | ||
270 | system("/sbin/brctl delif $bridge $iface") == 0 || | |
271 | die "can't del interface '$iface' from bridge '$bridge'\n"; | |
272 | ||
273 | } | |
274 | ||
275 | &$cleanup_firewall_bridge($iface); | |
276 | } | |
277 | ||
278 | sub copy_bridge_config { | |
279 | my ($br0, $br1) = @_; | |
280 | ||
281 | return if $br0 eq $br1; | |
282 | ||
283 | my $br_configs = [ 'ageing_time', 'stp_state', 'priority', 'forward_delay', | |
284 | 'hello_time', 'max_age', 'multicast_snooping', 'multicast_querier']; | |
285 | ||
286 | foreach my $sysname (@$br_configs) { | |
287 | eval { | |
288 | my $v0 = PVE::Tools::file_read_firstline("/sys/class/net/$br0/bridge/$sysname"); | |
289 | my $v1 = PVE::Tools::file_read_firstline("/sys/class/net/$br1/bridge/$sysname"); | |
290 | if ($v0 ne $v1) { | |
291 | PVE::ProcFSTools::write_proc_entry("/sys/class/net/$br1/bridge/$sysname", $v0); | |
292 | } | |
293 | }; | |
294 | warn $@ if $@; | |
295 | } | |
296 | } | |
297 | ||
298 | sub activate_bridge_vlan_slave { | |
299 | my ($bridgevlan, $iface, $tag) = @_; | |
300 | my $ifacevlan = "${iface}.$tag"; | |
301 | ||
302 | # create vlan on $iface is not already exist | |
303 | if (! -d "/sys/class/net/$ifacevlan") { | |
304 | system("/sbin/vconfig add $iface $tag") == 0 || | |
305 | die "can't add vlan tag $tag to interface $iface\n"; | |
306 | } | |
307 | ||
308 | # be sure to have the $ifacevlan up | |
309 | &$activate_interface($ifacevlan); | |
310 | ||
311 | # test if $vlaniface is already enslaved in another bridge | |
312 | my $path= "/sys/class/net/$ifacevlan/brport/bridge"; | |
313 | if (-l $path) { | |
314 | my $tbridge = basename(readlink($path)); | |
315 | if ($tbridge ne $bridgevlan) { | |
316 | die "interface $ifacevlan already exist in bridge $tbridge\n"; | |
317 | } else { | |
318 | # Port already attached to bridge: do nothing. | |
319 | return; | |
320 | } | |
321 | } | |
322 | ||
323 | # add $ifacevlan to the bridge | |
324 | &$bridge_add_interface($bridgevlan, $ifacevlan); | |
325 | } | |
326 | ||
327 | sub activate_bridge_vlan { | |
328 | my ($bridge, $tag_param) = @_; | |
329 | ||
330 | die "bridge '$bridge' is not active\n" if ! -d "/sys/class/net/$bridge"; | |
331 | ||
332 | return $bridge if !defined($tag_param); # no vlan, simply return | |
333 | ||
334 | my $tag = int($tag_param); | |
335 | ||
336 | die "got strange vlan tag '$tag_param'\n" if $tag < 1 || $tag > 4094; | |
337 | ||
338 | my $bridgevlan = "${bridge}v$tag"; | |
339 | ||
340 | my @ifaces = (); | |
341 | my $dir = "/sys/class/net/$bridge/brif"; | |
342 | PVE::Tools::dir_glob_foreach($dir, '((eth|bond)\d+)', sub { | |
343 | push @ifaces, $_[0]; | |
344 | }); | |
345 | ||
346 | die "no physical interface on bridge '$bridge'\n" if scalar(@ifaces) == 0; | |
347 | ||
348 | # add bridgevlan if it doesn't already exist | |
349 | if (! -d "/sys/class/net/$bridgevlan") { | |
350 | system("/sbin/brctl addbr $bridgevlan") == 0 || | |
351 | die "can't add bridge $bridgevlan\n"; | |
352 | } | |
353 | ||
354 | # for each physical interface (eth or bridge) bind them to bridge vlan | |
355 | foreach my $iface (@ifaces) { | |
356 | activate_bridge_vlan_slave($bridgevlan, $iface, $tag); | |
357 | } | |
358 | ||
359 | #fixme: set other bridge flags | |
360 | ||
361 | # be sure to have the bridge up | |
362 | system("/sbin/ip link set $bridgevlan up") == 0 || | |
363 | die "can't up bridge $bridgevlan\n"; | |
364 | ||
365 | return $bridgevlan; | |
366 | } | |
367 | ||
368 | 1; |