X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=src%2FPVE%2FLXC.pm;h=9033322e0a7f1c93b0394f89225da5156bebc09c;hb=f8cc2ebc93c1541e054f95350f06be47da2d592e;hp=12310efaccbf4cd0221630c717ad5da12426d79d;hpb=80d5611161473542c43267cf442fa1a241296e19;p=pve-container.git diff --git a/src/PVE/LXC.pm b/src/PVE/LXC.pm index 12310ef..9033322 100644 --- a/src/PVE/LXC.pm +++ b/src/PVE/LXC.pm @@ -10,22 +10,37 @@ use Socket; use File::Path; use File::Spec; use Cwd qw(); -use Fcntl qw(O_RDONLY O_NOFOLLOW O_DIRECTORY); -use Errno qw(ELOOP ENOTDIR EROFS ECONNREFUSED); +use Fcntl qw(O_RDONLY O_WRONLY O_NOFOLLOW O_DIRECTORY); +use Errno qw(ELOOP ENOTDIR EROFS ECONNREFUSED ENOSYS EEXIST); use IO::Socket::UNIX; use PVE::Exception qw(raise_perm_exc); use PVE::Storage; use PVE::SafeSyslog; use PVE::INotify; -use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full O_PATH); +use PVE::JSONSchema qw(get_standard_option); +use PVE::Tools qw( + dir_glob_foreach file_get_contents file_set_contents lock_file + lock_file_full AT_FDCWD O_PATH $IPV4RE $IPV6RE +); use PVE::CpuSet; use PVE::Network; use PVE::AccessControl; use PVE::ProcFSTools; -use PVE::Syscall; +use PVE::Syscall qw(:fsmount); use PVE::LXC::Config; +use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne); +use PVE::LXC::Tools; +use PVE::LXC::CGroup; + use Time::HiRes qw (gettimeofday); +my $have_sdn; +eval { + require PVE::Network::SDN::Zones; + $have_sdn = 1; +}; + +my $LXC_CONFIG_PATH = '/usr/share/lxc/config'; my $nodename = PVE::INotify::nodename(); @@ -42,17 +57,11 @@ sub config_list { my $d = $ids->{$vmid}; next if !$d->{node} || $d->{node} ne $nodename; next if !$d->{type} || $d->{type} ne 'lxc'; - $res->{$vmid}->{type} = 'lxc'; + $res->{$vmid} = { type => 'lxc', vmid => $vmid }; } return $res; } -sub destroy_config { - my ($vmid) = @_; - - unlink PVE::LXC::Config->config_file($vmid, $nodename); -} - # container status helpers sub list_active_containers { @@ -98,27 +107,63 @@ sub get_container_disk_usage { my $last_proc_vmid_stat; -my $parse_cpuacct_stat = sub { - my ($vmid, $unprivileged) = @_; - - my $raw = read_cgroup_value('cpuacct', $vmid, $unprivileged, 'cpuacct.stat', 1); - - my $stat = {}; - - if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) { - - $stat->{utime} = $1; - $stat->{stime} = $2; - +our $vmstatus_return_properties = { + vmid => get_standard_option('pve-vmid'), + status => { + description => "LXC Container status.", + type => 'string', + enum => ['stopped', 'running'], + }, + maxmem => { + description => "Maximum memory in bytes.", + type => 'integer', + optional => 1, + renderer => 'bytes', + }, + maxswap => { + description => "Maximum SWAP memory in bytes.", + type => 'integer', + optional => 1, + renderer => 'bytes', + }, + maxdisk => { + description => "Root disk size in bytes.", + type => 'integer', + optional => 1, + renderer => 'bytes', + }, + name => { + description => "Container name.", + type => 'string', + optional => 1, + }, + uptime => { + description => "Uptime.", + type => 'integer', + optional => 1, + renderer => 'duration', + }, + cpus => { + description => "Maximum usable CPUs.", + type => 'number', + optional => 1, + }, + lock => { + description => "The current config lock, if any.", + type => 'string', + optional => 1, + }, + tags => { + description => "The current configured tags, if any.", + type => 'string', + optional => 1, } - - return $stat; }; sub vmstatus { my ($opt_vmid) = @_; - my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list(); + my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc', vmid => $opt_vmid }} : config_list(); my $active_hash = list_active_containers(); @@ -137,7 +182,7 @@ sub vmstatus { eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); }; warn $@ if $@; # ignore errors (consider them stopped) - $d->{status} = $d->{pid} ? 'running' : 'stopped'; + $d->{status} = $active_hash->{$vmid} ? 'running' : 'stopped'; my $cfspath = PVE::LXC::Config->cfs_config_path($vmid); my $conf = PVE::Cluster::cfs_read_file($cfspath) || {}; @@ -151,6 +196,7 @@ sub vmstatus { $d->{cpus} = $cpucount if !$d->{cpus}; $d->{lock} = $conf->{lock} || ''; + $d->{tags} = $conf->{tags} if defined($conf->{tags}); if ($d->{pid}) { my $res = get_container_disk_usage($vmid, $d->{pid}); @@ -182,6 +228,7 @@ sub vmstatus { $d->{diskwrite} = 0; $d->{template} = PVE::LXC::Config->is_template($conf); + $d->{lock} = $conf->{lock} if $conf->{lock}; } foreach my $vmid (keys %$list) { @@ -195,48 +242,52 @@ sub vmstatus { my $unpriv = $unprivileged->{$vmid}; - my $memory_stat = read_cgroup_list('memory', $vmid, $unpriv, 'memory.stat'); - my $mem_usage_in_bytes = read_cgroup_value('memory', $vmid, $unpriv, 'memory.usage_in_bytes'); + my $cgroups = PVE::LXC::CGroup->new($vmid); - $d->{mem} = $mem_usage_in_bytes - $memory_stat->{total_cache}; - $d->{swap} = read_cgroup_value('memory', $vmid, $unpriv, 'memory.memsw.usage_in_bytes') - $mem_usage_in_bytes; - - my $blkio_bytes = read_cgroup_value('blkio', $vmid, $unpriv, 'blkio.throttle.io_service_bytes', 1); - my @bytes = split(/\n/, $blkio_bytes); - foreach my $byte (@bytes) { - if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) { - $d->{diskread} += $2 if $key eq 'Read'; - $d->{diskwrite} += $2 if $key eq 'Write'; - } + if (defined(my $mem = $cgroups->get_memory_stat())) { + $d->{mem} = $mem->{mem}; + $d->{swap} = $mem->{swap}; + } else { + $d->{mem} = 0; + $d->{swap} = 0; } - my $pstat = $parse_cpuacct_stat->($vmid, $unpriv); - - my $used = $pstat->{utime} + $pstat->{stime}; - - my $old = $last_proc_vmid_stat->{$vmid}; - if (!$old) { - $last_proc_vmid_stat->{$vmid} = { - time => $cdtime, - used => $used, - cpu => 0, - }; - next; + if (defined(my $blkio = $cgroups->get_io_stats())) { + $d->{diskread} = $blkio->{diskread}; + $d->{diskwrite} = $blkio->{diskwrite}; + } else { + $d->{diskread} = 0; + $d->{diskwrite} = 0; } - my $dtime = ($cdtime - $old->{time}) * $cpucount * $cpuinfo->{user_hz}; + if (defined(my $cpu = $cgroups->get_cpu_stat())) { + # Total time (in milliseconds) used up by the cpu. + my $used_ms = $cpu->{utime} + $cpu->{stime}; - if ($dtime > 1000) { - my $dutime = $used - $old->{used}; + my $old = $last_proc_vmid_stat->{$vmid}; + if (!$old) { + $last_proc_vmid_stat->{$vmid} = { + time => $cdtime, + used => $used_ms, + cpu => 0, + }; + next; + } - $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus}; - $last_proc_vmid_stat->{$vmid} = { - time => $cdtime, - used => $used, - cpu => $d->{cpu}, - }; + my $delta_ms = ($cdtime - $old->{time}) * $cpucount * 1000.0; + if ($delta_ms > 1000.0) { + my $delta_used_ms = $used_ms - $old->{used}; + $d->{cpu} = (($delta_used_ms / $delta_ms) * $cpucount) / $d->{cpus}; + $last_proc_vmid_stat->{$vmid} = { + time => $cdtime, + used => $used_ms, + cpu => $d->{cpu}, + }; + } else { + $d->{cpu} = $old->{cpu}; + } } else { - $d->{cpu} = $old->{cpu}; + $d->{cpu} = 0; } } @@ -257,33 +308,6 @@ sub vmstatus { return $list; } -sub read_cgroup_list($$$$) { - my ($group, $vmid, $unprivileged, $name) = @_; - - my $content = read_cgroup_value($group, $vmid, $unprivileged, $name, 1); - - return { split(/\s+/, $content) }; -} - -sub read_cgroup_value($$$$$) { - my ($group, $vmid, $unprivileged, $name, $full) = @_; - - my $nsdir = $unprivileged ? '' : 'ns/'; - my $path = "/sys/fs/cgroup/$group/lxc/$vmid/${nsdir}$name"; - - return PVE::Tools::file_get_contents($path) if $full; - - return PVE::Tools::file_read_firstline($path); -} - -sub write_cgroup_value { - my ($group, $vmid, $name, $value) = @_; - - my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name"; - PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path; - -} - sub find_lxc_console_pids { my $res = {}; @@ -325,6 +349,44 @@ sub find_lxc_pid { return $pid; } +sub open_pid_fd($) { + my ($pid) = @_; + sysopen(my $fd, "/proc/$pid", O_RDONLY | O_DIRECTORY) + or die "failed to open /proc/$pid pid fd\n"; + return $fd; +} + +sub open_lxc_pid { + my ($vmid) = @_; + + # Find the pid and open: + my $pid = find_lxc_pid($vmid); + my $fd = open_pid_fd($pid); + + # Verify: + my $pid2 = find_lxc_pid($vmid); + + return () if $pid != $pid2; + return ($pid, $fd); +} + +sub open_ppid { + my ($pid) = @_; + + # Find the parent pid via proc and open it: + my $stat = PVE::ProcFSTools::read_proc_pid_stat($pid); + my $ppid = $stat->{ppid} // die "failed to get parent pid\n"; + + my $fd = open_pid_fd($ppid); + + # Verify: + $stat = PVE::ProcFSTools::read_proc_pid_stat($pid); + my $ppid2 = $stat->{ppid} // die "failed to get parent pid for verification\n"; + + return () if $ppid != $ppid2; + return ($ppid, $fd); +} + # Note: we cannot use Net:IP, because that only allows strict # CIDR networks sub parse_ipv4_cidr { @@ -339,6 +401,172 @@ sub parse_ipv4_cidr { die "unable to parse ipv4 address/mask\n"; } +sub get_cgroup_subsystems { + my $v1 = {}; + my $v2 = 0; + my $data = PVE::Tools::file_get_contents('/proc/self/cgroup'); + while ($data =~ /^\d+:([^:\n]*):.*$/gm) { + my $type = $1; + if (length($type)) { + $v1->{$_} = 1 foreach split(/,/, $type); + } else { + $v2 = 1; + } + } + return wantarray ? ($v1, $v2) : $v1; +} + +# With seccomp trap to userspace we now have the ability to optionally forward +# certain syscalls to the "host" to handle (via our pve-lxc-syscalld daemon). +# +# This means that there are cases where we need to create an extra seccomp +# profile for the container to load. +# +# This returns a configuration snippet added to the raw lxc config. +sub make_seccomp_config { + my ($conf, $vmid, $conf_dir, $unprivileged, $features) = @_; + # User-configured profile has precedence, note that the user's entry would + # be written 'after' this line anyway... + if (PVE::LXC::Config->has_lxc_entry($conf, 'lxc.seccomp.profile')) { + # Warn the user if this conflicts with a feature: + my $warn = join(', ', grep { $features->{$_} } qw(keyctl mknod)); + warn "explicitly configured lxc.seccomp.profile overrides the following settings: $warn\n" + if length($warn) > 0; + return ''; + } + + # Privileged containers keep using the default (which is already part of + # the files included via lxc.include, so we don't need to write it out, + # that way it stays admin-configurable via /usr/share/lxc/config/... as + # well) + return '' if !$unprivileged; + + my $rules = { + keyctl => ['errno 38'], + }; + + my $raw_conf = ''; + + # Unprivileged containers will get keyctl() disabled by default as a + # workaround for systemd-networkd behavior. But we have an option to + # explicitly enable it: + if ($features->{keyctl}) { + delete $rules->{keyctl}; + } + + # By default, unprivileged containers cannot use `mknod` at all. + # Since lxc 3.2, we can use seccomp's trap to userspace feature for this, + # but for now this is experimental, so it has to be enabled via a feature + # flag. + # Note that we only handle block and char devices (like lxd), the rest we + # leave up to the kernel. We may in the future remove this if seccomp gets + # a way to tell the kernel to "continue" a syscall. + if ($features->{mknod}) { + my ($ok, $kernel) = PVE::ProcFSTools::check_kernel_release(5, 3); + if (!$ok) { + die "'mknod' feature requested, but kernel too old (found $kernel, required >= 5.3)\n"; + } + + $raw_conf .= "lxc.seccomp.notify.proxy = unix:/run/pve/lxc-syscalld.sock\n"; + $raw_conf .= "lxc.seccomp.notify.cookie = $vmid\n"; + + $rules->{mknod} = [ + # condition: (mode & S_IFMT) == S_IFCHR + 'notify [1,8192,SCMP_CMP_MASKED_EQ,61440]', + # condition: (mode & S_IFMT) == S_IFBLK + 'notify [1,24576,SCMP_CMP_MASKED_EQ,61440]', + ]; + $rules->{mknodat} = [ + # condition: (mode & S_IFMT) == S_IFCHR + 'notify [2,8192,SCMP_CMP_MASKED_EQ,61440]', + # condition: (mode & S_IFMT) == S_IFBLK + 'notify [2,24576,SCMP_CMP_MASKED_EQ,61440]', + ]; + } + + # Now build the custom seccomp rule text... + my $extra_rules = join("\n", map { + my $syscall = $_; + map { "$syscall $_" } $rules->{$syscall}->@* + } sort keys %$rules) . "\n"; + + return $raw_conf if $extra_rules eq "\n"; + + # We still have the "most common" config readily available, so don't write + # out that one: + if ($raw_conf eq '' && $extra_rules eq "keyctl errno 38\n") { + # we have no extra $raw_conf and use the same we had in pve 6.1: + return "lxc.seccomp.profile = $LXC_CONFIG_PATH/pve-userns.seccomp\n"; + } + + # Write the rule file to the container's config path: + my $rule_file = "$conf_dir/rules.seccomp"; + my $rule_data = file_get_contents("$LXC_CONFIG_PATH/common.seccomp") + . $extra_rules; + file_set_contents($rule_file, $rule_data); + $raw_conf .= "lxc.seccomp.profile = $rule_file\n"; + + return $raw_conf; +} + +# Since lxc-3.0.2 we can have lxc generate a profile for the container +# automatically. The default should be equivalent to the old +# `lxc-container-default-cgns` profile. +# +# Additionally this also added `lxc.apparmor.raw` which can be used to inject +# additional lines into the profile. We can use that to allow mounting specific +# file systems. +sub make_apparmor_config { + my ($conf, $unprivileged, $features) = @_; + + # user-configured profile has precedence, but first we go through our own + # code to figure out whether we should warn the user: + + my $raw = "lxc.apparmor.profile = generated\n"; + my @profile_uses; + + if ($features->{fuse}) { + # For the informational warning: + push @profile_uses, 'features:fuse'; + } + + # There's lxc.apparmor.allow_nesting now, which will add the necessary + # apparmor lines, create an apparmor namespace for the container, but also + # adds proc and sysfs mounts to /dev/.lxc/{proc,sys}. These do not have + # lxcfs mounted over them, because that would prevent the container from + # mounting new instances of them for nested containers. + if ($features->{nesting}) { + push @profile_uses, 'features:nesting'; + $raw .= "lxc.apparmor.allow_nesting = 1\n" + } else { + # In the default profile in /etc/apparmor.d we patch this in because + # otherwise a container can for example run `chown` on /sys, breaking + # access to it for non-CAP_DAC_OVERRIDE tools on the host: + $raw .= "lxc.apparmor.raw = deny mount -> /proc/,\n"; + $raw .= "lxc.apparmor.raw = deny mount -> /sys/,\n"; + # Preferably we could use the 'remount' flag but this does not sit well + # with apparmor_parser currently: + # mount options=(rw, nosuid, nodev, noexec, remount) -> /sys/, + } + + if (my $mount = $features->{mount}) { + push @profile_uses, 'features:mount'; + foreach my $fs (PVE::Tools::split_list($mount)) { + $raw .= "lxc.apparmor.raw = mount fstype=$fs,\n"; + } + } + + # More to come? + + if (PVE::LXC::Config->has_lxc_entry($conf, 'lxc.apparmor.profile')) { + if (length(my $used = join(', ', @profile_uses))) { + warn "explicitly configured lxc.apparmor.profile overrides the following settings: $used\n"; + } + return ''; + } + + return $raw; +} sub update_lxc_config { my ($vmid, $conf) = @_; @@ -357,18 +585,34 @@ sub update_lxc_config { die "missing 'arch' - internal error" if !$conf->{arch}; $raw .= "lxc.arch = $conf->{arch}\n"; - my $unprivileged = $conf->{unprivileged}; - my $custom_idmap = grep { $_->[0] eq 'lxc.idmap' } @{$conf->{lxc}}; + my $custom_idmap = PVE::LXC::Config->has_lxc_entry($conf, 'lxc.idmap'); + my $unprivileged = $conf->{unprivileged} || $custom_idmap; my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error"; - my $inc ="/usr/share/lxc/config/$ostype.common.conf"; - $inc ="/usr/share/lxc/config/common.conf" if !-f $inc; + my $cfgpath = '/usr/share/lxc/config'; + my $inc = "$cfgpath/$ostype.common.conf"; + $inc ="$cfgpath/common.conf" if !-f $inc; $raw .= "lxc.include = $inc\n"; - if ($unprivileged || $custom_idmap) { - $inc = "/usr/share/lxc/config/$ostype.userns.conf"; - $inc = "/usr/share/lxc/config/userns.conf" if !-f $inc; - $raw .= "lxc.include = $inc\n" + if ($unprivileged) { + $inc = "$cfgpath/$ostype.userns.conf"; + $inc = "$cfgpath/userns.conf" if !-f $inc; + $raw .= "lxc.include = $inc\n"; + } + + my $features = PVE::LXC::Config->parse_features($conf->{features}); + + $raw .= make_seccomp_config($conf, $vmid, $dir, $unprivileged, $features); + $raw .= make_apparmor_config($conf, $unprivileged, $features); + if ($features->{fuse}) { + $raw .= "lxc.apparmor.raw = mount fstype=fuse,\n"; + $raw .= "lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file 0 0\n"; + } + + if ($unprivileged && !$features->{force_rw_sys}) { + # unpriv. CT default to sys:rw, but that doesn't always plays well with + # systemd, e.g., systemd-networkd https://systemd.io/CONTAINER_INTERFACE/ + $raw .= "lxc.mount.auto = sys:mixed\n"; } # WARNING: DO NOT REMOVE this without making sure that loop device nodes @@ -378,6 +622,8 @@ sub update_lxc_config { # files while the container is running! $raw .= "lxc.monitor.unshare = 1\n"; + my $cgv1 = get_cgroup_subsystems(); + # Should we read them from /etc/subuid? if ($unprivileged && !$custom_idmap) { $raw .= "lxc.idmap = u 0 100000 65536\n"; @@ -386,7 +632,7 @@ sub update_lxc_config { if (!PVE::LXC::Config->has_dev_console($conf)) { $raw .= "lxc.console.path = none\n"; - $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n"; + $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n" if $cgv1->{devices}; } my $ttycount = PVE::LXC::Config->get_tty_count($conf); @@ -398,23 +644,27 @@ sub update_lxc_config { my $utsname = $conf->{hostname} || "CT$vmid"; $raw .= "lxc.uts.name = $utsname\n"; - my $memory = $conf->{memory} || 512; - my $swap = $conf->{swap} // 0; + if ($cgv1->{memory}) { + my $memory = $conf->{memory} || 512; + my $swap = $conf->{swap} // 0; - my $lxcmem = int($memory*1024*1024); - $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n"; + my $lxcmem = int($memory*1024*1024); + $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n"; - my $lxcswap = int(($memory + $swap)*1024*1024); - $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n"; - - if (my $cpulimit = $conf->{cpulimit}) { - $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n"; - my $value = int(100000*$cpulimit); - $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n"; + my $lxcswap = int(($memory + $swap)*1024*1024); + $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n"; } - my $shares = $conf->{cpuunits} || 1024; - $raw .= "lxc.cgroup.cpu.shares = $shares\n"; + if ($cgv1->{cpu}) { + if (my $cpulimit = $conf->{cpulimit}) { + $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n"; + my $value = int(100000*$cpulimit); + $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n"; + } + + my $shares = $conf->{cpuunits} || 1024; + $raw .= "lxc.cgroup.cpu.shares = $shares\n"; + } die "missing 'rootfs' configuration\n" if !defined($conf->{rootfs}); @@ -443,10 +693,14 @@ sub update_lxc_config { } } + my $cpuset; + my $cpuset_cgroup = eval { PVE::LXC::CGroup::cpuset_controller_path() }; + if (defined($cpuset_cgroup)) { + $cpuset = eval { PVE::CpuSet->new_from_path("$cpuset_cgroup/lxc", 1) } + || PVE::CpuSet->new_from_path($cpuset_cgroup, 1); + } my $cores = $conf->{cores}; - if (!$had_cpuset && $cores) { - my $cpuset = eval { PVE::CpuSet->new_from_cgroup('lxc', 'effective_cpus') }; - $cpuset = PVE::CpuSet->new_from_cgroup('', 'effective_cpus') if !$cpuset; + if (!$had_cpuset && $cores && $cpuset) { my @members = $cpuset->members(); while (scalar(@members) > $cores) { my $randidx = int(rand(scalar(@members))); @@ -455,7 +709,7 @@ sub update_lxc_config { } $raw .= "lxc.cgroup.cpuset.cpus = ".$cpuset->short_string()."\n"; } - + File::Path::mkpath("$dir/rootfs"); PVE::Tools::file_set_contents("$dir/config", $raw); @@ -487,17 +741,20 @@ sub verify_searchdomain_list { } sub get_console_command { - my ($vmid, $conf, $noescapechar) = @_; + my ($vmid, $conf, $escapechar) = @_; + + # '-1' as $escapechar disables keyboard escape sequence + # any other passed char (a-z) will result in my $cmode = PVE::LXC::Config->get_cmode($conf); my $cmd = []; if ($cmode eq 'console') { push @$cmd, 'lxc-console', '-n', $vmid, '-t', 0; - push @$cmd, '-e', -1 if $noescapechar; + push @$cmd, '-e', $escapechar if $escapechar; } elsif ($cmode eq 'tty') { push @$cmd, 'lxc-console', '-n', $vmid; - push @$cmd, '-e', -1 if $noescapechar; + push @$cmd, '-e', $escapechar if $escapechar; } elsif ($cmode eq 'shell') { push @$cmd, 'lxc-attach', '--clear-env', '-n', $vmid; } else { @@ -541,7 +798,12 @@ sub delete_mountpoint_volume { return if PVE::LXC::Config->classify_mountpoint($volume) ne 'volume'; my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume); - PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner; + + if ($vmid == $owner) { + PVE::Storage::vdisk_free($storage_cfg, $volume); + } else { + warn "ignore deletion of '$volume', CT $vmid isn't the owner!\n"; + } } sub destroy_lxc_container { @@ -558,46 +820,22 @@ sub destroy_lxc_container { if (defined $replacement_conf) { PVE::LXC::Config->write_config($vmid, $replacement_conf); } else { - destroy_config($vmid); + PVE::LXC::Config->destroy_config($vmid); } - - #my $cmd = ['lxc-destroy', '-n', $vmid ]; - #PVE::Tools::run_command($cmd); } sub vm_stop_cleanup { my ($storage_cfg, $vmid, $conf, $keepActive) = @_; - - eval { - if (!$keepActive) { - my $vollist = PVE::LXC::Config->get_vm_volumes($conf); - PVE::Storage::deactivate_volumes($storage_cfg, $vollist); - } + return if $keepActive; + + eval { + my $vollist = PVE::LXC::Config->get_vm_volumes($conf); + PVE::Storage::deactivate_volumes($storage_cfg, $vollist); }; warn $@ if $@; # avoid errors - just warn } -my $safe_num_ne = sub { - my ($a, $b) = @_; - - return 0 if !defined($a) && !defined($b); - return 1 if !defined($a); - return 1 if !defined($b); - - return $a != $b; -}; - -my $safe_string_ne = sub { - my ($a, $b) = @_; - - return 0 if !defined($a) && !defined($b); - return 1 if !defined($a); - return 1 if !defined($b); - - return $a ne $b; -}; - sub update_net { my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_; @@ -612,8 +850,8 @@ sub update_net { if (my $oldnetcfg = $conf->{$opt}) { my $oldnet = PVE::LXC::Config->parse_lxc_network($oldnetcfg); - if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) || - &$safe_string_ne($oldnet->{name}, $newnet->{name})) { + if (safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) || + safe_string_ne($oldnet->{name}, $newnet->{name})) { PVE::Network::veth_delete($veth); delete $conf->{$opt}; @@ -622,9 +860,9 @@ sub update_net { hotplug_net($vmid, $conf, $opt, $newnet, $netid); } else { - if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) || - &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) || - &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) { + if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) || + safe_num_ne($oldnet->{tag}, $newnet->{tag}) || + safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) { if ($oldnet->{bridge}) { PVE::Network::tap_unplug($veth); @@ -635,12 +873,17 @@ sub update_net { PVE::LXC::Config->write_config($vmid, $conf); } - PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate}); + if ($have_sdn) { + PVE::Network::SDN::Zones::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate}); + } else { + PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate}); + } + # This includes the rate: foreach (qw(bridge tag firewall rate)) { $oldnet->{$_} = $newnet->{$_} if $newnet->{$_}; } - } elsif (&$safe_string_ne($oldnet->{rate}, $newnet->{rate})) { + } elsif (safe_string_ne($oldnet->{rate}, $newnet->{rate})) { # Rate can be applied on its own but any change above needs to # include the rate in tap_plug since OVS resets everything. PVE::Network::tap_rate_limit($veth, $newnet->{rate}); @@ -663,8 +906,13 @@ sub hotplug_net { my $vethpeer = $veth . "p"; my $eth = $newnet->{name}; - PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr}); - PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate}); + if ($have_sdn) { + PVE::Network::SDN::Zones::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr}); + PVE::Network::SDN::Zones::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate}); + } else { + PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr}); + PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate}); + } # attach peer in container my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ]; @@ -708,9 +956,10 @@ sub update_ipconfig { my $newip = $newnet->{$ip}; my $newgw = $newnet->{$gw}; my $oldip = $optdata->{$ip}; + my $oldgw = $optdata->{$gw}; - my $change_ip = &$safe_string_ne($oldip, $newip); - my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw); + my $change_ip = safe_string_ne($oldip, $newip); + my $change_gw = safe_string_ne($oldgw, $newgw); return if !$change_ip && !$change_gw; @@ -753,6 +1002,11 @@ sub update_ipconfig { # warn and continue warn $@ if $@; } + if ($oldgw && $oldip && !PVE::Network::is_ip_in_cidr($oldgw, $oldip)) { + eval { &$ipcmd($family_opt, 'route', 'del', $oldgw, 'dev', $eth); }; + # warn if the route was deleted manually + warn $@ if $@; + } } # from this point on we save the configuration @@ -793,15 +1047,34 @@ sub update_ipconfig { } +my $open_namespace = sub { + my ($vmid, $pid, $kind) = @_; + sysopen my $fd, "/proc/$pid/ns/$kind", O_RDONLY + or die "failed to open $kind namespace of container $vmid: $!\n"; + return $fd; +}; + my $enter_namespace = sub { - my ($vmid, $pid, $which, $type) = @_; - sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY - or die "failed to open $which namespace of container $vmid: $!\n"; + my ($vmid, $pid, $kind, $type) = @_; + my $fd = $open_namespace->($vmid, $pid, $kind); PVE::Tools::setns(fileno($fd), $type) - or die "failed to enter $which namespace of container $vmid: $!\n"; + or die "failed to enter $kind namespace of container $vmid: $!\n"; close $fd; }; +my $get_container_namespace = sub { + my ($vmid, $pid, $kind) = @_; + + my $pidfd; + if (!defined($pid)) { + # Pin the pid while we're grabbing its stuff from /proc + ($pid, $pidfd) = open_lxc_pid($vmid) + or die "failed to open pidfd of container $vmid\'s init process\n"; + } + + return $open_namespace->($vmid, $pid, $kind); +}; + my $do_syncfs = sub { my ($vmid, $pid, $socket) = @_; @@ -867,17 +1140,26 @@ sub template_create { my $storecfg = PVE::Storage::config(); - my $rootinfo = PVE::LXC::Config->parse_ct_rootfs($conf->{rootfs}); - my $volid = $rootinfo->{volume}; + PVE::LXC::Config->foreach_mountpoint($conf, sub { + my ($ms, $mountpoint) = @_; + + my $volid = $mountpoint->{volume}; + + die "Template feature is not available for '$volid'\n" + if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid); + }); + + PVE::LXC::Config->foreach_mountpoint($conf, sub { + my ($ms, $mountpoint) = @_; - die "Template feature is not available for '$volid'\n" - if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid); + my $volid = $mountpoint->{volume}; - PVE::Storage::activate_volumes($storecfg, [$volid]); + PVE::Storage::activate_volumes($storecfg, [$volid]); - my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid); - $rootinfo->{volume} = $template_volid; - $conf->{rootfs} = PVE::LXC::Config->print_ct_mountpoint($rootinfo, 1); + my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid); + $mountpoint->{volume} = $template_volid; + $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq "rootfs"); + }); PVE::LXC::Config->write_config($vmid, $conf); } @@ -903,6 +1185,12 @@ sub check_ct_modify_config_perm { } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' || $opt eq 'searchdomain' || $opt eq 'hostname') { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']); + } elsif ($opt eq 'features') { + # For now this is restricted to root@pam + raise_perm_exc("changing feature flags is only allowed for root\@pam"); + } elsif ($opt eq 'hookscript') { + # For now this is restricted to root@pam + raise_perm_exc("changing the hookscript is only allowed for root\@pam"); } else { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']); } @@ -924,6 +1212,8 @@ sub umount_all { my $rootdir = "/var/lib/lxc/$vmid/rootfs"; my $volid_list = PVE::LXC::Config->get_vm_volumes($conf); + my $res = 1; + PVE::LXC::Config->foreach_mountpoint_reverse($conf, sub { my ($ms, $mountpoint) = @_; @@ -942,12 +1232,15 @@ sub umount_all { }; if (my $err = $@) { if ($noerr) { + $res = 0; warn $err; } else { die $err; } } }); + + return $res; # tell caller if (some) umounts failed for the noerr case } sub mount_all { @@ -959,13 +1252,15 @@ sub mount_all { my $volid_list = PVE::LXC::Config->get_vm_volumes($conf); PVE::Storage::activate_volumes($storage_cfg, $volid_list); + my (undef, $rootuid, $rootgid) = parse_id_maps($conf); + eval { PVE::LXC::Config->foreach_mountpoint($conf, sub { my ($ms, $mountpoint) = @_; $mountpoint->{ro} = 0 if $ignore_ro; - mountpoint_mount($mountpoint, $rootdir, $storage_cfg); + mountpoint_mount($mountpoint, $rootdir, $storage_cfg, undef, $rootuid, $rootgid); }); }; if (my $err = $@) { @@ -1002,7 +1297,7 @@ sub query_loopdev { # The loop device is always detached afterwards (or set to autoclear). # Returns the loop device. sub run_with_loopdev { - my ($func, $file) = @_; + my ($func, $file, $readonly) = @_; my $device = query_loopdev($file); # Try to reuse an existing device if ($device) { @@ -1018,7 +1313,14 @@ sub run_with_loopdev { $device = $1; } }; - PVE::Tools::run_command(['losetup', '--show', '-f', $file], outfunc => $parser); + my $losetup_cmd = [ + 'losetup', + '--show', + '-f', + $file, + ]; + push @$losetup_cmd, '-r' if $readonly; + PVE::Tools::run_command($losetup_cmd, outfunc => $parser); die "failed to setup loop device for $file\n" if !$device; eval { &$func($device); }; my $err = $@; @@ -1033,16 +1335,24 @@ sub run_with_loopdev { # * the 2nd deepest directory (parent of the above) # * directory name of the last directory # So that the path $2/$3 should lead to $1 afterwards. -sub walk_tree_nofollow($$$) { - my ($start, $subdir, $mkdir) = @_; - - # splitdir() returns '' for empty components including the leading / - my @comps = grep { length($_)>0 } File::Spec->splitdir($subdir); +sub walk_tree_nofollow($$$;$$) { + my ($start, $subdir, $mkdir, $rootuid, $rootgid) = @_; sysopen(my $fd, $start, O_PATH | O_DIRECTORY) or die "failed to open start directory $start: $!\n"; - my $dir = $start; + return walk_tree_nofollow_fd($start, $fd, $subdir, $mkdir, $rootuid, $rootgid); +} + + +sub walk_tree_nofollow_fd($$$$;$$) { + my ($start_dirname, $start_fd, $subdir, $mkdir, $rootuid, $rootgid) = @_; + + # splitdir() returns '' for empty components including the leading / + my @comps = grep { length($_)>0 } File::Spec->splitdir($subdir); + + my $fd = $start_fd; + my $dir = $start_dirname; my $last_component = undef; my $second = $fd; foreach my $component (@comps) { @@ -1063,16 +1373,19 @@ sub walk_tree_nofollow($$$) { $next = PVE::Tools::openat(fileno($fd), $component, O_NOFOLLOW | O_DIRECTORY); die "failed to create path: $dir: $!\n" if !$next; + + PVE::Tools::fchownat(fileno($next), '', $rootuid, $rootgid, PVE::Tools::AT_EMPTY_PATH) + if defined($rootuid) && defined($rootgid); } - close $second if defined($last_component); + close $second if defined($last_component) && $second != $start_fd; $last_component = $component; $second = $fd; $fd = $next; } return ($fd, defined($last_component) && $second, $last_component) if wantarray; - close $second if defined($last_component); + close $second if defined($last_component) && $second != $start_fd; return $fd; } @@ -1156,20 +1469,77 @@ sub bindmount { # from $rootdir and $mount and walk the path from $rootdir to the final # directory to check for symlinks. sub __mount_prepare_rootdir { - my ($rootdir, $mount) = @_; + my ($rootdir, $mount, $rootuid, $rootgid) = @_; $rootdir =~ s!/+!/!g; $rootdir =~ s!/+$!!; my $mount_path = "$rootdir/$mount"; - my ($mpfd, $parentfd, $last_dir) = walk_tree_nofollow($rootdir, $mount, 1); + my ($mpfd, $parentfd, $last_dir) = walk_tree_nofollow($rootdir, $mount, 1, $rootuid, $rootgid); return ($rootdir, $mount_path, $mpfd, $parentfd, $last_dir); } # use $rootdir = undef to just return the corresponding mount path sub mountpoint_mount { - my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_; + my ($mountpoint, $rootdir, $storage_cfg, $snapname, $rootuid, $rootgid) = @_; + return __mountpoint_mount($mountpoint, $rootdir, $storage_cfg, $snapname, $rootuid, $rootgid, undef); +} + +sub mountpoint_stage { + my ($mountpoint, $stage_dir, $storage_cfg, $snapname, $rootuid, $rootgid) = @_; + my ($path, $loop, $dev) = + __mountpoint_mount($mountpoint, $stage_dir, $storage_cfg, $snapname, $rootuid, $rootgid, 1); + + if (!defined($path)) { + return undef if $! == ENOSYS; + die "failed to mount subvolume: $!\n"; + } + + # We clone the mount point and leave it there in order to keep them connected to eg. loop + # devices in case we're hotplugging (which would allow contaienrs to unmount the new mount + # point). + my $err; + my $fd = PVE::Tools::open_tree(&AT_FDCWD, $stage_dir, &OPEN_TREE_CLOEXEC | &OPEN_TREE_CLONE) + or die "open_tree() on mount point failed: $!\n"; + + return wantarray ? ($path, $loop, $dev, $fd) : $fd; +} + +sub mountpoint_insert_staged { + my ($mount_fd, $rootdir_fd, $mp_dir, $opt, $rootuid, $rootgid) = @_; + + if (!defined($rootdir_fd)) { + sysopen($rootdir_fd, '.', O_PATH | O_DIRECTORY) + or die "failed to open '.': $!\n"; + } + + my $dest_fd = walk_tree_nofollow_fd('/', $rootdir_fd, $mp_dir, 1, $rootuid, $rootgid); + + PVE::Tools::move_mount( + fileno($mount_fd), + '', + fileno($dest_fd), + '', + &MOVE_MOUNT_F_EMPTY_PATH | &MOVE_MOUNT_T_EMPTY_PATH, + ) or die "failed to move '$opt' into container hierarchy: $!\n"; +} + +# Use $stage_mount, $rootdir is treated as a temporary path to "stage" the file system. The user +# can then open a file descriptor to it which can be used with the `move_mount` syscall. +# Note that if the kernel does not support the new mount API, this will not perform any action +# and return `undef` with $! = ENOSYS. +sub __mountpoint_mount { + my ($mountpoint, $rootdir, $storage_cfg, $snapname, $rootuid, $rootgid, $stage_mount) = @_; + + if (defined($stage_mount) && !PVE::LXC::Tools::can_use_new_mount_api()) { + $! = ENOSYS; + return undef; + } + + # When staging mount points we always mount to $rootdir directly (iow. as if `mp=/`). + # This is required since __mount_prepare_rootdir() will return handles to the parent directory + # which we use in __bindmount_verify()! + my $mount = $stage_mount ? '/': $mountpoint->{mp}; my $volid = $mountpoint->{volume}; - my $mount = $mountpoint->{mp}; my $type = $mountpoint->{type}; my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota}; my $mounted_dev; @@ -1183,32 +1553,42 @@ sub mountpoint_mount { if (defined($rootdir)) { ($rootdir, $mount_path, $mpfd, $parentfd, $last_dir) = - __mount_prepare_rootdir($rootdir, $mount); + __mount_prepare_rootdir($rootdir, $mount, $rootuid, $rootgid); + } + + if (defined($stage_mount)) { + $mount_path = $rootdir; } my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1); die "unknown snapshot path for '$volid'" if !$storage && defined($snapname); - my $optstring = ''; + my $optlist = []; + + if (my $mountopts = $mountpoint->{mountoptions}) { + my @opts = split(/;/, $mountpoint->{mountoptions}); + push @$optlist, grep { PVE::LXC::Config::is_valid_mount_option($_) } @opts; + } + my $acl = $mountpoint->{acl}; if (defined($acl)) { - $optstring .= ($acl ? 'acl' : 'noacl'); + push @$optlist, ($acl ? 'acl' : 'noacl'); } + + my $optstring = join(',', @$optlist); my $readonly = $mountpoint->{ro}; - my @extra_opts = ('-o', $optstring) if $optstring; + my @extra_opts; + @extra_opts = ('-o', $optstring) if $optstring; if ($storage) { my $scfg = PVE::Storage::storage_config($storage_cfg, $storage); - # early sanity checks: - # we otherwise call realpath on the rbd url - die "containers on rbd storage without krbd are not supported\n" - if $scfg->{type} eq 'rbd' && !$scfg->{krbd}; + my $path = PVE::Storage::map_volume($storage_cfg, $volid, $snapname); - my $path = PVE::Storage::path($storage_cfg, $volid, $snapname); + $path = PVE::Storage::path($storage_cfg, $volid, $snapname) if !defined($path); my ($vtype, undef, undef, undef, undef, $isBase, $format) = PVE::Storage::parse_volname($storage_cfg, $volid); @@ -1217,19 +1597,17 @@ sub mountpoint_mount { if ($format eq 'subvol') { if ($mount_path) { - if ($snapname) { + my (undef, $name) = PVE::Storage::parse_volname($storage_cfg, $volid); + if (defined($snapname)) { + $name .= "\@$snapname"; if ($scfg->{type} eq 'zfspool') { - my $path_arg = $path; - $path_arg =~ s!^/+!!; - PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]); + PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', "$scfg->{pool}/$name", $mount_path]); } else { die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n"; } } else { if (defined($acl) && $scfg->{type} eq 'zfspool') { my $acltype = ($acl ? 'acltype=posixacl' : 'acltype=noacl'); - my (undef, $name) = PVE::Storage::parse_volname($storage_cfg, $volid); - $name .= "\@$snapname" if defined($snapname); PVE::Tools::run_command(['zfs', 'set', $acltype, "$scfg->{pool}/$name"]); } bindmount($path, $parentfd, $last_dir//$rootdir, $mount_path, $readonly, @extra_opts); @@ -1263,7 +1641,7 @@ sub mountpoint_mount { }; my $use_loopdev = 0; if ($scfg->{path}) { - $mounted_dev = run_with_loopdev($domount, $path); + $mounted_dev = run_with_loopdev($domount, $path, $readonly); $use_loopdev = 1; } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') { @@ -1293,6 +1671,91 @@ sub mountpoint_mount { die "unsupported storage"; } +sub mountpoint_hotplug($$$) { + my ($vmid, $conf, $opt, $mp, $storage_cfg) = @_; + + my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf); + + # We do the rest in a fork with an unshared mount namespace, because: + # -) change our papparmor profile to that of /usr/bin/lxc-start + # -) we're now going to 'stage' # the mountpoint, then grab it, then move into the + # container's namespace, then mount it. + + PVE::Tools::run_fork(sub { + # Pin the container pid longer, we also need to get its monitor/parent: + my ($ct_pid, $ct_pidfd) = open_lxc_pid($vmid) + or die "failed to open pidfd of container $vmid\'s init process\n"; + + my ($monitor_pid, $monitor_pidfd) = open_ppid($ct_pid) + or die "failed to open pidfd of container $vmid\'s monitor process\n"; + + my $ct_mnt_ns = $get_container_namespace->($vmid, $ct_pid, 'mnt'); + my $monitor_mnt_ns = $get_container_namespace->($vmid, $monitor_pid, 'mnt'); + + # Grab a file descriptor to our apparmor label file so we can change into the 'lxc-start' + # profile to lower our privileges to the same level we have in the start hook: + sysopen(my $aa_fd, "/proc/self/attr/current", O_WRONLY) + or die "failed to open '/proc/self/attr/current' for writing: $!\n"; + # But switch namespaces first, to make sure the namespace switches aren't blocked by + # apparmor. + + # Change into the monitor's mount namespace. We "pin" the mount into the monitor's + # namespace for it to remain active there since the container will be able to unmount + # hotplugged mount points and thereby potentially free up loop devices, which is a security + # concern. + PVE::Tools::setns(fileno($monitor_mnt_ns), PVE::Tools::CLONE_NEWNS); + chdir('/') + or die "failed to change root directory within the monitor's mount namespace: $!\n"; + + my $dir = get_staging_mount_path($opt); + + # Now switch our apparmor profile before mounting: + my $data = 'changeprofile /usr/bin/lxc-start'; + if (syswrite($aa_fd, $data, length($data)) != length($data)) { + die "failed to change apparmor profile: $!\n"; + } + # Check errors on close as well: + close($aa_fd) + or die "failed to change apparmor profile (close() failed): $!\n"; + + my $mount_fd = mountpoint_stage($mp, $dir, $storage_cfg, undef, $rootuid, $rootgid); + + PVE::Tools::setns(fileno($ct_mnt_ns), PVE::Tools::CLONE_NEWNS); + chdir('/') + or die "failed to change root directory within the container's mount namespace: $!\n"; + + mountpoint_insert_staged($mount_fd, undef, $mp->{mp}, $opt, $rootuid, $rootgid); + }); +} + +# Create a directory in the mountpoint staging tempfs. +sub get_staging_mount_path($) { + my ($opt) = @_; + + my $target = get_staging_tempfs() . "/$opt"; + if (!mkdir($target) && $! != EEXIST) { + die "failed to create directory $target: $!\n"; + } + + return $target; +} + +# Mount /run/pve/mountpoints as tmpfs +sub get_staging_tempfs() { + # We choose a path in /var/lib/lxc/ here because the lxc-start apparmor profile restricts most + # mounts to that. + my $target = '/var/lib/lxc/.pve-staged-mounts'; + if (!mkdir($target)) { + return $target if $! == EEXIST; + die "failed to create directory $target: $!\n"; + } + + PVE::Tools::mount("none", $target, 'tmpfs', 0, "size=8k,mode=755") + or die "failed to mount $target as tmpfs: $!\n"; + + return $target; +} + sub mkfs { my ($dev, $rootuid, $rootgid) = @_; @@ -1315,7 +1778,9 @@ sub format_disk { PVE::Storage::activate_volumes($storage_cfg, [$volid]); - my $path = PVE::Storage::path($storage_cfg, $volid); + my $path = PVE::Storage::map_volume($storage_cfg, $volid); + + $path = PVE::Storage::path($storage_cfg, $volid) if !defined($path); my ($vtype, undef, undef, undef, undef, $isBase, $format) = PVE::Storage::parse_volname($storage_cfg, $volid); @@ -1346,7 +1811,7 @@ sub alloc_disk { eval { my $do_format = 0; - if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') { + if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'cifs' ) { if ($size_kb > 0) { $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb); @@ -1368,7 +1833,6 @@ sub alloc_disk { } elsif ($scfg->{type} eq 'rbd') { - die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd}; $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb); $do_format = 1; } else { @@ -1390,7 +1854,7 @@ sub alloc_disk { our $NEW_DISK_RE = qr/^([^:\s]+):(\d+(\.\d+)?)$/; sub create_disks { - my ($storecfg, $vmid, $settings, $conf) = @_; + my ($storecfg, $vmid, $settings, $conf, $pending) = @_; my $vollist = []; @@ -1417,10 +1881,14 @@ sub create_disks { push @$vollist, $volid; $mountpoint->{volume} = $volid; $mountpoint->{size} = $size_kb * 1024; - $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs'); + if ($pending) { + $conf->{pending}->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs'); + } else { + $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs'); + } } else { - # use specified/existing volid/dir/device - $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs'); + # use specified/existing volid/dir/device + $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs'); } }); @@ -1434,11 +1902,155 @@ sub create_disks { # free allocated images on error if (my $err = $@) { destroy_disks($storecfg, $vollist); - die $err; + die $err; } return $vollist; } +sub update_disksize { + my ($vmid, $conf, $all_volumes) = @_; + + my $changes; + my $prefix = "CT $vmid:"; + + my $update_mp = sub { + my ($key, $mp, @param) = @_; + my $size = $all_volumes->{$mp->{volume}}->{size} // 0; + + if (!defined($mp->{size}) || $size != $mp->{size}) { + $changes = 1; + print "$prefix updated volume size of '$mp->{volume}' in config.\n"; + $mp->{size} = $size; + my $nomp = 1 if ($key eq 'rootfs'); + $conf->{$key} = PVE::LXC::Config->print_ct_mountpoint($mp, $nomp); + } + }; + + PVE::LXC::Config->foreach_mountpoint($conf, $update_mp); + + return $changes; +} + +sub update_unused { + my ($vmid, $conf, $all_volumes) = @_; + + my $changes; + my $prefix = "CT $vmid:"; + + # Note: it is allowed to define multiple storage entries with the same path + # (alias), so we need to check both 'volid' and real 'path' (two different + # volid can point to the same path). + + # used and unused disks + my $refpath = {}; + my $orphans = {}; + + foreach my $opt (keys %$conf) { + next if ($opt !~ m/^unused\d+$/); + my $vol = $all_volumes->{$conf->{$opt}}; + $refpath->{$vol->{path}} = $vol->{volid}; + } + + foreach my $key (keys %$all_volumes) { + my $vol = $all_volumes->{$key}; + my $in_use = PVE::LXC::Config->is_volume_in_use($conf, $vol->{volid}); + my $path = $vol->{path}; + + if ($in_use) { + $refpath->{$path} = $key; + delete $orphans->{$path}; + } else { + if ((!$orphans->{$path}) && (!$refpath->{$path})) { + $orphans->{$path} = $key; + } + } + } + + for my $key (keys %$orphans) { + my $disk = $orphans->{$key}; + my $unused = PVE::LXC::Config->add_unused_volume($conf, $disk); + + if ($unused) { + $changes = 1; + print "$prefix add unreferenced volume '$disk' as '$unused' to config.\n"; + } + } + + return $changes; +} + +sub scan_volids { + my ($cfg, $vmid) = @_; + + my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid); + + my $all_volumes = {}; + foreach my $storeid (keys %$info) { + foreach my $item (@{$info->{$storeid}}) { + my $volid = $item->{volid}; + next if !($volid && $item->{size}); + $item->{path} = PVE::Storage::path($cfg, $volid); + $all_volumes->{$volid} = $item; + } + } + + return $all_volumes; +} + +sub rescan { + my ($vmid, $nolock, $dryrun) = @_; + + my $cfg = PVE::Storage::config(); + + # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage + # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html + foreach my $stor (keys %{$cfg->{ids}}) { + delete($cfg->{ids}->{$stor}) if !$cfg->{ids}->{$stor}->{content}->{rootdir}; + } + + print "rescan volumes...\n"; + my $all_volumes = scan_volids($cfg, $vmid); + + my $updatefn = sub { + my ($vmid) = @_; + + my $changes; + my $conf = PVE::LXC::Config->load_config($vmid); + + PVE::LXC::Config->check_lock($conf); + + my $vm_volids = {}; + foreach my $volid (keys %$all_volumes) { + my $info = $all_volumes->{$volid}; + $vm_volids->{$volid} = $info if $info->{vmid} == $vmid; + } + + my $upu = update_unused($vmid, $conf, $vm_volids); + my $upd = update_disksize($vmid, $conf, $vm_volids); + $changes = $upu || $upd; + + PVE::LXC::Config->write_config($vmid, $conf) if $changes && !$dryrun; + }; + + if (defined($vmid)) { + if ($nolock) { + &$updatefn($vmid); + } else { + PVE::LXC::Config->lock_config($vmid, $updatefn, $vmid); + } + } else { + my $vmlist = config_list(); + foreach my $vmid (keys %$vmlist) { + if ($nolock) { + &$updatefn($vmid); + } else { + PVE::LXC::Config->lock_config($vmid, $updatefn, $vmid); + } + } + } +} + + # bash completion helper sub complete_os_templates { @@ -1544,6 +2156,13 @@ sub userns_command { sub vm_start { my ($vmid, $conf, $skiplock) = @_; + # apply pending changes while starting + if (scalar(keys %{$conf->{pending}})) { + my $storecfg = PVE::Storage::config(); + PVE::LXC::Config->vmconfig_apply_pending($vmid, $conf, $storecfg); + $conf = PVE::LXC::Config->load_config($vmid); # update/reload + } + update_lxc_config($vmid, $conf); my $skiplock_flag_fn = "/run/lxc/skiplock-$vmid"; @@ -1555,11 +2174,13 @@ sub vm_start { my $cmd = ['systemctl', 'start', "pve-container\@$vmid"]; + PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1); eval { PVE::Tools::run_command($cmd); }; if (my $err = $@) { unlink $skiplock_flag_fn; die $err; } + PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start'); return; } @@ -1568,8 +2189,11 @@ sub vm_start { # This is necessary because we want the post-stop hook to have completed its # unmount-all step, but post-stop happens after lxc puts the container into the # STOPPED state. +# $kill - if true it will always do an immediate hard-stop +# $shutdown_timeout - the timeout to wait for a gracefull shutdown +# $kill_after_timeout - if true, send a hardstop if shutdown timed out sub vm_stop { - my ($vmid, $kill, $shutdown_timeout, $exit_timeout) = @_; + my ($vmid, $kill, $shutdown_timeout, $kill_after_timeout) = @_; # Open the container's command socket. my $path = "\0/var/lib/lxc/$vmid/command"; @@ -1582,16 +2206,24 @@ sub vm_stop { die "failed to open container ${vmid}'s command socket: $!\n"; } + my $conf = PVE::LXC::Config->load_config($vmid); + PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop'); + # Stop the container: my $cmd = ['lxc-stop', '-n', $vmid]; if ($kill) { push @$cmd, '--kill'; # doesn't allow timeouts - } elsif (defined($shutdown_timeout)) { - push @$cmd, '--timeout', $shutdown_timeout; - # Give run_command 5 extra seconds - $shutdown_timeout += 5; + } else { + # lxc-stop uses a default timeout + push @$cmd, '--nokill' if !$kill_after_timeout; + + if (defined($shutdown_timeout)) { + push @$cmd, '--timeout', $shutdown_timeout; + # Give run_command 5 extra seconds + $shutdown_timeout += 5; + } } eval { PVE::Tools::run_command($cmd, timeout => $shutdown_timeout) }; @@ -1599,18 +2231,25 @@ sub vm_stop { warn $@ if $@; } - my $result = 1; - my $wait = sub { $result = <$sock>; }; - if (defined($exit_timeout)) { - PVE::Tools::run_with_timeout($exit_timeout, $wait); - } else { - $wait->(); - } + my $result = <$sock>; return if !defined $result; # monitor is gone and the ct has stopped. die "container did not stop\n"; } +sub vm_reboot { + my ($vmid, $timeout, $skiplock) = @_; + + PVE::LXC::Config->lock_config($vmid, sub { + return if !check_running($vmid); + + vm_stop($vmid, 0, $timeout, 1); # kill if timeout exceeds + + my $conf = PVE::LXC::Config->load_config($vmid); + vm_start($vmid, $conf); + }); +} + sub run_unshared { my ($code) = @_; @@ -1624,31 +2263,38 @@ sub run_unshared { } my $copy_volume = sub { - my ($src_volid, $src, $dst_volid, $dest, $storage_cfg, $snapname) = @_; + my ($src_volid, $src, $dst_volid, $dest, $storage_cfg, $snapname, $bwlimit, $rootuid, $rootgid) = @_; - my $src_mp = { volume => $src_volid, mp => '/' }; + my $src_mp = { volume => $src_volid, mp => '/', ro => 1 }; $src_mp->{type} = PVE::LXC::Config->classify_mountpoint($src_volid); - my $dst_mp = { volume => $dst_volid, mp => '/' }; + my $dst_mp = { volume => $dst_volid, mp => '/', ro => 0 }; $dst_mp->{type} = PVE::LXC::Config->classify_mountpoint($dst_volid); my @mounted; eval { # mount and copy mkdir $src; - mountpoint_mount($src_mp, $src, $storage_cfg, $snapname); + mountpoint_mount($src_mp, $src, $storage_cfg, $snapname, $rootuid, $rootgid); push @mounted, $src; mkdir $dest; - mountpoint_mount($dst_mp, $dest, $storage_cfg); + mountpoint_mount($dst_mp, $dest, $storage_cfg, undef, $rootuid, $rootgid); push @mounted, $dest; + $bwlimit //= 0; + PVE::Tools::run_command(['/usr/bin/rsync', '--stats', '-X', '-A', '--numeric-ids', '-aH', '--whole-file', '--sparse', '--one-file-system', - "$src/", $dest]); + "--bwlimit=$bwlimit", "$src/", $dest]); }; my $err = $@; + + # Wait for rsync's children to release dest so that + # consequent file operations (umount, remove) are possible + while ((system {"fuser"} "fuser", "-s", $dest) == 0) {sleep 1}; + foreach my $mount (reverse @mounted) { - eval { PVE::Tools::run_command(['/bin/umount', '--lazy', $mount], errfunc => sub{})}; + eval { PVE::Tools::run_command(['/bin/umount', $mount], errfunc => sub{})}; warn "Can't umount $mount\n" if $@; } @@ -1662,7 +2308,7 @@ my $copy_volume = sub { # Should not be called after unsharing the mount namespace! sub copy_volume { - my ($mp, $vmid, $storage, $storage_cfg, $conf, $snapname) = @_; + my ($mp, $vmid, $storage, $storage_cfg, $conf, $snapname, $bwlimit) = @_; die "cannot copy volumes of type $mp->{type}\n" if $mp->{type} ne 'volume'; File::Path::make_path("/var/lib/lxc/$vmid"); @@ -1677,6 +2323,8 @@ sub copy_volume { # (not-mounted) directory. my $new_volid; eval { + # Make sure $mp contains a correct size. + $mp->{size} = PVE::Storage::volume_size_info($storage_cfg, $mp->{volume}); my $needs_chown; ($new_volid, $needs_chown) = alloc_disk($storage_cfg, $vmid, $storage, $mp->{size}/1024, $rootuid, $rootgid); if ($needs_chown) { @@ -1686,7 +2334,7 @@ sub copy_volume { } run_unshared(sub { - $copy_volume->($mp->{volume}, $src, $new_volid, $dest, $storage_cfg, $snapname); + $copy_volume->($mp->{volume}, $src, $new_volid, $dest, $storage_cfg, $snapname, $bwlimit, $rootuid, $rootgid); }); }; if (my $err = $@) {