use File::Path;
use File::Spec;
use Cwd qw();
-use Fcntl qw(O_RDONLY O_NOFOLLOW O_DIRECTORY);
-use Errno qw(ELOOP ENOTDIR EROFS ECONNREFUSED);
+use Fcntl qw(O_RDONLY O_WRONLY O_NOFOLLOW O_DIRECTORY);
+use Errno qw(ELOOP ENOTDIR EROFS ECONNREFUSED ENOSYS EEXIST);
use IO::Socket::UNIX;
use PVE::Exception qw(raise_perm_exc);
use PVE::SafeSyslog;
use PVE::INotify;
use PVE::JSONSchema qw(get_standard_option);
-use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full O_PATH);
+use PVE::Tools qw(
+ dir_glob_foreach file_get_contents file_set_contents lock_file
+ lock_file_full AT_FDCWD O_PATH $IPV4RE $IPV6RE
+);
use PVE::CpuSet;
use PVE::Network;
use PVE::AccessControl;
use PVE::ProcFSTools;
-use PVE::Syscall;
+use PVE::Syscall qw(:fsmount);
use PVE::LXC::Config;
-use PVE::GuestHelpers;
+use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
+use PVE::LXC::Tools;
+use PVE::LXC::CGroup;
use Time::HiRes qw (gettimeofday);
+my $have_sdn;
+eval {
+ require PVE::Network::SDN::Zones;
+ $have_sdn = 1;
+};
my $LXC_CONFIG_PATH = '/usr/share/lxc/config';
return $res;
}
-sub destroy_config {
- my ($vmid) = @_;
-
- my $config_fn = PVE::LXC::Config->config_file($vmid, $nodename);
- unlink $config_fn or die "failed to remove config file: $!\n";
-}
-
# container status helpers
sub list_active_containers {
my $last_proc_vmid_stat;
-my $parse_cpuacct_stat = sub {
- my ($vmid, $unprivileged) = @_;
-
- my $raw = read_cgroup_value('cpuacct', $vmid, $unprivileged, 'cpuacct.stat', 1);
-
- my $stat = {};
-
- if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
-
- $stat->{utime} = $1;
- $stat->{stime} = $2;
-
- }
-
- return $stat;
-};
-
our $vmstatus_return_properties = {
vmid => get_standard_option('pve-vmid'),
status => {
optional => 1,
},
lock => {
- description => "The current lock, if any.",
+ description => "The current config lock, if any.",
+ type => 'string',
+ optional => 1,
+ },
+ tags => {
+ description => "The current configured tags, if any.",
type => 'string',
optional => 1,
}
$d->{cpus} = $cpucount if !$d->{cpus};
$d->{lock} = $conf->{lock} || '';
+ $d->{tags} = $conf->{tags} if defined($conf->{tags});
if ($d->{pid}) {
my $res = get_container_disk_usage($vmid, $d->{pid});
my $unpriv = $unprivileged->{$vmid};
- if (-d '/sys/fs/cgroup/memory') {
- my $memory_stat = read_cgroup_list('memory', $vmid, $unpriv, 'memory.stat');
- my $mem_usage_in_bytes = read_cgroup_value('memory', $vmid, $unpriv, 'memory.usage_in_bytes');
+ my $cgroups = PVE::LXC::CGroup->new($vmid);
- $d->{mem} = $mem_usage_in_bytes - $memory_stat->{total_cache};
- $d->{swap} = read_cgroup_value('memory', $vmid, $unpriv, 'memory.memsw.usage_in_bytes') - $mem_usage_in_bytes;
+ if (defined(my $mem = $cgroups->get_memory_stat())) {
+ $d->{mem} = $mem->{mem};
+ $d->{swap} = $mem->{swap};
} else {
$d->{mem} = 0;
$d->{swap} = 0;
}
- if (-d '/sys/fs/cgroup/blkio') {
- my $blkio_bytes = read_cgroup_value('blkio', $vmid, 0, 'blkio.throttle.io_service_bytes', 1); # don't check if unpriv
- my @bytes = split(/\n/, $blkio_bytes);
- foreach my $byte (@bytes) {
- if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
- $d->{diskread} += $2 if $key eq 'Read';
- $d->{diskwrite} += $2 if $key eq 'Write';
- }
- }
+ if (defined(my $blkio = $cgroups->get_io_stats())) {
+ $d->{diskread} = $blkio->{diskread};
+ $d->{diskwrite} = $blkio->{diskwrite};
} else {
$d->{diskread} = 0;
$d->{diskwrite} = 0;
}
- if (-d '/sys/fs/cgroup/cpuacct') {
- my $pstat = $parse_cpuacct_stat->($vmid, $unpriv);
-
- my $used = $pstat->{utime} + $pstat->{stime};
+ if (defined(my $cpu = $cgroups->get_cpu_stat())) {
+ # Total time (in milliseconds) used up by the cpu.
+ my $used_ms = $cpu->{utime} + $cpu->{stime};
my $old = $last_proc_vmid_stat->{$vmid};
if (!$old) {
$last_proc_vmid_stat->{$vmid} = {
time => $cdtime,
- used => $used,
+ used => $used_ms,
cpu => 0,
};
next;
}
- my $dtime = ($cdtime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
-
- if ($dtime > 1000) {
- my $dutime = $used - $old->{used};
-
- $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
+ my $delta_ms = ($cdtime - $old->{time}) * $cpucount * 1000.0;
+ if ($delta_ms > 1000.0) {
+ my $delta_used_ms = $used_ms - $old->{used};
+ $d->{cpu} = (($delta_used_ms / $delta_ms) * $cpucount) / $d->{cpus};
$last_proc_vmid_stat->{$vmid} = {
time => $cdtime,
- used => $used,
+ used => $used_ms,
cpu => $d->{cpu},
};
} else {
return $list;
}
-sub read_cgroup_list($$$$) {
- my ($group, $vmid, $unprivileged, $name) = @_;
-
- my $content = read_cgroup_value($group, $vmid, $unprivileged, $name, 1);
-
- return { split(/\s+/, $content) };
-}
-
-sub read_cgroup_value($$$$$) {
- my ($group, $vmid, $unprivileged, $name, $full) = @_;
-
- my $nsdir = $unprivileged ? '' : 'ns/';
- my $path = "/sys/fs/cgroup/$group/lxc/$vmid/${nsdir}$name";
-
- return PVE::Tools::file_get_contents($path) if $full;
-
- return PVE::Tools::file_read_firstline($path);
-}
-
-sub write_cgroup_value {
- my ($group, $vmid, $name, $value) = @_;
-
- my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
- PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
-
-}
-
sub find_lxc_console_pids {
my $res = {};
return $pid;
}
+sub open_pid_fd($) {
+ my ($pid) = @_;
+ sysopen(my $fd, "/proc/$pid", O_RDONLY | O_DIRECTORY)
+ or die "failed to open /proc/$pid pid fd\n";
+ return $fd;
+}
+
+sub open_lxc_pid {
+ my ($vmid) = @_;
+
+ # Find the pid and open:
+ my $pid = find_lxc_pid($vmid);
+ my $fd = open_pid_fd($pid);
+
+ # Verify:
+ my $pid2 = find_lxc_pid($vmid);
+
+ return () if $pid != $pid2;
+ return ($pid, $fd);
+}
+
+sub open_ppid {
+ my ($pid) = @_;
+
+ # Find the parent pid via proc and open it:
+ my $stat = PVE::ProcFSTools::read_proc_pid_stat($pid);
+ my $ppid = $stat->{ppid} // die "failed to get parent pid\n";
+
+ my $fd = open_pid_fd($ppid);
+
+ # Verify:
+ $stat = PVE::ProcFSTools::read_proc_pid_stat($pid);
+ my $ppid2 = $stat->{ppid} // die "failed to get parent pid for verification\n";
+
+ return () if $ppid != $ppid2;
+ return ($ppid, $fd);
+}
+
# Note: we cannot use Net:IP, because that only allows strict
# CIDR networks
sub parse_ipv4_cidr {
return wantarray ? ($v1, $v2) : $v1;
}
-# Currently we do not need to create seccomp profile 'files' as the only
-# choice our configuration actually allows is "with or without keyctl()",
-# so we distinguish between using lxc's "default" seccomp profile and our
-# added pve-userns.seccomp file.
+# With seccomp trap to userspace we now have the ability to optionally forward
+# certain syscalls to the "host" to handle (via our pve-lxc-syscalld daemon).
#
-# This returns a configuration line added to the raw lxc config.
+# This means that there are cases where we need to create an extra seccomp
+# profile for the container to load.
+#
+# This returns a configuration snippet added to the raw lxc config.
sub make_seccomp_config {
- my ($conf, $unprivileged, $features) = @_;
+ my ($conf, $vmid, $conf_dir, $unprivileged, $features) = @_;
# User-configured profile has precedence, note that the user's entry would
# be written 'after' this line anyway...
if (PVE::LXC::Config->has_lxc_entry($conf, 'lxc.seccomp.profile')) {
# Warn the user if this conflicts with a feature:
- if ($features->{keyctl}) {
- warn "explicitly configured lxc.seccomp.profile overrides the following settings: features:keyctl\n";
- }
+ my $warn = join(', ', grep { $features->{$_} } qw(keyctl mknod));
+ warn "explicitly configured lxc.seccomp.profile overrides the following settings: $warn\n"
+ if length($warn) > 0;
return '';
}
# well)
return '' if !$unprivileged;
+ my $rules = {
+ keyctl => ['errno 38'],
+ };
+
+ my $raw_conf = '';
+
# Unprivileged containers will get keyctl() disabled by default as a
# workaround for systemd-networkd behavior. But we have an option to
# explicitly enable it:
- return '' if $features->{keyctl};
+ if ($features->{keyctl}) {
+ delete $rules->{keyctl};
+ }
+
+ # By default, unprivileged containers cannot use `mknod` at all.
+ # Since lxc 3.2, we can use seccomp's trap to userspace feature for this,
+ # but for now this is experimental, so it has to be enabled via a feature
+ # flag.
+ # Note that we only handle block and char devices (like lxd), the rest we
+ # leave up to the kernel. We may in the future remove this if seccomp gets
+ # a way to tell the kernel to "continue" a syscall.
+ if ($features->{mknod}) {
+ my ($ok, $kernel) = PVE::ProcFSTools::check_kernel_release(5, 3);
+ if (!$ok) {
+ die "'mknod' feature requested, but kernel too old (found $kernel, required >= 5.3)\n";
+ }
- # Finally we're in an unprivileged container without `keyctl` set
- # explicitly. We have a file prepared for this:
- return "lxc.seccomp.profile = $LXC_CONFIG_PATH/pve-userns.seccomp\n";
+ $raw_conf .= "lxc.seccomp.notify.proxy = unix:/run/pve/lxc-syscalld.sock\n";
+ $raw_conf .= "lxc.seccomp.notify.cookie = $vmid\n";
+
+ $rules->{mknod} = [
+ # condition: (mode & S_IFMT) == S_IFCHR
+ 'notify [1,8192,SCMP_CMP_MASKED_EQ,61440]',
+ # condition: (mode & S_IFMT) == S_IFBLK
+ 'notify [1,24576,SCMP_CMP_MASKED_EQ,61440]',
+ ];
+ $rules->{mknodat} = [
+ # condition: (mode & S_IFMT) == S_IFCHR
+ 'notify [2,8192,SCMP_CMP_MASKED_EQ,61440]',
+ # condition: (mode & S_IFMT) == S_IFBLK
+ 'notify [2,24576,SCMP_CMP_MASKED_EQ,61440]',
+ ];
+ }
+
+ # Now build the custom seccomp rule text...
+ my $extra_rules = join("\n", map {
+ my $syscall = $_;
+ map { "$syscall $_" } $rules->{$syscall}->@*
+ } sort keys %$rules) . "\n";
+
+ return $raw_conf if $extra_rules eq "\n";
+
+ # We still have the "most common" config readily available, so don't write
+ # out that one:
+ if ($raw_conf eq '' && $extra_rules eq "keyctl errno 38\n") {
+ # we have no extra $raw_conf and use the same we had in pve 6.1:
+ return "lxc.seccomp.profile = $LXC_CONFIG_PATH/pve-userns.seccomp\n";
+ }
+
+ # Write the rule file to the container's config path:
+ my $rule_file = "$conf_dir/rules.seccomp";
+ my $rule_data = file_get_contents("$LXC_CONFIG_PATH/common.seccomp")
+ . $extra_rules;
+ file_set_contents($rule_file, $rule_data);
+ $raw_conf .= "lxc.seccomp.profile = $rule_file\n";
+
+ return $raw_conf;
}
# Since lxc-3.0.2 we can have lxc generate a profile for the container
my $features = PVE::LXC::Config->parse_features($conf->{features});
- $raw .= make_seccomp_config($conf, $unprivileged, $features);
+ $raw .= make_seccomp_config($conf, $vmid, $dir, $unprivileged, $features);
$raw .= make_apparmor_config($conf, $unprivileged, $features);
if ($features->{fuse}) {
$raw .= "lxc.apparmor.raw = mount fstype=fuse,\n";
$raw .= "lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file 0 0\n";
}
+ if ($unprivileged && !$features->{force_rw_sys}) {
+ # unpriv. CT default to sys:rw, but that doesn't always plays well with
+ # systemd, e.g., systemd-networkd https://systemd.io/CONTAINER_INTERFACE/
+ $raw .= "lxc.mount.auto = sys:mixed\n";
+ }
+
# WARNING: DO NOT REMOVE this without making sure that loop device nodes
# cannot be exposed to the container with r/w access (cgroup perms).
# When this is enabled mounts will still remain in the monitor's namespace
$raw .= "lxc.net.$ind.mtu = $d->{mtu}\n" if defined($d->{mtu});
}
- if ($cgv1->{cpuset}) {
- my $had_cpuset = 0;
- if (my $lxcconf = $conf->{lxc}) {
- foreach my $entry (@$lxcconf) {
- my ($k, $v) = @$entry;
- $had_cpuset = 1 if $k eq 'lxc.cgroup.cpuset.cpus';
- $raw .= "$k = $v\n";
- }
+ my $had_cpuset = 0;
+ if (my $lxcconf = $conf->{lxc}) {
+ foreach my $entry (@$lxcconf) {
+ my ($k, $v) = @$entry;
+ $had_cpuset = 1 if $k eq 'lxc.cgroup.cpuset.cpus';
+ $raw .= "$k = $v\n";
}
+ }
- my $cores = $conf->{cores};
- if (!$had_cpuset && $cores) {
- my $cpuset = eval { PVE::CpuSet->new_from_cgroup('lxc', 'effective_cpus') };
- $cpuset = PVE::CpuSet->new_from_cgroup('', 'effective_cpus') if !$cpuset;
- my @members = $cpuset->members();
- while (scalar(@members) > $cores) {
- my $randidx = int(rand(scalar(@members)));
- $cpuset->delete($members[$randidx]);
- splice(@members, $randidx, 1); # keep track of the changes
- }
- $raw .= "lxc.cgroup.cpuset.cpus = ".$cpuset->short_string()."\n";
+ my $cpuset;
+ my $cpuset_cgroup = eval { PVE::LXC::CGroup::cpuset_controller_path() };
+ if (defined($cpuset_cgroup)) {
+ $cpuset = eval { PVE::CpuSet->new_from_path("$cpuset_cgroup/lxc", 1) }
+ || PVE::CpuSet->new_from_path($cpuset_cgroup, 1);
+ }
+ my $cores = $conf->{cores};
+ if (!$had_cpuset && $cores && $cpuset) {
+ my @members = $cpuset->members();
+ while (scalar(@members) > $cores) {
+ my $randidx = int(rand(scalar(@members)));
+ $cpuset->delete($members[$randidx]);
+ splice(@members, $randidx, 1); # keep track of the changes
}
+ $raw .= "lxc.cgroup.cpuset.cpus = ".$cpuset->short_string()."\n";
}
File::Path::mkpath("$dir/rootfs");
return if PVE::LXC::Config->classify_mountpoint($volume) ne 'volume';
my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
- PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
+
+ if ($vmid == $owner) {
+ PVE::Storage::vdisk_free($storage_cfg, $volume);
+ } else {
+ warn "ignore deletion of '$volume', CT $vmid isn't the owner!\n";
+ }
}
sub destroy_lxc_container {
if (defined $replacement_conf) {
PVE::LXC::Config->write_config($vmid, $replacement_conf);
} else {
- destroy_config($vmid);
+ PVE::LXC::Config->destroy_config($vmid);
}
-
- #my $cmd = ['lxc-destroy', '-n', $vmid ];
- #PVE::Tools::run_command($cmd);
}
sub vm_stop_cleanup {
my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
-
- eval {
- if (!$keepActive) {
- my $vollist = PVE::LXC::Config->get_vm_volumes($conf);
- PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
- }
+ return if $keepActive;
+
+ eval {
+ my $vollist = PVE::LXC::Config->get_vm_volumes($conf);
+ PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
};
warn $@ if $@; # avoid errors - just warn
}
-my $safe_num_ne = sub {
- my ($a, $b) = @_;
-
- return 0 if !defined($a) && !defined($b);
- return 1 if !defined($a);
- return 1 if !defined($b);
-
- return $a != $b;
-};
-
-my $safe_string_ne = sub {
- my ($a, $b) = @_;
-
- return 0 if !defined($a) && !defined($b);
- return 1 if !defined($a);
- return 1 if !defined($b);
-
- return $a ne $b;
-};
-
sub update_net {
my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
if (my $oldnetcfg = $conf->{$opt}) {
my $oldnet = PVE::LXC::Config->parse_lxc_network($oldnetcfg);
- if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
- &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
+ if (safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
+ safe_string_ne($oldnet->{name}, $newnet->{name})) {
PVE::Network::veth_delete($veth);
delete $conf->{$opt};
hotplug_net($vmid, $conf, $opt, $newnet, $netid);
} else {
- if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
- &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
- &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
+ if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
+ safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
+ safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
if ($oldnet->{bridge}) {
PVE::Network::tap_unplug($veth);
PVE::LXC::Config->write_config($vmid, $conf);
}
- PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
+ if ($have_sdn) {
+ PVE::Network::SDN::Zones::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
+ } else {
+ PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
+ }
+
# This includes the rate:
foreach (qw(bridge tag firewall rate)) {
$oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
}
- } elsif (&$safe_string_ne($oldnet->{rate}, $newnet->{rate})) {
+ } elsif (safe_string_ne($oldnet->{rate}, $newnet->{rate})) {
# Rate can be applied on its own but any change above needs to
# include the rate in tap_plug since OVS resets everything.
PVE::Network::tap_rate_limit($veth, $newnet->{rate});
my $vethpeer = $veth . "p";
my $eth = $newnet->{name};
- PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
- PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
+ if ($have_sdn) {
+ PVE::Network::SDN::Zones::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
+ PVE::Network::SDN::Zones::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
+ } else {
+ PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
+ PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
+ }
# attach peer in container
my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
my $oldip = $optdata->{$ip};
my $oldgw = $optdata->{$gw};
- my $change_ip = &$safe_string_ne($oldip, $newip);
- my $change_gw = &$safe_string_ne($oldgw, $newgw);
+ my $change_ip = safe_string_ne($oldip, $newip);
+ my $change_gw = safe_string_ne($oldgw, $newgw);
return if !$change_ip && !$change_gw;
}
+my $open_namespace = sub {
+ my ($vmid, $pid, $kind) = @_;
+ sysopen my $fd, "/proc/$pid/ns/$kind", O_RDONLY
+ or die "failed to open $kind namespace of container $vmid: $!\n";
+ return $fd;
+};
+
my $enter_namespace = sub {
- my ($vmid, $pid, $which, $type) = @_;
- sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY
- or die "failed to open $which namespace of container $vmid: $!\n";
+ my ($vmid, $pid, $kind, $type) = @_;
+ my $fd = $open_namespace->($vmid, $pid, $kind);
PVE::Tools::setns(fileno($fd), $type)
- or die "failed to enter $which namespace of container $vmid: $!\n";
+ or die "failed to enter $kind namespace of container $vmid: $!\n";
close $fd;
};
+my $get_container_namespace = sub {
+ my ($vmid, $pid, $kind) = @_;
+
+ my $pidfd;
+ if (!defined($pid)) {
+ # Pin the pid while we're grabbing its stuff from /proc
+ ($pid, $pidfd) = open_lxc_pid($vmid)
+ or die "failed to open pidfd of container $vmid\'s init process\n";
+ }
+
+ return $open_namespace->($vmid, $pid, $kind);
+};
+
my $do_syncfs = sub {
my ($vmid, $pid, $socket) = @_;
my $rootdir = "/var/lib/lxc/$vmid/rootfs";
my $volid_list = PVE::LXC::Config->get_vm_volumes($conf);
+ my $res = 1;
+
PVE::LXC::Config->foreach_mountpoint_reverse($conf, sub {
my ($ms, $mountpoint) = @_;
};
if (my $err = $@) {
if ($noerr) {
+ $res = 0;
warn $err;
} else {
die $err;
}
}
});
+
+ return $res; # tell caller if (some) umounts failed for the noerr case
}
sub mount_all {
my $volid_list = PVE::LXC::Config->get_vm_volumes($conf);
PVE::Storage::activate_volumes($storage_cfg, $volid_list);
+ my (undef, $rootuid, $rootgid) = parse_id_maps($conf);
+
eval {
PVE::LXC::Config->foreach_mountpoint($conf, sub {
my ($ms, $mountpoint) = @_;
$mountpoint->{ro} = 0 if $ignore_ro;
- mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
+ mountpoint_mount($mountpoint, $rootdir, $storage_cfg, undef, $rootuid, $rootgid);
});
};
if (my $err = $@) {
# * the 2nd deepest directory (parent of the above)
# * directory name of the last directory
# So that the path $2/$3 should lead to $1 afterwards.
-sub walk_tree_nofollow($$$) {
- my ($start, $subdir, $mkdir) = @_;
-
- # splitdir() returns '' for empty components including the leading /
- my @comps = grep { length($_)>0 } File::Spec->splitdir($subdir);
+sub walk_tree_nofollow($$$;$$) {
+ my ($start, $subdir, $mkdir, $rootuid, $rootgid) = @_;
sysopen(my $fd, $start, O_PATH | O_DIRECTORY)
or die "failed to open start directory $start: $!\n";
- my $dir = $start;
+ return walk_tree_nofollow_fd($start, $fd, $subdir, $mkdir, $rootuid, $rootgid);
+}
+
+
+sub walk_tree_nofollow_fd($$$$;$$) {
+ my ($start_dirname, $start_fd, $subdir, $mkdir, $rootuid, $rootgid) = @_;
+
+ # splitdir() returns '' for empty components including the leading /
+ my @comps = grep { length($_)>0 } File::Spec->splitdir($subdir);
+
+ my $fd = $start_fd;
+ my $dir = $start_dirname;
my $last_component = undef;
my $second = $fd;
foreach my $component (@comps) {
$next = PVE::Tools::openat(fileno($fd), $component, O_NOFOLLOW | O_DIRECTORY);
die "failed to create path: $dir: $!\n" if !$next;
+
+ PVE::Tools::fchownat(fileno($next), '', $rootuid, $rootgid, PVE::Tools::AT_EMPTY_PATH)
+ if defined($rootuid) && defined($rootgid);
}
- close $second if defined($last_component);
+ close $second if defined($last_component) && $second != $start_fd;
$last_component = $component;
$second = $fd;
$fd = $next;
}
return ($fd, defined($last_component) && $second, $last_component) if wantarray;
- close $second if defined($last_component);
+ close $second if defined($last_component) && $second != $start_fd;
return $fd;
}
# from $rootdir and $mount and walk the path from $rootdir to the final
# directory to check for symlinks.
sub __mount_prepare_rootdir {
- my ($rootdir, $mount) = @_;
+ my ($rootdir, $mount, $rootuid, $rootgid) = @_;
$rootdir =~ s!/+!/!g;
$rootdir =~ s!/+$!!;
my $mount_path = "$rootdir/$mount";
- my ($mpfd, $parentfd, $last_dir) = walk_tree_nofollow($rootdir, $mount, 1);
+ my ($mpfd, $parentfd, $last_dir) = walk_tree_nofollow($rootdir, $mount, 1, $rootuid, $rootgid);
return ($rootdir, $mount_path, $mpfd, $parentfd, $last_dir);
}
# use $rootdir = undef to just return the corresponding mount path
sub mountpoint_mount {
- my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
+ my ($mountpoint, $rootdir, $storage_cfg, $snapname, $rootuid, $rootgid) = @_;
+ return __mountpoint_mount($mountpoint, $rootdir, $storage_cfg, $snapname, $rootuid, $rootgid, undef);
+}
+
+sub mountpoint_stage {
+ my ($mountpoint, $stage_dir, $storage_cfg, $snapname, $rootuid, $rootgid) = @_;
+ my ($path, $loop, $dev) =
+ __mountpoint_mount($mountpoint, $stage_dir, $storage_cfg, $snapname, $rootuid, $rootgid, 1);
+
+ if (!defined($path)) {
+ return undef if $! == ENOSYS;
+ die "failed to mount subvolume: $!\n";
+ }
+
+ # We clone the mount point and leave it there in order to keep them connected to eg. loop
+ # devices in case we're hotplugging (which would allow contaienrs to unmount the new mount
+ # point).
+ my $err;
+ my $fd = PVE::Tools::open_tree(&AT_FDCWD, $stage_dir, &OPEN_TREE_CLOEXEC | &OPEN_TREE_CLONE)
+ or die "open_tree() on mount point failed: $!\n";
+
+ return wantarray ? ($path, $loop, $dev, $fd) : $fd;
+}
+
+sub mountpoint_insert_staged {
+ my ($mount_fd, $rootdir_fd, $mp_dir, $opt, $rootuid, $rootgid) = @_;
+
+ if (!defined($rootdir_fd)) {
+ sysopen($rootdir_fd, '.', O_PATH | O_DIRECTORY)
+ or die "failed to open '.': $!\n";
+ }
+
+ my $dest_fd = walk_tree_nofollow_fd('/', $rootdir_fd, $mp_dir, 1, $rootuid, $rootgid);
+
+ PVE::Tools::move_mount(
+ fileno($mount_fd),
+ '',
+ fileno($dest_fd),
+ '',
+ &MOVE_MOUNT_F_EMPTY_PATH | &MOVE_MOUNT_T_EMPTY_PATH,
+ ) or die "failed to move '$opt' into container hierarchy: $!\n";
+}
+
+# Use $stage_mount, $rootdir is treated as a temporary path to "stage" the file system. The user
+# can then open a file descriptor to it which can be used with the `move_mount` syscall.
+# Note that if the kernel does not support the new mount API, this will not perform any action
+# and return `undef` with $! = ENOSYS.
+sub __mountpoint_mount {
+ my ($mountpoint, $rootdir, $storage_cfg, $snapname, $rootuid, $rootgid, $stage_mount) = @_;
+
+ if (defined($stage_mount) && !PVE::LXC::Tools::can_use_new_mount_api()) {
+ $! = ENOSYS;
+ return undef;
+ }
+
+ # When staging mount points we always mount to $rootdir directly (iow. as if `mp=/`).
+ # This is required since __mount_prepare_rootdir() will return handles to the parent directory
+ # which we use in __bindmount_verify()!
+ my $mount = $stage_mount ? '/': $mountpoint->{mp};
my $volid = $mountpoint->{volume};
- my $mount = $mountpoint->{mp};
my $type = $mountpoint->{type};
my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota};
my $mounted_dev;
if (defined($rootdir)) {
($rootdir, $mount_path, $mpfd, $parentfd, $last_dir) =
- __mount_prepare_rootdir($rootdir, $mount);
+ __mount_prepare_rootdir($rootdir, $mount, $rootuid, $rootgid);
+ }
+
+ if (defined($stage_mount)) {
+ $mount_path = $rootdir;
}
my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
- my $optstring = '';
+ my $optlist = [];
+
+ if (my $mountopts = $mountpoint->{mountoptions}) {
+ my @opts = split(/;/, $mountpoint->{mountoptions});
+ push @$optlist, grep { PVE::LXC::Config::is_valid_mount_option($_) } @opts;
+ }
+
my $acl = $mountpoint->{acl};
if (defined($acl)) {
- $optstring .= ($acl ? 'acl' : 'noacl');
+ push @$optlist, ($acl ? 'acl' : 'noacl');
}
+
+ my $optstring = join(',', @$optlist);
my $readonly = $mountpoint->{ro};
my @extra_opts;
if ($format eq 'subvol') {
if ($mount_path) {
- if ($snapname) {
+ my (undef, $name) = PVE::Storage::parse_volname($storage_cfg, $volid);
+ if (defined($snapname)) {
+ $name .= "\@$snapname";
if ($scfg->{type} eq 'zfspool') {
- my $path_arg = $path;
- $path_arg =~ s!^/+!!;
- PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
+ PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', "$scfg->{pool}/$name", $mount_path]);
} else {
die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
}
} else {
if (defined($acl) && $scfg->{type} eq 'zfspool') {
my $acltype = ($acl ? 'acltype=posixacl' : 'acltype=noacl');
- my (undef, $name) = PVE::Storage::parse_volname($storage_cfg, $volid);
- $name .= "\@$snapname" if defined($snapname);
PVE::Tools::run_command(['zfs', 'set', $acltype, "$scfg->{pool}/$name"]);
}
bindmount($path, $parentfd, $last_dir//$rootdir, $mount_path, $readonly, @extra_opts);
die "unsupported storage";
}
+sub mountpoint_hotplug($$$) {
+ my ($vmid, $conf, $opt, $mp, $storage_cfg) = @_;
+
+ my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
+
+ # We do the rest in a fork with an unshared mount namespace, because:
+ # -) change our papparmor profile to that of /usr/bin/lxc-start
+ # -) we're now going to 'stage' # the mountpoint, then grab it, then move into the
+ # container's namespace, then mount it.
+
+ PVE::Tools::run_fork(sub {
+ # Pin the container pid longer, we also need to get its monitor/parent:
+ my ($ct_pid, $ct_pidfd) = open_lxc_pid($vmid)
+ or die "failed to open pidfd of container $vmid\'s init process\n";
+
+ my ($monitor_pid, $monitor_pidfd) = open_ppid($ct_pid)
+ or die "failed to open pidfd of container $vmid\'s monitor process\n";
+
+ my $ct_mnt_ns = $get_container_namespace->($vmid, $ct_pid, 'mnt');
+ my $monitor_mnt_ns = $get_container_namespace->($vmid, $monitor_pid, 'mnt');
+
+ # Grab a file descriptor to our apparmor label file so we can change into the 'lxc-start'
+ # profile to lower our privileges to the same level we have in the start hook:
+ sysopen(my $aa_fd, "/proc/self/attr/current", O_WRONLY)
+ or die "failed to open '/proc/self/attr/current' for writing: $!\n";
+ # But switch namespaces first, to make sure the namespace switches aren't blocked by
+ # apparmor.
+
+ # Change into the monitor's mount namespace. We "pin" the mount into the monitor's
+ # namespace for it to remain active there since the container will be able to unmount
+ # hotplugged mount points and thereby potentially free up loop devices, which is a security
+ # concern.
+ PVE::Tools::setns(fileno($monitor_mnt_ns), PVE::Tools::CLONE_NEWNS);
+ chdir('/')
+ or die "failed to change root directory within the monitor's mount namespace: $!\n";
+
+ my $dir = get_staging_mount_path($opt);
+
+ # Now switch our apparmor profile before mounting:
+ my $data = 'changeprofile /usr/bin/lxc-start';
+ if (syswrite($aa_fd, $data, length($data)) != length($data)) {
+ die "failed to change apparmor profile: $!\n";
+ }
+ # Check errors on close as well:
+ close($aa_fd)
+ or die "failed to change apparmor profile (close() failed): $!\n";
+
+ my $mount_fd = mountpoint_stage($mp, $dir, $storage_cfg, undef, $rootuid, $rootgid);
+
+ PVE::Tools::setns(fileno($ct_mnt_ns), PVE::Tools::CLONE_NEWNS);
+ chdir('/')
+ or die "failed to change root directory within the container's mount namespace: $!\n";
+
+ mountpoint_insert_staged($mount_fd, undef, $mp->{mp}, $opt, $rootuid, $rootgid);
+ });
+}
+
+# Create a directory in the mountpoint staging tempfs.
+sub get_staging_mount_path($) {
+ my ($opt) = @_;
+
+ my $target = get_staging_tempfs() . "/$opt";
+ if (!mkdir($target) && $! != EEXIST) {
+ die "failed to create directory $target: $!\n";
+ }
+
+ return $target;
+}
+
+# Mount /run/pve/mountpoints as tmpfs
+sub get_staging_tempfs() {
+ # We choose a path in /var/lib/lxc/ here because the lxc-start apparmor profile restricts most
+ # mounts to that.
+ my $target = '/var/lib/lxc/.pve-staged-mounts';
+ if (!mkdir($target)) {
+ return $target if $! == EEXIST;
+ die "failed to create directory $target: $!\n";
+ }
+
+ PVE::Tools::mount("none", $target, 'tmpfs', 0, "size=8k,mode=755")
+ or die "failed to mount $target as tmpfs: $!\n";
+
+ return $target;
+}
+
sub mkfs {
my ($dev, $rootuid, $rootgid) = @_;
our $NEW_DISK_RE = qr/^([^:\s]+):(\d+(\.\d+)?)$/;
sub create_disks {
- my ($storecfg, $vmid, $settings, $conf) = @_;
+ my ($storecfg, $vmid, $settings, $conf, $pending) = @_;
my $vollist = [];
push @$vollist, $volid;
$mountpoint->{volume} = $volid;
$mountpoint->{size} = $size_kb * 1024;
- $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
+ if ($pending) {
+ $conf->{pending}->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
+ } else {
+ $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
+ }
} else {
- # use specified/existing volid/dir/device
- $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
+ # use specified/existing volid/dir/device
+ $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
}
});
# free allocated images on error
if (my $err = $@) {
destroy_disks($storecfg, $vollist);
- die $err;
+ die $err;
}
return $vollist;
}
sub vm_start {
my ($vmid, $conf, $skiplock) = @_;
+ # apply pending changes while starting
+ if (scalar(keys %{$conf->{pending}})) {
+ my $storecfg = PVE::Storage::config();
+ PVE::LXC::Config->vmconfig_apply_pending($vmid, $conf, $storecfg);
+ $conf = PVE::LXC::Config->load_config($vmid); # update/reload
+ }
+
update_lxc_config($vmid, $conf);
my $skiplock_flag_fn = "/run/lxc/skiplock-$vmid";
# This is necessary because we want the post-stop hook to have completed its
# unmount-all step, but post-stop happens after lxc puts the container into the
# STOPPED state.
+# $kill - if true it will always do an immediate hard-stop
+# $shutdown_timeout - the timeout to wait for a gracefull shutdown
+# $kill_after_timeout - if true, send a hardstop if shutdown timed out
sub vm_stop {
- my ($vmid, $kill, $shutdown_timeout, $exit_timeout) = @_;
+ my ($vmid, $kill, $shutdown_timeout, $kill_after_timeout) = @_;
# Open the container's command socket.
my $path = "\0/var/lib/lxc/$vmid/command";
if ($kill) {
push @$cmd, '--kill'; # doesn't allow timeouts
- } elsif (defined($shutdown_timeout)) {
- push @$cmd, '--timeout', $shutdown_timeout;
- # Give run_command 5 extra seconds
- $shutdown_timeout += 5;
+ } else {
+ # lxc-stop uses a default timeout
+ push @$cmd, '--nokill' if !$kill_after_timeout;
+
+ if (defined($shutdown_timeout)) {
+ push @$cmd, '--timeout', $shutdown_timeout;
+ # Give run_command 5 extra seconds
+ $shutdown_timeout += 5;
+ }
}
eval { PVE::Tools::run_command($cmd, timeout => $shutdown_timeout) };
warn $@ if $@;
}
- my $result = 1;
- my $wait = sub { $result = <$sock>; };
- if (defined($exit_timeout)) {
- PVE::Tools::run_with_timeout($exit_timeout, $wait);
- } else {
- $wait->();
- }
+ my $result = <$sock>;
return if !defined $result; # monitor is gone and the ct has stopped.
die "container did not stop\n";
}
+sub vm_reboot {
+ my ($vmid, $timeout, $skiplock) = @_;
+
+ PVE::LXC::Config->lock_config($vmid, sub {
+ return if !check_running($vmid);
+
+ vm_stop($vmid, 0, $timeout, 1); # kill if timeout exceeds
+
+ my $conf = PVE::LXC::Config->load_config($vmid);
+ vm_start($vmid, $conf);
+ });
+}
+
sub run_unshared {
my ($code) = @_;
}
my $copy_volume = sub {
- my ($src_volid, $src, $dst_volid, $dest, $storage_cfg, $snapname) = @_;
+ my ($src_volid, $src, $dst_volid, $dest, $storage_cfg, $snapname, $bwlimit, $rootuid, $rootgid) = @_;
my $src_mp = { volume => $src_volid, mp => '/', ro => 1 };
$src_mp->{type} = PVE::LXC::Config->classify_mountpoint($src_volid);
eval {
# mount and copy
mkdir $src;
- mountpoint_mount($src_mp, $src, $storage_cfg, $snapname);
+ mountpoint_mount($src_mp, $src, $storage_cfg, $snapname, $rootuid, $rootgid);
push @mounted, $src;
mkdir $dest;
- mountpoint_mount($dst_mp, $dest, $storage_cfg);
+ mountpoint_mount($dst_mp, $dest, $storage_cfg, undef, $rootuid, $rootgid);
push @mounted, $dest;
+ $bwlimit //= 0;
+
PVE::Tools::run_command(['/usr/bin/rsync', '--stats', '-X', '-A', '--numeric-ids',
'-aH', '--whole-file', '--sparse', '--one-file-system',
- "$src/", $dest]);
+ "--bwlimit=$bwlimit", "$src/", $dest]);
};
my $err = $@;
+
+ # Wait for rsync's children to release dest so that
+ # consequent file operations (umount, remove) are possible
+ while ((system {"fuser"} "fuser", "-s", $dest) == 0) {sleep 1};
+
foreach my $mount (reverse @mounted) {
- eval { PVE::Tools::run_command(['/bin/umount', '--lazy', $mount], errfunc => sub{})};
+ eval { PVE::Tools::run_command(['/bin/umount', $mount], errfunc => sub{})};
warn "Can't umount $mount\n" if $@;
}
# Should not be called after unsharing the mount namespace!
sub copy_volume {
- my ($mp, $vmid, $storage, $storage_cfg, $conf, $snapname) = @_;
+ my ($mp, $vmid, $storage, $storage_cfg, $conf, $snapname, $bwlimit) = @_;
die "cannot copy volumes of type $mp->{type}\n" if $mp->{type} ne 'volume';
File::Path::make_path("/var/lib/lxc/$vmid");
}
run_unshared(sub {
- $copy_volume->($mp->{volume}, $src, $new_volid, $dest, $storage_cfg, $snapname);
+ $copy_volume->($mp->{volume}, $src, $new_volid, $dest, $storage_cfg, $snapname, $bwlimit, $rootuid, $rootgid);
});
};
if (my $err = $@) {
projects / pve-container.git / blobdiff