src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX qw(EINTR);
   7
   8 use Socket;
   9
  10 use File::Path;
  11 use File::Spec;
  12 use Cwd qw();
  13 use Fcntl qw(O_RDONLY O_NOFOLLOW O_DIRECTORY);
  14 use Errno qw(ELOOP ENOTDIR EROFS ECONNREFUSED);
  15 use IO::Socket::UNIX;
  16
  17 use PVE::Exception qw(raise_perm_exc);
  18 use PVE::Storage;
  19 use PVE::SafeSyslog;
  20 use PVE::INotify;
  21 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full O_PATH);
  22 use PVE::CpuSet;
  23 use PVE::Network;
  24 use PVE::AccessControl;
  25 use PVE::ProcFSTools;
  26 use PVE::Syscall;
  27 use PVE::LXC::Config;
  28 use Time::HiRes qw (gettimeofday);
  29
  30 my $nodename = PVE::INotify::nodename();
  31
  32 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  33
  34 sub config_list {
  35     my $vmlist = PVE::Cluster::get_vmlist();
  36     my $res = {};
  37     return $res if !$vmlist || !$vmlist->{ids};
  38     my $ids = $vmlist->{ids};
  39
  40     foreach my $vmid (keys %$ids) {
  41         next if !$vmid; # skip CT0
  42         my $d = $ids->{$vmid};
  43         next if !$d->{node} || $d->{node} ne $nodename;
  44         next if !$d->{type} || $d->{type} ne 'lxc';
  45         $res->{$vmid}->{type} = 'lxc';
  46     }
  47     return $res;
  48 }
  49
  50 sub destroy_config {
  51     my ($vmid) = @_;
  52
  53     unlink PVE::LXC::Config->config_file($vmid, $nodename);
  54 }
  55
  56 # container status helpers
  57
  58 sub list_active_containers {
  59
  60     my $filename = "/proc/net/unix";
  61
  62     # similar test is used by lcxcontainers.c: list_active_containers
  63     my $res = {};
  64
  65     my $fh = IO::File->new ($filename, "r");
  66     return $res if !$fh;
  67
  68     while (defined(my $line = <$fh>)) {
  69         if ($line =~ m/^[a-f0-9]+:\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\d+\s+(\S+)$/) {
  70             my $path = $1;
  71             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
  72                 $res->{$1} = 1;
  73             }
  74         }
  75     }
  76
  77     close($fh);
  78
  79     return $res;
  80 }
  81
  82 # warning: this is slow
  83 sub check_running {
  84     my ($vmid) = @_;
  85
  86     my $active_hash = list_active_containers();
  87
  88     return 1 if defined($active_hash->{$vmid});
  89
  90     return undef;
  91 }
  92
  93 sub get_container_disk_usage {
  94     my ($vmid, $pid) = @_;
  95
  96     return PVE::Tools::df("/proc/$pid/root/", 1);
  97 }
  98
  99 my $last_proc_vmid_stat;
 100
 101 my $parse_cpuacct_stat = sub {
 102     my ($vmid, $unprivileged) = @_;
 103
 104     my $raw = read_cgroup_value('cpuacct', $vmid, $unprivileged, 'cpuacct.stat', 1);
 105
 106     my $stat = {};
 107
 108     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 109
 110         $stat->{utime} = $1;
 111         $stat->{stime} = $2;
 112
 113     }
 114
 115     return $stat;
 116 };
 117
 118 sub vmstatus {
 119     my ($opt_vmid) = @_;
 120
 121     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 122
 123     my $active_hash = list_active_containers();
 124
 125     my $cpucount = $cpuinfo->{cpus} || 1;
 126
 127     my $cdtime = gettimeofday;
 128
 129     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 130
 131     my $unprivileged = {};
 132
 133     foreach my $vmid (keys %$list) {
 134         my $d = $list->{$vmid};
 135
 136         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 137         warn $@ if $@; # ignore errors (consider them stopped)
 138
 139         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 140
 141         my $cfspath = PVE::LXC::Config->cfs_config_path($vmid);
 142         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 143
 144         $unprivileged->{$vmid} = $conf->{unprivileged};
 145
 146         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 147         $d->{name} =~ s/[\s]//g;
 148
 149         $d->{cpus} = $conf->{cores} || $conf->{cpulimit};
 150         $d->{cpus} = $cpucount if !$d->{cpus};
 151
 152         $d->{lock} = $conf->{lock} || '';
 153
 154         if ($d->{pid}) {
 155             my $res = get_container_disk_usage($vmid, $d->{pid});
 156             $d->{disk} = $res->{used};
 157             $d->{maxdisk} = $res->{total};
 158         } else {
 159             $d->{disk} = 0;
 160             # use 4GB by default ??
 161             if (my $rootfs = $conf->{rootfs}) {
 162                 my $rootinfo = PVE::LXC::Config->parse_ct_rootfs($rootfs);
 163                 $d->{maxdisk} = $rootinfo->{size} || (4*1024*1024*1024);
 164             } else {
 165                 $d->{maxdisk} = 4*1024*1024*1024;
 166             }
 167         }
 168
 169         $d->{mem} = 0;
 170         $d->{swap} = 0;
 171         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 172         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 173
 174         $d->{uptime} = 0;
 175         $d->{cpu} = 0;
 176
 177         $d->{netout} = 0;
 178         $d->{netin} = 0;
 179
 180         $d->{diskread} = 0;
 181         $d->{diskwrite} = 0;
 182
 183         $d->{template} = PVE::LXC::Config->is_template($conf);
 184     }
 185
 186     foreach my $vmid (keys %$list) {
 187         my $d = $list->{$vmid};
 188         my $pid = $d->{pid};
 189
 190         next if !$pid; # skip stopped CTs
 191
 192         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 193         $d->{uptime} = time - $ctime; # the method lxcfs uses
 194
 195         my $unpriv = $unprivileged->{$vmid};
 196
 197         my $memory_stat = read_cgroup_list('memory', $vmid, $unpriv, 'memory.stat');
 198         my $mem_usage_in_bytes = read_cgroup_value('memory', $vmid, $unpriv, 'memory.usage_in_bytes');
 199
 200         $d->{mem} = $mem_usage_in_bytes - $memory_stat->{total_cache};
 201         $d->{swap} = read_cgroup_value('memory', $vmid, $unpriv, 'memory.memsw.usage_in_bytes') - $mem_usage_in_bytes;
 202
 203         my $blkio_bytes = read_cgroup_value('blkio', $vmid, $unpriv, 'blkio.throttle.io_service_bytes', 1);
 204         my @bytes = split(/\n/, $blkio_bytes);
 205         foreach my $byte (@bytes) {
 206             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 207                 $d->{diskread} += $2 if $key eq 'Read';
 208                 $d->{diskwrite} += $2 if $key eq 'Write';
 209             }
 210         }
 211
 212         my $pstat = $parse_cpuacct_stat->($vmid, $unpriv);
 213
 214         my $used = $pstat->{utime} + $pstat->{stime};
 215
 216         my $old = $last_proc_vmid_stat->{$vmid};
 217         if (!$old) {
 218             $last_proc_vmid_stat->{$vmid} = {
 219                 time => $cdtime,
 220                 used => $used,
 221                 cpu => 0,
 222             };
 223             next;
 224         }
 225
 226         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 227
 228         if ($dtime > 1000) {
 229             my $dutime = $used -  $old->{used};
 230
 231             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 232             $last_proc_vmid_stat->{$vmid} = {
 233                 time => $cdtime,
 234                 used => $used,
 235                 cpu => $d->{cpu},
 236             };
 237         } else {
 238             $d->{cpu} = $old->{cpu};
 239         }
 240     }
 241
 242     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 243
 244     foreach my $dev (keys %$netdev) {
 245         next if $dev !~ m/^veth([1-9]\d*)i/;
 246         my $vmid = $1;
 247         my $d = $list->{$vmid};
 248
 249         next if !$d;
 250
 251         $d->{netout} += $netdev->{$dev}->{receive};
 252         $d->{netin} += $netdev->{$dev}->{transmit};
 253
 254     }
 255
 256     return $list;
 257 }
 258
 259 sub read_cgroup_list($$$$) {
 260     my ($group, $vmid, $unprivileged, $name) = @_;
 261
 262     my $content = read_cgroup_value($group, $vmid, $unprivileged, $name, 1);
 263
 264     return { split(/\s+/, $content) };
 265 }
 266
 267 sub read_cgroup_value($$$$$) {
 268     my ($group, $vmid, $unprivileged, $name, $full) = @_;
 269
 270     my $nsdir = $unprivileged ? '' : 'ns/';
 271     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/${nsdir}$name";
 272
 273     return PVE::Tools::file_get_contents($path) if $full;
 274
 275     return PVE::Tools::file_read_firstline($path);
 276 }
 277
 278 sub write_cgroup_value {
 279    my ($group, $vmid, $name, $value) = @_;
 280
 281    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 282    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 283
 284 }
 285
 286 sub find_lxc_console_pids {
 287
 288     my $res = {};
 289
 290     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 291         my ($pid) = @_;
 292
 293         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 294         return if !$cmdline;
 295
 296         my @args = split(/\0/, $cmdline);
 297
 298         # search for lxc-console -n <vmid>
 299         return if scalar(@args) != 3;
 300         return if $args[1] ne '-n';
 301         return if $args[2] !~ m/^\d+$/;
 302         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 303
 304         my $vmid = $args[2];
 305
 306         push @{$res->{$vmid}}, $pid;
 307     });
 308
 309     return $res;
 310 }
 311
 312 sub find_lxc_pid {
 313     my ($vmid) = @_;
 314
 315     my $pid = undef;
 316     my $parser = sub {
 317         my $line = shift;
 318         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 319     };
 320     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
 321
 322     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 323
 324     return $pid;
 325 }
 326
 327 # Note: we cannot use Net:IP, because that only allows strict
 328 # CIDR networks
 329 sub parse_ipv4_cidr {
 330     my ($cidr, $noerr) = @_;
 331
 332     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
 333         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
 334     }
 335
 336     return undef if $noerr;
 337
 338     die "unable to parse ipv4 address/mask\n";
 339 }
 340
 341
 342 sub update_lxc_config {
 343     my ($vmid, $conf) = @_;
 344
 345     my $dir = "/var/lib/lxc/$vmid";
 346
 347     if ($conf->{template}) {
 348
 349         unlink "$dir/config";
 350
 351         return;
 352     }
 353
 354     my $raw = '';
 355
 356     die "missing 'arch' - internal error" if !$conf->{arch};
 357     $raw .= "lxc.arch = $conf->{arch}\n";
 358
 359     my $unprivileged = $conf->{unprivileged};
 360     my $custom_idmap = grep { $_->[0] eq 'lxc.idmap' } @{$conf->{lxc}};
 361
 362     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
 363
 364     my $inc ="/usr/share/lxc/config/$ostype.common.conf";
 365     $inc ="/usr/share/lxc/config/common.conf" if !-f $inc;
 366     $raw .= "lxc.include = $inc\n";
 367     if ($unprivileged || $custom_idmap) {
 368         $inc = "/usr/share/lxc/config/$ostype.userns.conf";
 369         $inc = "/usr/share/lxc/config/userns.conf" if !-f $inc;
 370         $raw .= "lxc.include = $inc\n"
 371     }
 372
 373     # WARNING: DO NOT REMOVE this without making sure that loop device nodes
 374     # cannot be exposed to the container with r/w access (cgroup perms).
 375     # When this is enabled mounts will still remain in the monitor's namespace
 376     # after the container unmounted them and thus will not detach from their
 377     # files while the container is running!
 378     $raw .= "lxc.monitor.unshare = 1\n";
 379
 380     # Should we read them from /etc/subuid?
 381     if ($unprivileged && !$custom_idmap) {
 382         $raw .= "lxc.idmap = u 0 100000 65536\n";
 383         $raw .= "lxc.idmap = g 0 100000 65536\n";
 384     }
 385
 386     if (!PVE::LXC::Config->has_dev_console($conf)) {
 387         $raw .= "lxc.console.path = none\n";
 388         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
 389     }
 390
 391     my $ttycount = PVE::LXC::Config->get_tty_count($conf);
 392     $raw .= "lxc.tty.max = $ttycount\n";
 393
 394     # some init scripts expect a linux terminal (turnkey).
 395     $raw .= "lxc.environment = TERM=linux\n";
 396
 397     my $utsname = $conf->{hostname} || "CT$vmid";
 398     $raw .= "lxc.uts.name = $utsname\n";
 399
 400     my $memory = $conf->{memory} || 512;
 401     my $swap = $conf->{swap} // 0;
 402
 403     my $lxcmem = int($memory*1024*1024);
 404     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
 405
 406     my $lxcswap = int(($memory + $swap)*1024*1024);
 407     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
 408
 409     if (my $cpulimit = $conf->{cpulimit}) {
 410         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
 411         my $value = int(100000*$cpulimit);
 412         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
 413     }
 414
 415     my $shares = $conf->{cpuunits} || 1024;
 416     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
 417
 418     die "missing 'rootfs' configuration\n"
 419         if !defined($conf->{rootfs});
 420
 421     my $mountpoint = PVE::LXC::Config->parse_ct_rootfs($conf->{rootfs});
 422
 423     $raw .= "lxc.rootfs.path = $dir/rootfs\n";
 424
 425     foreach my $k (sort keys %$conf) {
 426         next if $k !~ m/^net(\d+)$/;
 427         my $ind = $1;
 428         my $d = PVE::LXC::Config->parse_lxc_network($conf->{$k});
 429         $raw .= "lxc.net.$ind.type = veth\n";
 430         $raw .= "lxc.net.$ind.veth.pair = veth${vmid}i${ind}\n";
 431         $raw .= "lxc.net.$ind.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
 432         $raw .= "lxc.net.$ind.name = $d->{name}\n" if defined($d->{name});
 433         $raw .= "lxc.net.$ind.mtu = $d->{mtu}\n" if defined($d->{mtu});
 434     }
 435
 436     my $had_cpuset = 0;
 437     if (my $lxcconf = $conf->{lxc}) {
 438         foreach my $entry (@$lxcconf) {
 439             my ($k, $v) = @$entry;
 440             $had_cpuset = 1 if $k eq 'lxc.cgroup.cpuset.cpus';
 441             $raw .= "$k = $v\n";
 442         }
 443     }
 444
 445     my $cores = $conf->{cores};
 446     if (!$had_cpuset && $cores) {
 447         my $cpuset = eval { PVE::CpuSet->new_from_cgroup('lxc', 'effective_cpus') };
 448         $cpuset = PVE::CpuSet->new_from_cgroup('', 'effective_cpus') if !$cpuset;
 449         my @members = $cpuset->members();
 450         while (scalar(@members) > $cores) {
 451             my $randidx = int(rand(scalar(@members)));
 452             $cpuset->delete($members[$randidx]);
 453             splice(@members, $randidx, 1); # keep track of the changes
 454         }
 455         $raw .= "lxc.cgroup.cpuset.cpus = ".$cpuset->short_string()."\n";
 456     }
 457
 458     File::Path::mkpath("$dir/rootfs");
 459
 460     PVE::Tools::file_set_contents("$dir/config", $raw);
 461 }
 462
 463 # verify and cleanup nameserver list (replace \0 with ' ')
 464 sub verify_nameserver_list {
 465     my ($nameserver_list) = @_;
 466
 467     my @list = ();
 468     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
 469         PVE::JSONSchema::pve_verify_ip($server);
 470         push @list, $server;
 471     }
 472
 473     return join(' ', @list);
 474 }
 475
 476 sub verify_searchdomain_list {
 477     my ($searchdomain_list) = @_;
 478
 479     my @list = ();
 480     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
 481         # todo: should we add checks for valid dns domains?
 482         push @list, $server;
 483     }
 484
 485     return join(' ', @list);
 486 }
 487
 488 sub get_console_command {
 489     my ($vmid, $conf) = @_;
 490
 491     my $cmode = PVE::LXC::Config->get_cmode($conf);
 492
 493     if ($cmode eq 'console') {
 494         return ['lxc-console', '-n',  $vmid, '-t', 0];
 495     } elsif ($cmode eq 'tty') {
 496         return ['lxc-console', '-n',  $vmid];
 497     } elsif ($cmode eq 'shell') {
 498         return ['lxc-attach', '--clear-env', '-n', $vmid];
 499     } else {
 500         die "internal error";
 501     }
 502 }
 503
 504 sub get_primary_ips {
 505     my ($conf) = @_;
 506
 507     # return data from net0
 508
 509     return undef if !defined($conf->{net0});
 510     my $net = PVE::LXC::Config->parse_lxc_network($conf->{net0});
 511
 512     my $ipv4 = $net->{ip};
 513     if ($ipv4) {
 514         if ($ipv4 =~ /^(dhcp|manual)$/) {
 515             $ipv4 = undef
 516         } else {
 517             $ipv4 =~ s!/\d+$!!;
 518         }
 519     }
 520     my $ipv6 = $net->{ip6};
 521     if ($ipv6) {
 522         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
 523             $ipv6 = undef;
 524         } else {
 525             $ipv6 =~ s!/\d+$!!;
 526         }
 527     }
 528
 529     return ($ipv4, $ipv6);
 530 }
 531
 532 sub delete_mountpoint_volume {
 533     my ($storage_cfg, $vmid, $volume) = @_;
 534
 535     return if PVE::LXC::Config->classify_mountpoint($volume) ne 'volume';
 536
 537     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
 538     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
 539 }
 540
 541 sub destroy_lxc_container {
 542     my ($storage_cfg, $vmid, $conf, $replacement_conf) = @_;
 543
 544     PVE::LXC::Config->foreach_mountpoint($conf, sub {
 545         my ($ms, $mountpoint) = @_;
 546         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
 547     });
 548
 549     rmdir "/var/lib/lxc/$vmid/rootfs";
 550     unlink "/var/lib/lxc/$vmid/config";
 551     rmdir "/var/lib/lxc/$vmid";
 552     if (defined $replacement_conf) {
 553         PVE::LXC::Config->write_config($vmid, $replacement_conf);
 554     } else {
 555         destroy_config($vmid);
 556     }
 557
 558     #my $cmd = ['lxc-destroy', '-n', $vmid ];
 559     #PVE::Tools::run_command($cmd);
 560 }
 561
 562 sub vm_stop_cleanup {
 563     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
 564
 565     eval {
 566         if (!$keepActive) {
 567
 568             my $vollist = PVE::LXC::Config->get_vm_volumes($conf);
 569             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
 570         }
 571     };
 572     warn $@ if $@; # avoid errors - just warn
 573 }
 574
 575 my $safe_num_ne = sub {
 576     my ($a, $b) = @_;
 577
 578     return 0 if !defined($a) && !defined($b);
 579     return 1 if !defined($a);
 580     return 1 if !defined($b);
 581
 582     return $a != $b;
 583 };
 584
 585 my $safe_string_ne = sub {
 586     my ($a, $b) = @_;
 587
 588     return 0 if !defined($a) && !defined($b);
 589     return 1 if !defined($a);
 590     return 1 if !defined($b);
 591
 592     return $a ne $b;
 593 };
 594
 595 sub update_net {
 596     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
 597
 598     if ($newnet->{type} ne 'veth') {
 599         # for when there are physical interfaces
 600         die "cannot update interface of type $newnet->{type}";
 601     }
 602
 603     my $veth = "veth${vmid}i${netid}";
 604     my $eth = $newnet->{name};
 605
 606     if (my $oldnetcfg = $conf->{$opt}) {
 607         my $oldnet = PVE::LXC::Config->parse_lxc_network($oldnetcfg);
 608
 609         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
 610             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
 611
 612             PVE::Network::veth_delete($veth);
 613             delete $conf->{$opt};
 614             PVE::LXC::Config->write_config($vmid, $conf);
 615
 616             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
 617
 618         } else {
 619             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
 620                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
 621                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
 622
 623                 if ($oldnet->{bridge}) {
 624                     PVE::Network::tap_unplug($veth);
 625                     foreach (qw(bridge tag firewall)) {
 626                         delete $oldnet->{$_};
 627                     }
 628                     $conf->{$opt} = PVE::LXC::Config->print_lxc_network($oldnet);
 629                     PVE::LXC::Config->write_config($vmid, $conf);
 630                 }
 631
 632                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
 633                 # This includes the rate:
 634                 foreach (qw(bridge tag firewall rate)) {
 635                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
 636                 }
 637             } elsif (&$safe_string_ne($oldnet->{rate}, $newnet->{rate})) {
 638                 # Rate can be applied on its own but any change above needs to
 639                 # include the rate in tap_plug since OVS resets everything.
 640                 PVE::Network::tap_rate_limit($veth, $newnet->{rate});
 641                 $oldnet->{rate} = $newnet->{rate}
 642             }
 643             $conf->{$opt} = PVE::LXC::Config->print_lxc_network($oldnet);
 644             PVE::LXC::Config->write_config($vmid, $conf);
 645         }
 646     } else {
 647         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
 648     }
 649
 650     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
 651 }
 652
 653 sub hotplug_net {
 654     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
 655
 656     my $veth = "veth${vmid}i${netid}";
 657     my $vethpeer = $veth . "p";
 658     my $eth = $newnet->{name};
 659
 660     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
 661     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
 662
 663     # attach peer in container
 664     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
 665     PVE::Tools::run_command($cmd);
 666
 667     # link up peer in container
 668     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
 669     PVE::Tools::run_command($cmd);
 670
 671     my $done = { type => 'veth' };
 672     foreach (qw(bridge tag firewall hwaddr name)) {
 673         $done->{$_} = $newnet->{$_} if $newnet->{$_};
 674     }
 675     $conf->{$opt} = PVE::LXC::Config->print_lxc_network($done);
 676
 677     PVE::LXC::Config->write_config($vmid, $conf);
 678 }
 679
 680 sub update_ipconfig {
 681     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
 682
 683     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
 684
 685     my $optdata = PVE::LXC::Config->parse_lxc_network($conf->{$opt});
 686     my $deleted = [];
 687     my $added = [];
 688     my $nscmd = sub {
 689         my $cmdargs = shift;
 690         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
 691     };
 692     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
 693
 694     my $change_ip_config = sub {
 695         my ($ipversion) = @_;
 696
 697         my $family_opt = "-$ipversion";
 698         my $suffix = $ipversion == 4 ? '' : $ipversion;
 699         my $gw= "gw$suffix";
 700         my $ip= "ip$suffix";
 701
 702         my $newip = $newnet->{$ip};
 703         my $newgw = $newnet->{$gw};
 704         my $oldip = $optdata->{$ip};
 705
 706         my $change_ip = &$safe_string_ne($oldip, $newip);
 707         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
 708
 709         return if !$change_ip && !$change_gw;
 710
 711         # step 1: add new IP, if this fails we cancel
 712         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
 713         if ($change_ip && $is_real_ip) {
 714             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
 715             if (my $err = $@) {
 716                 warn $err;
 717                 return;
 718             }
 719         }
 720
 721         # step 2: replace gateway
 722         #   If this fails we delete the added IP and cancel.
 723         #   If it succeeds we save the config and delete the old IP, ignoring
 724         #   errors. The config is then saved.
 725         # Note: 'ip route replace' can add
 726         if ($change_gw) {
 727             if ($newgw) {
 728                 eval {
 729                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
 730                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
 731                     }
 732                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
 733                 };
 734                 if (my $err = $@) {
 735                     warn $err;
 736                     # the route was not replaced, the old IP is still available
 737                     # rollback (delete new IP) and cancel
 738                     if ($change_ip) {
 739                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
 740                         warn $@ if $@; # no need to die here
 741                     }
 742                     return;
 743                 }
 744             } else {
 745                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
 746                 # if the route was not deleted, the guest might have deleted it manually
 747                 # warn and continue
 748                 warn $@ if $@;
 749             }
 750         }
 751
 752         # from this point on we save the configuration
 753         # step 3: delete old IP ignoring errors
 754         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
 755             # We need to enable promote_secondaries, otherwise our newly added
 756             # address will be removed along with the old one.
 757             my $promote = 0;
 758             eval {
 759                 if ($ipversion == 4) {
 760                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
 761                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
 762                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
 763                 }
 764                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
 765             };
 766             warn $@ if $@; # no need to die here
 767
 768             if ($ipversion == 4) {
 769                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
 770             }
 771         }
 772
 773         foreach my $property ($ip, $gw) {
 774             if ($newnet->{$property}) {
 775                 $optdata->{$property} = $newnet->{$property};
 776             } else {
 777                 delete $optdata->{$property};
 778             }
 779         }
 780         $conf->{$opt} = PVE::LXC::Config->print_lxc_network($optdata);
 781         PVE::LXC::Config->write_config($vmid, $conf);
 782         $lxc_setup->setup_network($conf);
 783     };
 784
 785     &$change_ip_config(4);
 786     &$change_ip_config(6);
 787
 788 }
 789
 790 my $enter_namespace = sub {
 791     my ($vmid, $pid, $which, $type) = @_;
 792     sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY
 793         or die "failed to open $which namespace of container $vmid: $!\n";
 794     PVE::Tools::setns(fileno($fd), $type)
 795         or die "failed to enter $which namespace of container $vmid: $!\n";
 796     close $fd;
 797 };
 798
 799 my $do_syncfs = sub {
 800     my ($vmid, $pid, $socket) = @_;
 801
 802     &$enter_namespace($vmid, $pid, 'mnt', PVE::Tools::CLONE_NEWNS);
 803
 804     # Tell the parent process to start reading our /proc/mounts
 805     print {$socket} "go\n";
 806     $socket->flush();
 807
 808     # Receive /proc/self/mounts
 809     my $mountdata = do { local $/ = undef; <$socket> };
 810     close $socket;
 811
 812     # Now sync all mountpoints...
 813     my $mounts = PVE::ProcFSTools::parse_mounts($mountdata);
 814     foreach my $mp (@$mounts) {
 815         my ($what, $dir, $fs) = @$mp;
 816         next if $fs eq 'fuse.lxcfs';
 817         eval { PVE::Tools::sync_mountpoint($dir); };
 818         warn $@ if $@;
 819     }
 820 };
 821
 822 sub sync_container_namespace {
 823     my ($vmid) = @_;
 824     my $pid = find_lxc_pid($vmid);
 825
 826     # SOCK_DGRAM is nicer for barriers but cannot be slurped
 827     socketpair my $pfd, my $cfd, AF_UNIX, SOCK_STREAM, PF_UNSPEC
 828         or die "failed to create socketpair: $!\n";
 829
 830     my $child = fork();
 831     die "fork failed: $!\n" if !defined($child);
 832
 833     if (!$child) {
 834         eval {
 835             close $pfd;
 836             &$do_syncfs($vmid, $pid, $cfd);
 837         };
 838         if (my $err = $@) {
 839             warn $err;
 840             POSIX::_exit(1);
 841         }
 842         POSIX::_exit(0);
 843     }
 844     close $cfd;
 845     my $go = <$pfd>;
 846     die "failed to enter container namespace\n" if $go ne "go\n";
 847
 848     open my $mounts, '<', "/proc/$child/mounts"
 849         or die "failed to open container's /proc/mounts: $!\n";
 850     my $mountdata = do { local $/ = undef; <$mounts> };
 851     close $mounts;
 852     print {$pfd} $mountdata;
 853     close $pfd;
 854
 855     while (waitpid($child, 0) != $child) {}
 856     die "failed to sync container namespace\n" if $? != 0;
 857 }
 858
 859 sub template_create {
 860     my ($vmid, $conf) = @_;
 861
 862     my $storecfg = PVE::Storage::config();
 863
 864     my $rootinfo = PVE::LXC::Config->parse_ct_rootfs($conf->{rootfs});
 865     my $volid = $rootinfo->{volume};
 866
 867     die "Template feature is not available for '$volid'\n"
 868         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
 869
 870     PVE::Storage::activate_volumes($storecfg, [$volid]);
 871
 872     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
 873     $rootinfo->{volume} = $template_volid;
 874     $conf->{rootfs} = PVE::LXC::Config->print_ct_mountpoint($rootinfo, 1);
 875
 876     PVE::LXC::Config->write_config($vmid, $conf);
 877 }
 878
 879 sub check_ct_modify_config_perm {
 880     my ($rpcenv, $authuser, $vmid, $pool, $newconf, $delete) = @_;
 881
 882     return 1 if $authuser eq 'root@pam';
 883
 884     my $check = sub {
 885         my ($opt, $delete) = @_;
 886         if ($opt eq 'cores' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
 887             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
 888         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
 889             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
 890             return if $delete;
 891             my $data = $opt eq 'rootfs' ? PVE::LXC::Config->parse_ct_rootfs($newconf->{$opt})
 892                                         : PVE::LXC::Config->parse_ct_mountpoint($newconf->{$opt});
 893             raise_perm_exc("mount point type $data->{type} is only allowed for root\@pam")
 894                 if $data->{type} ne 'volume';
 895         } elsif ($opt eq 'memory' || $opt eq 'swap') {
 896             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
 897         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
 898                  $opt eq 'searchdomain' || $opt eq 'hostname') {
 899             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
 900         } else {
 901             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
 902         }
 903     };
 904
 905     foreach my $opt (keys %$newconf) {
 906         &$check($opt, 0);
 907     }
 908     foreach my $opt (@$delete) {
 909         &$check($opt, 1);
 910     }
 911
 912     return 1;
 913 }
 914
 915 sub umount_all {
 916     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
 917
 918     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
 919     my $volid_list = PVE::LXC::Config->get_vm_volumes($conf);
 920
 921     PVE::LXC::Config->foreach_mountpoint_reverse($conf, sub {
 922         my ($ms, $mountpoint) = @_;
 923
 924         my $volid = $mountpoint->{volume};
 925         my $mount = $mountpoint->{mp};
 926
 927         return if !$volid || !$mount;
 928
 929         my $mount_path = "$rootdir/$mount";
 930         $mount_path =~ s!/+!/!g;
 931
 932         return if !PVE::ProcFSTools::is_mounted($mount_path);
 933
 934         eval {
 935             PVE::Tools::run_command(['umount', '-d', $mount_path]);
 936         };
 937         if (my $err = $@) {
 938             if ($noerr) {
 939                 warn $err;
 940             } else {
 941                 die $err;
 942             }
 943         }
 944     });
 945 }
 946
 947 sub mount_all {
 948     my ($vmid, $storage_cfg, $conf, $ignore_ro) = @_;
 949
 950     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
 951     File::Path::make_path($rootdir);
 952
 953     my $volid_list = PVE::LXC::Config->get_vm_volumes($conf);
 954     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
 955
 956     eval {
 957         PVE::LXC::Config->foreach_mountpoint($conf, sub {
 958             my ($ms, $mountpoint) = @_;
 959
 960             $mountpoint->{ro} = 0 if $ignore_ro;
 961
 962             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
 963         });
 964     };
 965     if (my $err = $@) {
 966         warn "mounting container failed\n";
 967         umount_all($vmid, $storage_cfg, $conf, 1);
 968         die $err;
 969     }
 970
 971     return $rootdir;
 972 }
 973
 974
 975 sub mountpoint_mount_path {
 976     my ($mountpoint, $storage_cfg, $snapname) = @_;
 977
 978     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
 979 }
 980
 981 sub query_loopdev {
 982     my ($path) = @_;
 983     my $found;
 984     my $parser = sub {
 985         my $line = shift;
 986         if ($line =~ m@^(/dev/loop\d+):@) {
 987             $found = $1;
 988         }
 989     };
 990     my $cmd = ['losetup', '--associated', $path];
 991     PVE::Tools::run_command($cmd, outfunc => $parser);
 992     return $found;
 993 }
 994
 995 # Run a function with a file attached to a loop device.
 996 # The loop device is always detached afterwards (or set to autoclear).
 997 # Returns the loop device.
 998 sub run_with_loopdev {
 999     my ($func, $file) = @_;
1000     my $device = query_loopdev($file);
1001     # Try to reuse an existing device
1002     if ($device) {
1003         # We assume that whoever setup the loop device is responsible for
1004         # detaching it.
1005         &$func($device);
1006         return $device;
1007     }
1008
1009     my $parser = sub {
1010         my $line = shift;
1011         if ($line =~ m@^(/dev/loop\d+)$@) {
1012             $device = $1;
1013         }
1014     };
1015     PVE::Tools::run_command(['losetup', '--show', '-f', $file], outfunc => $parser);
1016     die "failed to setup loop device for $file\n" if !$device;
1017     eval { &$func($device); };
1018     my $err = $@;
1019     PVE::Tools::run_command(['losetup', '-d', $device]);
1020     die $err if $err;
1021     return $device;
1022 }
1023
1024 # In scalar mode: returns a file handle to the deepest directory node.
1025 # In list context: returns a list of:
1026 #   * the deepest directory node
1027 #   * the 2nd deepest directory (parent of the above)
1028 #   * directory name of the last directory
1029 # So that the path $2/$3 should lead to $1 afterwards.
1030 sub walk_tree_nofollow($$$) {
1031     my ($start, $subdir, $mkdir) = @_;
1032
1033     # splitdir() returns '' for empty components including the leading /
1034     my @comps = grep { length($_)>0 } File::Spec->splitdir($subdir);
1035
1036     sysopen(my $fd, $start, O_PATH | O_DIRECTORY)
1037         or die "failed to open start directory $start: $!\n";
1038
1039     my $dir = $start;
1040     my $last_component = undef;
1041     my $second = $fd;
1042     foreach my $component (@comps) {
1043         $dir .= "/$component";
1044         my $next = PVE::Tools::openat(fileno($fd), $component, O_NOFOLLOW | O_DIRECTORY);
1045
1046         if (!$next) {
1047             # failed, check for symlinks and try to create the path
1048             die "symlink encountered at: $dir\n" if $! == ELOOP || $! == ENOTDIR;
1049             die "cannot open directory $dir: $!\n" if !$mkdir;
1050
1051             # We don't check for errors on mkdirat() here and just try to
1052             # openat() again, since at least one error (EEXIST) is an
1053             # expected possibility if multiple containers start
1054             # simultaneously. If someone else injects a symlink now then
1055             # the subsequent openat() will fail due to O_NOFOLLOW anyway.
1056             PVE::Tools::mkdirat(fileno($fd), $component, 0755);
1057
1058             $next = PVE::Tools::openat(fileno($fd), $component, O_NOFOLLOW | O_DIRECTORY);
1059             die "failed to create path: $dir: $!\n" if !$next;
1060         }
1061
1062         close $second if defined($last_component);
1063         $last_component = $component;
1064         $second = $fd;
1065         $fd = $next;
1066     }
1067
1068     return ($fd, defined($last_component) && $second, $last_component) if wantarray;
1069     close $second if defined($last_component);
1070     return $fd;
1071 }
1072
1073 # To guard against symlink attack races against other currently running
1074 # containers with shared recursive bind mount hierarchies we prepare a
1075 # directory handle for the directory we're mounting over to verify the
1076 # mountpoint afterwards.
1077 sub __bindmount_prepare {
1078     my ($hostroot, $dir) = @_;
1079     my $srcdh = walk_tree_nofollow($hostroot, $dir, 0);
1080     return $srcdh;
1081 }
1082
1083 # Assuming we mount to rootfs/a/b/c, verify with the directory handle to 'b'
1084 # ($parentfd) that 'b/c' (openat($parentfd, 'c')) really leads to the directory
1085 # we intended to bind mount.
1086 sub __bindmount_verify {
1087     my ($srcdh, $parentfd, $last_dir, $ro) = @_;
1088     my $destdh;
1089     if ($parentfd) {
1090         # Open the mount point path coming from the parent directory since the
1091         # filehandle we would have gotten as first result of walk_tree_nofollow
1092         # earlier is still a handle to the underlying directory instead of the
1093         # mounted path.
1094         $destdh = PVE::Tools::openat(fileno($parentfd), $last_dir, PVE::Tools::O_PATH | O_NOFOLLOW | O_DIRECTORY);
1095         die "failed to open mount point: $!\n" if !$destdh;
1096         if ($ro) {
1097             my $dot = '.';
1098             # no separate function because 99% of the time it's the wrong thing to use.
1099             if (syscall(PVE::Syscall::faccessat, fileno($destdh), $dot, &POSIX::W_OK, 0) != -1) {
1100                 die "failed to mark bind mount read only\n";
1101             }
1102             die "read-only check failed: $!\n" if $! != EROFS;
1103         }
1104     } else {
1105         # For the rootfs we don't have a parentfd so we open the path directly.
1106         # Note that this means bindmounting any prefix of the host's
1107         # /var/lib/lxc/$vmid path into another container is considered a grave
1108         # security error.
1109         sysopen $destdh, $last_dir, O_PATH | O_DIRECTORY;
1110         die "failed to open mount point: $!\n" if !$destdh;
1111     }
1112
1113     my ($srcdev, $srcinode) = stat($srcdh);
1114     my ($dstdev, $dstinode) = stat($destdh);
1115     close $srcdh;
1116     close $destdh;
1117
1118     return ($srcdev == $dstdev && $srcinode == $dstinode);
1119 }
1120
1121 # Perform the actual bind mounting:
1122 sub __bindmount_do {
1123     my ($dir, $dest, $ro, @extra_opts) = @_;
1124     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $dir, $dest]);
1125     if ($ro) {
1126         eval { PVE::Tools::run_command(['mount', '-o', 'bind,remount,ro', $dest]); };
1127         if (my $err = $@) {
1128             warn "bindmount error\n";
1129             # don't leave writable bind-mounts behind...
1130             PVE::Tools::run_command(['umount', $dest]);
1131             die $err;
1132         }
1133     }
1134 }
1135
1136 sub bindmount {
1137     my ($dir, $parentfd, $last_dir, $dest, $ro, @extra_opts) = @_;
1138
1139     my $srcdh = __bindmount_prepare('/', $dir);
1140
1141     __bindmount_do($dir, $dest, $ro, @extra_opts);
1142
1143     if (!__bindmount_verify($srcdh, $parentfd, $last_dir, $ro)) {
1144         PVE::Tools::run_command(['umount', $dest]);
1145         die "detected mount path change at: $dir\n";
1146     }
1147 }
1148
1149 # Cleanup $rootdir a bit (double and trailing slashes), build the mount path
1150 # from $rootdir and $mount and walk the path from $rootdir to the final
1151 # directory to check for symlinks.
1152 sub __mount_prepare_rootdir {
1153     my ($rootdir, $mount) = @_;
1154     $rootdir =~ s!/+!/!g;
1155     $rootdir =~ s!/+$!!;
1156     my $mount_path = "$rootdir/$mount";
1157     my ($mpfd, $parentfd, $last_dir) = walk_tree_nofollow($rootdir, $mount, 1);
1158     return ($rootdir, $mount_path, $mpfd, $parentfd, $last_dir);
1159 }
1160
1161 # use $rootdir = undef to just return the corresponding mount path
1162 sub mountpoint_mount {
1163     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
1164
1165     my $volid = $mountpoint->{volume};
1166     my $mount = $mountpoint->{mp};
1167     my $type = $mountpoint->{type};
1168     my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota};
1169     my $mounted_dev;
1170
1171     return if !$volid || !$mount;
1172
1173     $mount =~ s!/+!/!g;
1174
1175     my $mount_path;
1176     my ($mpfd, $parentfd, $last_dir);
1177
1178     if (defined($rootdir)) {
1179         ($rootdir, $mount_path, $mpfd, $parentfd, $last_dir) =
1180             __mount_prepare_rootdir($rootdir, $mount);
1181     }
1182
1183     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1184
1185     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
1186
1187     my $optstring = '';
1188     my $acl = $mountpoint->{acl};
1189     if (defined($acl)) {
1190         $optstring .= ($acl ? 'acl' : 'noacl');
1191     }
1192     my $readonly = $mountpoint->{ro};
1193
1194     my @extra_opts = ('-o', $optstring) if $optstring;
1195
1196     if ($storage) {
1197
1198         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1199
1200         # early sanity checks:
1201         # we otherwise call realpath on the rbd url
1202         die "containers on rbd storage without krbd are not supported\n"
1203             if $scfg->{type} eq 'rbd' && !$scfg->{krbd};
1204
1205         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
1206
1207         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1208             PVE::Storage::parse_volname($storage_cfg, $volid);
1209
1210         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
1211
1212         if ($format eq 'subvol') {
1213             if ($mount_path) {
1214                 if ($snapname) {
1215                     if ($scfg->{type} eq 'zfspool') {
1216                         my $path_arg = $path;
1217                         $path_arg =~ s!^/+!!;
1218                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
1219                     } else {
1220                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
1221                     }
1222                 } else {
1223                     if (defined($acl) && $scfg->{type} eq 'zfspool') {
1224                         my $acltype = ($acl ? 'acltype=posixacl' : 'acltype=noacl');
1225                         my (undef, $name) = PVE::Storage::parse_volname($storage_cfg, $volid);
1226                         $name .= "\@$snapname" if defined($snapname);
1227                         PVE::Tools::run_command(['zfs', 'set', $acltype, "$scfg->{pool}/$name"]);
1228                     }
1229                     bindmount($path, $parentfd, $last_dir//$rootdir, $mount_path, $readonly, @extra_opts);
1230                     warn "cannot enable quota control for bind mounted subvolumes\n" if $quota;
1231                 }
1232             }
1233             return wantarray ? ($path, 0, undef) : $path;
1234         } elsif ($format eq 'raw' || $format eq 'iso') {
1235             # NOTE: 'mount' performs canonicalization without the '-c' switch, which for
1236             # device-mapper devices is special-cased to use the /dev/mapper symlinks.
1237             # Our autodev hook expects the /dev/dm-* device currently
1238             # and will create the /dev/mapper symlink accordingly
1239             $path = Cwd::realpath($path);
1240             die "failed to get device path\n" if !$path;
1241             ($path) = ($path =~ /^(.*)$/s); #untaint
1242             my $domount = sub {
1243                 my ($path) = @_;
1244                 if ($mount_path) {
1245                     if ($format eq 'iso') {
1246                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
1247                     } elsif ($isBase || defined($snapname)) {
1248                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
1249                     } else {
1250                         if ($quota) {
1251                             push @extra_opts, '-o', 'usrjquota=aquota.user,grpjquota=aquota.group,jqfmt=vfsv0';
1252                         }
1253                         push @extra_opts, '-o', 'ro' if $readonly;
1254                         PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
1255                     }
1256                 }
1257             };
1258             my $use_loopdev = 0;
1259             if ($scfg->{path}) {
1260                 $mounted_dev = run_with_loopdev($domount, $path);
1261                 $use_loopdev = 1;
1262             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
1263                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
1264                 $mounted_dev = $path;
1265                 &$domount($path);
1266             } else {
1267                 die "unsupported storage type '$scfg->{type}'\n";
1268             }
1269             return wantarray ? ($path, $use_loopdev, $mounted_dev) : $path;
1270         } else {
1271             die "unsupported image format '$format'\n";
1272         }
1273     } elsif ($type eq 'device') {
1274         push @extra_opts, '-o', 'ro' if $readonly;
1275         push @extra_opts, '-o', 'usrjquota=aquota.user,grpjquota=aquota.group,jqfmt=vfsv0' if $quota;
1276         # See the NOTE above about devicemapper canonicalization
1277         my ($devpath) = (Cwd::realpath($volid) =~ /^(.*)$/s); # realpath() taints
1278         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
1279         return wantarray ? ($volid, 0, $devpath) : $volid;
1280     } elsif ($type eq 'bind') {
1281         die "directory '$volid' does not exist\n" if ! -d $volid;
1282         bindmount($volid, $parentfd, $last_dir//$rootdir, $mount_path, $readonly, @extra_opts) if $mount_path;
1283         warn "cannot enable quota control for bind mounts\n" if $quota;
1284         return wantarray ? ($volid, 0, undef) : $volid;
1285     }
1286
1287     die "unsupported storage";
1288 }
1289
1290 sub mkfs {
1291     my ($dev, $rootuid, $rootgid) = @_;
1292
1293     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
1294                              '-E', "root_owner=$rootuid:$rootgid",
1295                              $dev]);
1296 }
1297
1298 sub format_disk {
1299     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
1300
1301     if ($volid =~ m!^/dev/.+!) {
1302         mkfs($volid);
1303         return;
1304     }
1305
1306     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1307
1308     die "cannot format volume '$volid' with no storage\n" if !$storage;
1309
1310     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
1311
1312     my $path = PVE::Storage::path($storage_cfg, $volid);
1313
1314     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1315         PVE::Storage::parse_volname($storage_cfg, $volid);
1316
1317     die "cannot format volume '$volid' (format == $format)\n"
1318         if $format ne 'raw';
1319
1320     mkfs($path, $rootuid, $rootgid);
1321 }
1322
1323 sub destroy_disks {
1324     my ($storecfg, $vollist) = @_;
1325
1326     foreach my $volid (@$vollist) {
1327         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
1328         warn $@ if $@;
1329     }
1330 }
1331
1332 our $NEW_DISK_RE = qr/^([^:\s]+):(\d+(\.\d+)?)$/;
1333 sub create_disks {
1334     my ($storecfg, $vmid, $settings, $conf) = @_;
1335
1336     my $vollist = [];
1337
1338     eval {
1339         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
1340         my $chown_vollist = [];
1341
1342         PVE::LXC::Config->foreach_mountpoint($settings, sub {
1343             my ($ms, $mountpoint) = @_;
1344
1345             my $volid = $mountpoint->{volume};
1346             my $mp = $mountpoint->{mp};
1347
1348             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1349
1350             if ($storage && ($volid =~ $NEW_DISK_RE)) {
1351                 my ($storeid, $size_gb) = ($1, $2);
1352
1353                 my $size_kb = int(${size_gb}*1024) * 1024;
1354
1355                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
1356                 # fixme: use better naming ct-$vmid-disk-X.raw?
1357
1358                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
1359                     if ($size_kb > 0) {
1360                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
1361                                                            undef, $size_kb);
1362                         format_disk($storecfg, $volid, $rootuid, $rootgid);
1363                     } else {
1364                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
1365                                                            undef, 0);
1366                         push @$chown_vollist, $volid;
1367                     }
1368                 } elsif ($scfg->{type} eq 'zfspool') {
1369
1370                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
1371                                                        undef, $size_kb);
1372                     push @$chown_vollist, $volid;
1373                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
1374
1375                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
1376                     format_disk($storecfg, $volid, $rootuid, $rootgid);
1377
1378                 } elsif ($scfg->{type} eq 'rbd') {
1379
1380                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
1381                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
1382                     format_disk($storecfg, $volid, $rootuid, $rootgid);
1383                 } else {
1384                     die "unable to create containers on storage type '$scfg->{type}'\n";
1385                 }
1386                 push @$vollist, $volid;
1387                 $mountpoint->{volume} = $volid;
1388                 $mountpoint->{size} = $size_kb * 1024;
1389                 $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
1390             } else {
1391                 # use specified/existing volid/dir/device
1392                 $conf->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
1393             }
1394         });
1395
1396         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
1397         foreach my $volid (@$chown_vollist) {
1398             my $path = PVE::Storage::path($storecfg, $volid, undef);
1399             chown($rootuid, $rootgid, $path);
1400         }
1401         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
1402     };
1403     # free allocated images on error
1404     if (my $err = $@) {
1405         destroy_disks($storecfg, $vollist);
1406         die $err;
1407     }
1408     return $vollist;
1409 }
1410
1411 # bash completion helper
1412
1413 sub complete_os_templates {
1414     my ($cmdname, $pname, $cvalue) = @_;
1415
1416     my $cfg = PVE::Storage::config();
1417
1418     my $storeid;
1419
1420     if ($cvalue =~ m/^([^:]+):/) {
1421         $storeid = $1;
1422     }
1423
1424     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
1425     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
1426
1427     my $res = [];
1428     foreach my $id (keys %$data) {
1429         foreach my $item (@{$data->{$id}}) {
1430             push @$res, $item->{volid} if defined($item->{volid});
1431         }
1432     }
1433
1434     return $res;
1435 }
1436
1437 my $complete_ctid_full = sub {
1438     my ($running) = @_;
1439
1440     my $idlist = vmstatus();
1441
1442     my $active_hash = list_active_containers();
1443
1444     my $res = [];
1445
1446     foreach my $id (keys %$idlist) {
1447         my $d = $idlist->{$id};
1448         if (defined($running)) {
1449             next if $d->{template};
1450             next if $running && !$active_hash->{$id};
1451             next if !$running && $active_hash->{$id};
1452         }
1453         push @$res, $id;
1454
1455     }
1456     return $res;
1457 };
1458
1459 sub complete_ctid {
1460     return &$complete_ctid_full();
1461 }
1462
1463 sub complete_ctid_stopped {
1464     return &$complete_ctid_full(0);
1465 }
1466
1467 sub complete_ctid_running {
1468     return &$complete_ctid_full(1);
1469 }
1470
1471 sub parse_id_maps {
1472     my ($conf) = @_;
1473
1474     my $id_map = [];
1475     my $rootuid = 0;
1476     my $rootgid = 0;
1477
1478     my $lxc = $conf->{lxc};
1479     foreach my $entry (@$lxc) {
1480         my ($key, $value) = @$entry;
1481         # FIXME: remove the 'id_map' variant when lxc-3.0 arrives
1482         next if $key ne 'lxc.idmap' && $key ne 'lxc.id_map';
1483         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
1484             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
1485             push @$id_map, [$type, $ct, $host, $length];
1486             if ($ct == 0) {
1487                 $rootuid = $host if $type eq 'u';
1488                 $rootgid = $host if $type eq 'g';
1489             }
1490         } else {
1491             die "failed to parse idmap: $value\n";
1492         }
1493     }
1494
1495     if (!@$id_map && $conf->{unprivileged}) {
1496         # Should we read them from /etc/subuid?
1497         $id_map = [ ['u', '0', '100000', '65536'],
1498                     ['g', '0', '100000', '65536'] ];
1499         $rootuid = $rootgid = 100000;
1500     }
1501
1502     return ($id_map, $rootuid, $rootgid);
1503 }
1504
1505 sub userns_command {
1506     my ($id_map) = @_;
1507     if (@$id_map) {
1508         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
1509     }
1510     return [];
1511 }
1512
1513 # Helper to stop a container completely and make sure it has stopped completely.
1514 # This is necessary because we want the post-stop hook to have completed its
1515 # unmount-all step, but post-stop happens after lxc puts the container into the
1516 # STOPPED state.
1517 sub vm_stop {
1518     my ($vmid, $kill, $shutdown_timeout, $exit_timeout) = @_;
1519
1520     # Open the container's command socket.
1521     my $path = "\0/var/lib/lxc/$vmid/command";
1522     my $sock = IO::Socket::UNIX->new(
1523         Type => SOCK_STREAM(),
1524         Peer => $path,
1525     );
1526     if (!$sock) {
1527         return if $! == ECONNREFUSED; # The container is not running
1528         die "failed to open container ${vmid}'s command socket: $!\n";
1529     }
1530
1531     # Stop the container:
1532
1533     my $cmd = ['lxc-stop', '-n', $vmid];
1534
1535     if ($kill) {
1536         push @$cmd, '--kill'; # doesn't allow timeouts
1537     } elsif (defined($shutdown_timeout)) {
1538         push @$cmd, '--timeout', $shutdown_timeout;
1539         # Give run_command 5 extra seconds
1540         $shutdown_timeout += 5;
1541     }
1542
1543     eval { PVE::Tools::run_command($cmd, timeout => $shutdown_timeout) };
1544     if (my $err = $@) {
1545         warn $@ if $@;
1546     }
1547
1548     my $result = 1;
1549     my $wait = sub { $result = <$sock>; };
1550     if (defined($exit_timeout)) {
1551         PVE::Tools::run_with_timeout($exit_timeout, $wait);
1552     } else {
1553         $wait->();
1554     }
1555
1556     return if !defined $result; # monitor is gone and the ct has stopped.
1557     die "container did not stop\n";
1558 }
1559
1560 1;