src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format_description => 'volume',
  42         description => 'Volume, device or directory to mount into the container.',
  43     },
  44     backup => {
  45         type => 'boolean',
  46         format_description => '[1|0]',
  47         description => 'Whether to include the mountpoint in backups.',
  48         optional => 1,
  49     },
  50     size => {
  51         type => 'string',
  52         format => 'disk-size',
  53         format_description => 'DiskSize',
  54         description => 'Volume size (read only value).',
  55         optional => 1,
  56     },
  57 };
  58
  59 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  60     type => 'string', format => $rootfs_desc,
  61     description => "Use volume as container root.",
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 my $confdesc = {
  72     lock => {
  73         optional => 1,
  74         type => 'string',
  75         description => "Lock/unlock the VM.",
  76         enum => [qw(migrate backup snapshot rollback)],
  77     },
  78     onboot => {
  79         optional => 1,
  80         type => 'boolean',
  81         description => "Specifies whether a VM will be started during system bootup.",
  82         default => 0,
  83     },
  84     startup => get_standard_option('pve-startup-order'),
  85     template => {
  86         optional => 1,
  87         type => 'boolean',
  88         description => "Enable/disable Template.",
  89         default => 0,
  90     },
  91     arch => {
  92         optional => 1,
  93         type => 'string',
  94         enum => ['amd64', 'i386'],
  95         description => "OS architecture type.",
  96         default => 'amd64',
  97     },
  98     ostype => {
  99         optional => 1,
 100         type => 'string',
 101         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux'],
 102         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 103     },
 104     console => {
 105         optional => 1,
 106         type => 'boolean',
 107         description => "Attach a console device (/dev/console) to the container.",
 108         default => 1,
 109     },
 110     tty => {
 111         optional => 1,
 112         type => 'integer',
 113         description => "Specify the number of tty available to the container",
 114         minimum => 0,
 115         maximum => 6,
 116         default => 2,
 117     },
 118     cpulimit => {
 119         optional => 1,
 120         type => 'number',
 121         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 122         minimum => 0,
 123         maximum => 128,
 124         default => 0,
 125     },
 126     cpuunits => {
 127         optional => 1,
 128         type => 'integer',
 129         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 130         minimum => 0,
 131         maximum => 500000,
 132         default => 1024,
 133     },
 134     memory => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Amount of RAM for the VM in MB.",
 138         minimum => 16,
 139         default => 512,
 140     },
 141     swap => {
 142         optional => 1,
 143         type => 'integer',
 144         description => "Amount of SWAP for the VM in MB.",
 145         minimum => 0,
 146         default => 512,
 147     },
 148     hostname => {
 149         optional => 1,
 150         description => "Set a host name for the container.",
 151         type => 'string', format => 'dns-name',
 152         maxLength => 255,
 153     },
 154     description => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Container description. Only used on the configuration web interface.",
 158     },
 159     searchdomain => {
 160         optional => 1,
 161         type => 'string', format => 'dns-name-list',
 162         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 163     },
 164     nameserver => {
 165         optional => 1,
 166         type => 'string', format => 'address-list',
 167         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 168     },
 169     rootfs => get_standard_option('pve-ct-rootfs'),
 170     parent => {
 171         optional => 1,
 172         type => 'string', format => 'pve-configid',
 173         maxLength => 40,
 174         description => "Parent snapshot name. This is used internally, and should not be modified.",
 175     },
 176     snaptime => {
 177         optional => 1,
 178         description => "Timestamp for snapshots.",
 179         type => 'integer',
 180         minimum => 0,
 181     },
 182     cmode => {
 183         optional => 1,
 184         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 185         type => 'string',
 186         enum => ['shell', 'console', 'tty'],
 187         default => 'tty',
 188     },
 189     protection => {
 190         optional => 1,
 191         type => 'boolean',
 192         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 193         default => 0,
 194     },
 195     unprivileged => {
 196         optional => 1,
 197         type => 'boolean',
 198         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 199         default => 0,
 200     },
 201 };
 202
 203 my $valid_lxc_conf_keys = {
 204     'lxc.include' => 1,
 205     'lxc.arch' => 1,
 206     'lxc.utsname' => 1,
 207     'lxc.haltsignal' => 1,
 208     'lxc.rebootsignal' => 1,
 209     'lxc.stopsignal' => 1,
 210     'lxc.init_cmd' => 1,
 211     'lxc.network.type' => 1,
 212     'lxc.network.flags' => 1,
 213     'lxc.network.link' => 1,
 214     'lxc.network.mtu' => 1,
 215     'lxc.network.name' => 1,
 216     'lxc.network.hwaddr' => 1,
 217     'lxc.network.ipv4' => 1,
 218     'lxc.network.ipv4.gateway' => 1,
 219     'lxc.network.ipv6' => 1,
 220     'lxc.network.ipv6.gateway' => 1,
 221     'lxc.network.script.up' => 1,
 222     'lxc.network.script.down' => 1,
 223     'lxc.pts' => 1,
 224     'lxc.console.logfile' => 1,
 225     'lxc.console' => 1,
 226     'lxc.tty' => 1,
 227     'lxc.devttydir' => 1,
 228     'lxc.hook.autodev' => 1,
 229     'lxc.autodev' => 1,
 230     'lxc.kmsg' => 1,
 231     'lxc.mount' => 1,
 232     'lxc.mount.entry' => 1,
 233     'lxc.mount.auto' => 1,
 234     'lxc.rootfs' => 1,
 235     'lxc.rootfs.mount' => 1,
 236     'lxc.rootfs.options' => 1,
 237     # lxc.cgroup.*
 238     'lxc.cap.drop' => 1,
 239     'lxc.cap.keep' => 1,
 240     'lxc.aa_profile' => 1,
 241     'lxc.aa_allow_incomplete' => 1,
 242     'lxc.se_context' => 1,
 243     'lxc.seccomp' => 1,
 244     'lxc.id_map' => 1,
 245     'lxc.hook.pre-start' => 1,
 246     'lxc.hook.pre-mount' => 1,
 247     'lxc.hook.mount' => 1,
 248     'lxc.hook.start' => 1,
 249     'lxc.hook.stop' => 1,
 250     'lxc.hook.post-stop' => 1,
 251     'lxc.hook.clone' => 1,
 252     'lxc.hook.destroy' => 1,
 253     'lxc.loglevel' => 1,
 254     'lxc.logfile' => 1,
 255     'lxc.start.auto' => 1,
 256     'lxc.start.delay' => 1,
 257     'lxc.start.order' => 1,
 258     'lxc.group' => 1,
 259     'lxc.environment' => 1,
 260     'lxc.' => 1,
 261     'lxc.' => 1,
 262     'lxc.' => 1,
 263     'lxc.' => 1,
 264 };
 265
 266 my $netconf_desc = {
 267     type => {
 268         type => 'string',
 269         optional => 1,
 270         description => "Network interface type.",
 271         enum => [qw(veth)],
 272     },
 273     name => {
 274         type => 'string',
 275         format_description => 'String',
 276         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 277         pattern => '[-_.\w\d]+',
 278     },
 279     bridge => {
 280         type => 'string',
 281         format_description => 'vmbr<Number>',
 282         description => 'Bridge to attach the network device to.',
 283         pattern => '[-_.\w\d]+',
 284         optional => 1,
 285     },
 286     hwaddr => {
 287         type => 'string',
 288         format_description => 'MAC',
 289         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 290         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 291         optional => 1,
 292     },
 293     mtu => {
 294         type => 'integer',
 295         format_description => 'Number',
 296         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 297         minimum => 64, # minimum ethernet frame is 64 bytes
 298         optional => 1,
 299     },
 300     ip => {
 301         type => 'string',
 302         format => 'pve-ipv4-config',
 303         format_description => 'IPv4Format/CIDR',
 304         description => 'IPv4 address in CIDR format.',
 305         optional => 1,
 306     },
 307     gw => {
 308         type => 'string',
 309         format => 'ipv4',
 310         format_description => 'GatewayIPv4',
 311         description => 'Default gateway for IPv4 traffic.',
 312         optional => 1,
 313     },
 314     ip6 => {
 315         type => 'string',
 316         format => 'pve-ipv6-config',
 317         format_description => 'IPv6Format/CIDR',
 318         description => 'IPv6 address in CIDR format.',
 319         optional => 1,
 320     },
 321     gw6 => {
 322         type => 'string',
 323         format => 'ipv6',
 324         format_description => 'GatewayIPv6',
 325         description => 'Default gateway for IPv6 traffic.',
 326         optional => 1,
 327     },
 328     firewall => {
 329         type => 'boolean',
 330         format_description => '[1|0]',
 331         description => "Controls whether this interface's firewall rules should be used.",
 332         optional => 1,
 333     },
 334     tag => {
 335         type => 'integer',
 336         format_description => 'VlanNo',
 337         minimum => '2',
 338         maximum => '4094',
 339         description => "VLAN tag foro this interface.",
 340         optional => 1,
 341     },
 342 };
 343 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 344
 345 my $MAX_LXC_NETWORKS = 10;
 346 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 347     $confdesc->{"net$i"} = {
 348         optional => 1,
 349         type => 'string', format => $netconf_desc,
 350         description => "Specifies network interfaces for the container.",
 351     };
 352 }
 353
 354 my $mp_desc = {
 355     %$rootfs_desc,
 356     mp => {
 357         type => 'string',
 358         format_description => 'Path',
 359         description => 'Path to the mountpoint as seen from inside the container.',
 360     },
 361 };
 362 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 363
 364 my $unuseddesc = {
 365     optional => 1,
 366     type => 'string', format => 'pve-volume-id',
 367     description => "Reference to unused volumes.",
 368 };
 369
 370 my $MAX_MOUNT_POINTS = 10;
 371 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 372     $confdesc->{"mp$i"} = {
 373         optional => 1,
 374         type => 'string', format => $mp_desc,
 375         description => "Use volume as container mount point (experimental feature).",
 376         optional => 1,
 377     };
 378 }
 379
 380 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 381 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 382     $confdesc->{"unused$i"} = $unuseddesc;
 383 }
 384
 385 sub write_pct_config {
 386     my ($filename, $conf) = @_;
 387
 388     delete $conf->{snapstate}; # just to be sure
 389
 390     my $generate_raw_config = sub {
 391         my ($conf) = @_;
 392
 393         my $raw = '';
 394
 395         # add description as comment to top of file
 396         my $descr = $conf->{description} || '';
 397         foreach my $cl (split(/\n/, $descr)) {
 398             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 399         }
 400
 401         foreach my $key (sort keys %$conf) {
 402             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 403                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 404             my $value = $conf->{$key};
 405             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 406             $raw .= "$key: $value\n";
 407         }
 408
 409         if (my $lxcconf = $conf->{lxc}) {
 410             foreach my $entry (@$lxcconf) {
 411                 my ($k, $v) = @$entry;
 412                 $raw .= "$k: $v\n";
 413             }
 414         }
 415
 416         return $raw;
 417     };
 418
 419     my $raw = &$generate_raw_config($conf);
 420
 421     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 422         $raw .= "\n[$snapname]\n";
 423         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 424     }
 425
 426     return $raw;
 427 }
 428
 429 sub check_type {
 430     my ($key, $value) = @_;
 431
 432     die "unknown setting '$key'\n" if !$confdesc->{$key};
 433
 434     my $type = $confdesc->{$key}->{type};
 435
 436     if (!defined($value)) {
 437         die "got undefined value\n";
 438     }
 439
 440     if ($value =~ m/[\n\r]/) {
 441         die "property contains a line feed\n";
 442     }
 443
 444     if ($type eq 'boolean') {
 445         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 446         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 447         die "type check ('boolean') failed - got '$value'\n";
 448     } elsif ($type eq 'integer') {
 449         return int($1) if $value =~ m/^(\d+)$/;
 450         die "type check ('integer') failed - got '$value'\n";
 451     } elsif ($type eq 'number') {
 452         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 453         die "type check ('number') failed - got '$value'\n";
 454     } elsif ($type eq 'string') {
 455         if (my $fmt = $confdesc->{$key}->{format}) {
 456             PVE::JSONSchema::check_format($fmt, $value);
 457             return $value;
 458         }
 459         return $value;
 460     } else {
 461         die "internal error"
 462     }
 463 }
 464
 465 sub parse_pct_config {
 466     my ($filename, $raw) = @_;
 467
 468     return undef if !defined($raw);
 469
 470     my $res = {
 471         digest => Digest::SHA::sha1_hex($raw),
 472         snapshots => {},
 473     };
 474
 475     $filename =~ m|/lxc/(\d+).conf$|
 476         || die "got strange filename '$filename'";
 477
 478     my $vmid = $1;
 479
 480     my $conf = $res;
 481     my $descr = '';
 482     my $section = '';
 483
 484     my @lines = split(/\n/, $raw);
 485     foreach my $line (@lines) {
 486         next if $line =~ m/^\s*$/;
 487
 488         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 489             $section = $1;
 490             $conf->{description} = $descr if $descr;
 491             $descr = '';
 492             $conf = $res->{snapshots}->{$section} = {};
 493             next;
 494         }
 495
 496         if ($line =~ m/^\#(.*)\s*$/) {
 497             $descr .= PVE::Tools::decode_text($1) . "\n";
 498             next;
 499         }
 500
 501         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 502             my $key = $1;
 503             my $value = $3;
 504             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 505                 push @{$conf->{lxc}}, [$key, $value];
 506             } else {
 507                 warn "vm $vmid - unable to parse config: $line\n";
 508             }
 509         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 510             $descr .= PVE::Tools::decode_text($2);
 511         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 512             $conf->{snapstate} = $1;
 513         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 514             my $key = $1;
 515             my $value = $2;
 516             eval { $value = check_type($key, $value); };
 517             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 518             $conf->{$key} = $value;
 519         } else {
 520             warn "vm $vmid - unable to parse config: $line\n";
 521         }
 522     }
 523
 524     $conf->{description} = $descr if $descr;
 525
 526     delete $res->{snapstate}; # just to be sure
 527
 528     return $res;
 529 }
 530
 531 sub config_list {
 532     my $vmlist = PVE::Cluster::get_vmlist();
 533     my $res = {};
 534     return $res if !$vmlist || !$vmlist->{ids};
 535     my $ids = $vmlist->{ids};
 536
 537     foreach my $vmid (keys %$ids) {
 538         next if !$vmid; # skip CT0
 539         my $d = $ids->{$vmid};
 540         next if !$d->{node} || $d->{node} ne $nodename;
 541         next if !$d->{type} || $d->{type} ne 'lxc';
 542         $res->{$vmid}->{type} = 'lxc';
 543     }
 544     return $res;
 545 }
 546
 547 sub cfs_config_path {
 548     my ($vmid, $node) = @_;
 549
 550     $node = $nodename if !$node;
 551     return "nodes/$node/lxc/$vmid.conf";
 552 }
 553
 554 sub config_file {
 555     my ($vmid, $node) = @_;
 556
 557     my $cfspath = cfs_config_path($vmid, $node);
 558     return "/etc/pve/$cfspath";
 559 }
 560
 561 sub load_config {
 562     my ($vmid, $node) = @_;
 563
 564     $node = $nodename if !$node;
 565     my $cfspath = cfs_config_path($vmid, $node);
 566
 567     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 568     die "container $vmid does not exists\n" if !defined($conf);
 569
 570     return $conf;
 571 }
 572
 573 sub create_config {
 574     my ($vmid, $conf) = @_;
 575
 576     my $dir = "/etc/pve/nodes/$nodename/lxc";
 577     mkdir $dir;
 578
 579     write_config($vmid, $conf);
 580 }
 581
 582 sub destroy_config {
 583     my ($vmid) = @_;
 584
 585     unlink config_file($vmid, $nodename);
 586 }
 587
 588 sub write_config {
 589     my ($vmid, $conf) = @_;
 590
 591     my $cfspath = cfs_config_path($vmid);
 592
 593     PVE::Cluster::cfs_write_file($cfspath, $conf);
 594 }
 595
 596 # flock: we use one file handle per process, so lock file
 597 # can be called multiple times and succeeds for the same process.
 598
 599 my $lock_handles =  {};
 600 my $lockdir = "/run/lock/lxc";
 601
 602 sub lock_filename {
 603     my ($vmid) = @_;
 604
 605     return "$lockdir/pve-config-${vmid}.lock";
 606 }
 607
 608 sub lock_aquire {
 609     my ($vmid, $timeout) = @_;
 610
 611     $timeout = 10 if !$timeout;
 612     my $mode = LOCK_EX;
 613
 614     my $filename = lock_filename($vmid);
 615
 616     mkdir $lockdir if !-d $lockdir;
 617
 618     my $lock_func = sub {
 619         if (!$lock_handles->{$$}->{$filename}) {
 620             my $fh = new IO::File(">>$filename") ||
 621                 die "can't open file - $!\n";
 622             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 623         }
 624
 625         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 626             print STDERR "trying to aquire lock...";
 627             my $success;
 628             while(1) {
 629                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 630                 # try again on EINTR (see bug #273)
 631                 if ($success || ($! != EINTR)) {
 632                     last;
 633                 }
 634             }
 635             if (!$success) {
 636                 print STDERR " failed\n";
 637                 die "can't aquire lock - $!\n";
 638             }
 639
 640             print STDERR " OK\n";
 641         }
 642
 643         $lock_handles->{$$}->{$filename}->{refcount}++;
 644     };
 645
 646     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 647     my $err = $@;
 648     if ($err) {
 649         die "can't lock file '$filename' - $err";
 650     }
 651 }
 652
 653 sub lock_release {
 654     my ($vmid) = @_;
 655
 656     my $filename = lock_filename($vmid);
 657
 658     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 659         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 660         if ($refcount <= 0) {
 661             $lock_handles->{$$}->{$filename} = undef;
 662             close ($fh);
 663         }
 664     }
 665 }
 666
 667 sub lock_container {
 668     my ($vmid, $timeout, $code, @param) = @_;
 669
 670     my $res;
 671
 672     lock_aquire($vmid, $timeout);
 673     eval { $res = &$code(@param) };
 674     my $err = $@;
 675     lock_release($vmid);
 676
 677     die $err if $err;
 678
 679     return $res;
 680 }
 681
 682 sub option_exists {
 683     my ($name) = @_;
 684
 685     return defined($confdesc->{$name});
 686 }
 687
 688 # add JSON properties for create and set function
 689 sub json_config_properties {
 690     my $prop = shift;
 691
 692     foreach my $opt (keys %$confdesc) {
 693         next if $opt eq 'parent' || $opt eq 'snaptime';
 694         next if $prop->{$opt};
 695         $prop->{$opt} = $confdesc->{$opt};
 696     }
 697
 698     return $prop;
 699 }
 700
 701 sub json_config_properties_no_rootfs {
 702     my $prop = shift;
 703
 704     foreach my $opt (keys %$confdesc) {
 705         next if $prop->{$opt};
 706         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 707         $prop->{$opt} = $confdesc->{$opt};
 708     }
 709
 710     return $prop;
 711 }
 712
 713 # container status helpers
 714
 715 sub list_active_containers {
 716
 717     my $filename = "/proc/net/unix";
 718
 719     # similar test is used by lcxcontainers.c: list_active_containers
 720     my $res = {};
 721
 722     my $fh = IO::File->new ($filename, "r");
 723     return $res if !$fh;
 724
 725     while (defined(my $line = <$fh>)) {
 726         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 727             my $path = $1;
 728             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 729                 $res->{$1} = 1;
 730             }
 731         }
 732     }
 733
 734     close($fh);
 735
 736     return $res;
 737 }
 738
 739 # warning: this is slow
 740 sub check_running {
 741     my ($vmid) = @_;
 742
 743     my $active_hash = list_active_containers();
 744
 745     return 1 if defined($active_hash->{$vmid});
 746
 747     return undef;
 748 }
 749
 750 sub get_container_disk_usage {
 751     my ($vmid, $pid) = @_;
 752
 753     return PVE::Tools::df("/proc/$pid/root/", 1);
 754 }
 755
 756 my $last_proc_vmid_stat;
 757
 758 my $parse_cpuacct_stat = sub {
 759     my ($vmid) = @_;
 760
 761     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 762
 763     my $stat = {};
 764
 765     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 766
 767         $stat->{utime} = $1;
 768         $stat->{stime} = $2;
 769
 770     }
 771
 772     return $stat;
 773 };
 774
 775 sub vmstatus {
 776     my ($opt_vmid) = @_;
 777
 778     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 779
 780     my $active_hash = list_active_containers();
 781
 782     my $cpucount = $cpuinfo->{cpus} || 1;
 783
 784     my $cdtime = gettimeofday;
 785
 786     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 787
 788     foreach my $vmid (keys %$list) {
 789         my $d = $list->{$vmid};
 790
 791         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 792         warn $@ if $@; # ignore errors (consider them stopped)
 793
 794         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 795
 796         my $cfspath = cfs_config_path($vmid);
 797         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 798
 799         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 800         $d->{name} =~ s/[\s]//g;
 801
 802         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 803
 804         if ($d->{pid}) {
 805             my $res = get_container_disk_usage($vmid, $d->{pid});
 806             $d->{disk} = $res->{used};
 807             $d->{maxdisk} = $res->{total};
 808         } else {
 809             $d->{disk} = 0;
 810             # use 4GB by default ??
 811             if (my $rootfs = $conf->{rootfs}) {
 812                 my $rootinfo = parse_ct_mountpoint($rootfs);
 813                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 814             } else {
 815                 $d->{maxdisk} = 4*1024*1024*1024;
 816             }
 817         }
 818
 819         $d->{mem} = 0;
 820         $d->{swap} = 0;
 821         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 822         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 823
 824         $d->{uptime} = 0;
 825         $d->{cpu} = 0;
 826
 827         $d->{netout} = 0;
 828         $d->{netin} = 0;
 829
 830         $d->{diskread} = 0;
 831         $d->{diskwrite} = 0;
 832
 833         $d->{template} = is_template($conf);
 834     }
 835
 836     foreach my $vmid (keys %$list) {
 837         my $d = $list->{$vmid};
 838         my $pid = $d->{pid};
 839
 840         next if !$pid; # skip stopped CTs
 841
 842         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 843         $d->{uptime} = time - $ctime; # the method lxcfs uses
 844
 845         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 846         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 847
 848         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 849         my @bytes = split(/\n/, $blkio_bytes);
 850         foreach my $byte (@bytes) {
 851             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 852                 $d->{diskread} = $2 if $key eq 'Read';
 853                 $d->{diskwrite} = $2 if $key eq 'Write';
 854             }
 855         }
 856
 857         my $pstat = &$parse_cpuacct_stat($vmid);
 858
 859         my $used = $pstat->{utime} + $pstat->{stime};
 860
 861         my $old = $last_proc_vmid_stat->{$vmid};
 862         if (!$old) {
 863             $last_proc_vmid_stat->{$vmid} = {
 864                 time => $cdtime,
 865                 used => $used,
 866                 cpu => 0,
 867             };
 868             next;
 869         }
 870
 871         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 872
 873         if ($dtime > 1000) {
 874             my $dutime = $used -  $old->{used};
 875
 876             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 877             $last_proc_vmid_stat->{$vmid} = {
 878                 time => $cdtime,
 879                 used => $used,
 880                 cpu => $d->{cpu},
 881             };
 882         } else {
 883             $d->{cpu} = $old->{cpu};
 884         }
 885     }
 886
 887     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 888
 889     foreach my $dev (keys %$netdev) {
 890         next if $dev !~ m/^veth([1-9]\d*)i/;
 891         my $vmid = $1;
 892         my $d = $list->{$vmid};
 893
 894         next if !$d;
 895
 896         $d->{netout} += $netdev->{$dev}->{receive};
 897         $d->{netin} += $netdev->{$dev}->{transmit};
 898
 899     }
 900
 901     return $list;
 902 }
 903
 904 sub classify_mountpoint {
 905     my ($vol) = @_;
 906     if ($vol =~ m!^/!) {
 907         return 'device' if $vol =~ m!^/dev/!;
 908         return 'bind';
 909     }
 910     return 'volume';
 911 }
 912
 913 sub parse_ct_mountpoint {
 914     my ($data, $noerr) = @_;
 915
 916     $data //= '';
 917
 918     my $res;
 919     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 920     if ($@) {
 921         return undef if $noerr;
 922         die $@;
 923     }
 924
 925     if (defined(my $size = $res->{size})) {
 926         $size = PVE::JSONSchema::parse_size($size);
 927         if (!defined($size)) {
 928             return undef if $noerr;
 929             die "invalid size: $size\n";
 930         }
 931         $res->{size} = $size;
 932     }
 933
 934     $res->{type} = classify_mountpoint($res->{volume});
 935
 936     return $res;
 937 }
 938
 939 sub print_ct_mountpoint {
 940     my ($info, $nomp) = @_;
 941     my $skip = [ 'type' ];
 942     push @$skip, 'mp' if $nomp;
 943     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 944 }
 945
 946 sub print_lxc_network {
 947     my $net = shift;
 948     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 949 }
 950
 951 sub parse_lxc_network {
 952     my ($data) = @_;
 953
 954     my $res = {};
 955
 956     return $res if !$data;
 957
 958     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 959
 960     $res->{type} = 'veth';
 961     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 962
 963     return $res;
 964 }
 965
 966 sub read_cgroup_value {
 967     my ($group, $vmid, $name, $full) = @_;
 968
 969     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 970
 971     return PVE::Tools::file_get_contents($path) if $full;
 972
 973     return PVE::Tools::file_read_firstline($path);
 974 }
 975
 976 sub write_cgroup_value {
 977    my ($group, $vmid, $name, $value) = @_;
 978
 979    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 980    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 981
 982 }
 983
 984 sub find_lxc_console_pids {
 985
 986     my $res = {};
 987
 988     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 989         my ($pid) = @_;
 990
 991         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 992         return if !$cmdline;
 993
 994         my @args = split(/\0/, $cmdline);
 995
 996         # serach for lxc-console -n <vmid>
 997         return if scalar(@args) != 3;
 998         return if $args[1] ne '-n';
 999         return if $args[2] !~ m/^\d+$/;
1000         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1001
1002         my $vmid = $args[2];
1003
1004         push @{$res->{$vmid}}, $pid;
1005     });
1006
1007     return $res;
1008 }
1009
1010 sub find_lxc_pid {
1011     my ($vmid) = @_;
1012
1013     my $pid = undef;
1014     my $parser = sub {
1015         my $line = shift;
1016         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1017     };
1018     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1019
1020     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1021
1022     return $pid;
1023 }
1024
1025 # Note: we cannot use Net:IP, because that only allows strict
1026 # CIDR networks
1027 sub parse_ipv4_cidr {
1028     my ($cidr, $noerr) = @_;
1029
1030     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1031         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1032     }
1033
1034     return undef if $noerr;
1035
1036     die "unable to parse ipv4 address/mask\n";
1037 }
1038
1039 sub check_lock {
1040     my ($conf) = @_;
1041
1042     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1043 }
1044
1045 sub check_protection {
1046     my ($vm_conf, $err_msg) = @_;
1047
1048     if ($vm_conf->{protection}) {
1049         die "$err_msg - protection mode enabled\n";
1050     }
1051 }
1052
1053 sub update_lxc_config {
1054     my ($storage_cfg, $vmid, $conf) = @_;
1055
1056     my $dir = "/var/lib/lxc/$vmid";
1057
1058     if ($conf->{template}) {
1059
1060         unlink "$dir/config";
1061
1062         return;
1063     }
1064
1065     my $raw = '';
1066
1067     die "missing 'arch' - internal error" if !$conf->{arch};
1068     $raw .= "lxc.arch = $conf->{arch}\n";
1069
1070     my $unprivileged = $conf->{unprivileged};
1071     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1072
1073     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1074     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux)$/x) {
1075         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1076         if ($unprivileged || $custom_idmap) {
1077             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1078         }
1079     } else {
1080         die "implement me (ostype $ostype)";
1081     }
1082
1083     $raw .= "lxc.monitor.unshare = 1\n";
1084
1085     # Should we read them from /etc/subuid?
1086     if ($unprivileged && !$custom_idmap) {
1087         $raw .= "lxc.id_map = u 0 100000 65536\n";
1088         $raw .= "lxc.id_map = g 0 100000 65536\n";
1089     }
1090
1091     if (!has_dev_console($conf)) {
1092         $raw .= "lxc.console = none\n";
1093         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1094     }
1095
1096     my $ttycount = get_tty_count($conf);
1097     $raw .= "lxc.tty = $ttycount\n";
1098
1099     # some init scripts expects a linux terminal (turnkey).
1100     $raw .= "lxc.environment = TERM=linux\n";
1101
1102     my $utsname = $conf->{hostname} || "CT$vmid";
1103     $raw .= "lxc.utsname = $utsname\n";
1104
1105     my $memory = $conf->{memory} || 512;
1106     my $swap = $conf->{swap} // 0;
1107
1108     my $lxcmem = int($memory*1024*1024);
1109     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1110
1111     my $lxcswap = int(($memory + $swap)*1024*1024);
1112     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1113
1114     if (my $cpulimit = $conf->{cpulimit}) {
1115         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1116         my $value = int(100000*$cpulimit);
1117         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1118     }
1119
1120     my $shares = $conf->{cpuunits} || 1024;
1121     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1122
1123     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1124     $mountpoint->{mp} = '/';
1125
1126     $raw .= "lxc.rootfs = $dir/rootfs\n";
1127
1128     my $netcount = 0;
1129     foreach my $k (keys %$conf) {
1130         next if $k !~ m/^net(\d+)$/;
1131         my $ind = $1;
1132         my $d = parse_lxc_network($conf->{$k});
1133         $netcount++;
1134         $raw .= "lxc.network.type = veth\n";
1135         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1136         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1137         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1138         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1139     }
1140
1141     if (my $lxcconf = $conf->{lxc}) {
1142         foreach my $entry (@$lxcconf) {
1143             my ($k, $v) = @$entry;
1144             $netcount++ if $k eq 'lxc.network.type';
1145             $raw .= "$k = $v\n";
1146         }
1147     }
1148
1149     $raw .= "lxc.network.type = empty\n" if !$netcount;
1150
1151     File::Path::mkpath("$dir/rootfs");
1152
1153     PVE::Tools::file_set_contents("$dir/config", $raw);
1154 }
1155
1156 # verify and cleanup nameserver list (replace \0 with ' ')
1157 sub verify_nameserver_list {
1158     my ($nameserver_list) = @_;
1159
1160     my @list = ();
1161     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1162         PVE::JSONSchema::pve_verify_ip($server);
1163         push @list, $server;
1164     }
1165
1166     return join(' ', @list);
1167 }
1168
1169 sub verify_searchdomain_list {
1170     my ($searchdomain_list) = @_;
1171
1172     my @list = ();
1173     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1174         # todo: should we add checks for valid dns domains?
1175         push @list, $server;
1176     }
1177
1178     return join(' ', @list);
1179 }
1180
1181 sub add_unused_volume {
1182     my ($config, $volid) = @_;
1183
1184     my $key;
1185     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1186         my $test = "unused$ind";
1187         if (my $vid = $config->{$test}) {
1188             return if $vid eq $volid; # do not add duplicates
1189         } else {
1190             $key = $test;
1191         }
1192     }
1193
1194     die "To many unused volume - please delete them first.\n" if !$key;
1195
1196     $config->{$key} = $volid;
1197
1198     return $key;
1199 }
1200
1201 sub update_pct_config {
1202     my ($vmid, $conf, $running, $param, $delete) = @_;
1203
1204     my @nohotplug;
1205
1206     my $new_disks = 0;
1207     my @deleted_volumes;
1208
1209     my $rootdir;
1210     if ($running) {
1211         my $pid = find_lxc_pid($vmid);
1212         $rootdir = "/proc/$pid/root";
1213     }
1214
1215     my $hotplug_error = sub {
1216         if ($running) {
1217             push @nohotplug, @_;
1218             return 1;
1219         } else {
1220             return 0;
1221         }
1222     };
1223
1224     if (defined($delete)) {
1225         foreach my $opt (@$delete) {
1226             if (!exists($conf->{$opt})) {
1227                 warn "no such option: $opt\n";
1228                 next;
1229             }
1230
1231             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1232                 die "unable to delete required option '$opt'\n";
1233             } elsif ($opt eq 'swap') {
1234                 delete $conf->{$opt};
1235                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1236             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1237                 delete $conf->{$opt};
1238             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1239                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1240                 next if $hotplug_error->($opt);
1241                 delete $conf->{$opt};
1242             } elsif ($opt =~ m/^net(\d)$/) {
1243                 delete $conf->{$opt};
1244                 next if !$running;
1245                 my $netid = $1;
1246                 PVE::Network::veth_delete("veth${vmid}i$netid");
1247             } elsif ($opt eq 'protection') {
1248                 delete $conf->{$opt};
1249             } elsif ($opt =~ m/^unused(\d+)$/) {
1250                 next if $hotplug_error->($opt);
1251                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1252                 push @deleted_volumes, $conf->{$opt};
1253                 delete $conf->{$opt};
1254             } elsif ($opt =~ m/^mp(\d+)$/) {
1255                 next if $hotplug_error->($opt);
1256                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1257                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1258                 if ($mountpoint->{type} eq 'volume') {
1259                     add_unused_volume($conf, $mountpoint->{volume})
1260                 }
1261                 delete $conf->{$opt};
1262             } elsif ($opt eq 'unprivileged') {
1263                 die "unable to delete read-only option: '$opt'\n";
1264             } else {
1265                 die "implement me (delete: $opt)"
1266             }
1267             write_config($vmid, $conf) if $running;
1268         }
1269     }
1270
1271     # There's no separate swap size to configure, there's memory and "total"
1272     # memory (iow. memory+swap). This means we have to change them together.
1273     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1274     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1275     if (defined($wanted_memory) || defined($wanted_swap)) {
1276
1277         $wanted_memory //= ($conf->{memory} || 512);
1278         $wanted_swap //=  ($conf->{swap} || 0);
1279
1280         my $total = $wanted_memory + $wanted_swap;
1281         if ($running) {
1282             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1283             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1284         }
1285         $conf->{memory} = $wanted_memory;
1286         $conf->{swap} = $wanted_swap;
1287
1288         write_config($vmid, $conf) if $running;
1289     }
1290
1291     foreach my $opt (keys %$param) {
1292         my $value = $param->{$opt};
1293         if ($opt eq 'hostname') {
1294             $conf->{$opt} = $value;
1295         } elsif ($opt eq 'onboot') {
1296             $conf->{$opt} = $value ? 1 : 0;
1297         } elsif ($opt eq 'startup') {
1298             $conf->{$opt} = $value;
1299         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1300             next if $hotplug_error->($opt);
1301             $conf->{$opt} = $value;
1302         } elsif ($opt eq 'nameserver') {
1303             next if $hotplug_error->($opt);
1304             my $list = verify_nameserver_list($value);
1305             $conf->{$opt} = $list;
1306         } elsif ($opt eq 'searchdomain') {
1307             next if $hotplug_error->($opt);
1308             my $list = verify_searchdomain_list($value);
1309             $conf->{$opt} = $list;
1310         } elsif ($opt eq 'cpulimit') {
1311             next if $hotplug_error->($opt); # FIXME: hotplug
1312             $conf->{$opt} = $value;
1313         } elsif ($opt eq 'cpuunits') {
1314             $conf->{$opt} = $value;
1315             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1316         } elsif ($opt eq 'description') {
1317             $conf->{$opt} = PVE::Tools::encode_text($value);
1318         } elsif ($opt =~ m/^net(\d+)$/) {
1319             my $netid = $1;
1320             my $net = parse_lxc_network($value);
1321             if (!$running) {
1322                 $conf->{$opt} = print_lxc_network($net);
1323             } else {
1324                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1325             }
1326         } elsif ($opt eq 'protection') {
1327             $conf->{$opt} = $value ? 1 : 0;
1328         } elsif ($opt =~ m/^mp(\d+)$/) {
1329             next if $hotplug_error->($opt);
1330             check_protection($conf, "can't update CT $vmid drive '$opt'");
1331             $conf->{$opt} = $value;
1332             $new_disks = 1;
1333         } elsif ($opt eq 'rootfs') {
1334             check_protection($conf, "can't update CT $vmid drive '$opt'");
1335             die "implement me: $opt";
1336         } elsif ($opt eq 'unprivileged') {
1337             die "unable to modify read-only option: '$opt'\n";
1338         } else {
1339             die "implement me: $opt";
1340         }
1341         write_config($vmid, $conf) if $running;
1342     }
1343
1344     if (@deleted_volumes) {
1345         my $storage_cfg = PVE::Storage::config();
1346         foreach my $volume (@deleted_volumes) {
1347             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1348         }
1349     }
1350
1351     if ($new_disks) {
1352         my $storage_cfg = PVE::Storage::config();
1353         create_disks($storage_cfg, $vmid, $conf, $conf);
1354     }
1355
1356     # This should be the last thing we do here
1357     if ($running && scalar(@nohotplug)) {
1358         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1359     }
1360 }
1361
1362 sub has_dev_console {
1363     my ($conf) = @_;
1364
1365     return !(defined($conf->{console}) && !$conf->{console});
1366 }
1367
1368 sub get_tty_count {
1369     my ($conf) = @_;
1370
1371     return $conf->{tty} // $confdesc->{tty}->{default};
1372 }
1373
1374 sub get_cmode {
1375     my ($conf) = @_;
1376
1377     return $conf->{cmode} // $confdesc->{cmode}->{default};
1378 }
1379
1380 sub get_console_command {
1381     my ($vmid, $conf) = @_;
1382
1383     my $cmode = get_cmode($conf);
1384
1385     if ($cmode eq 'console') {
1386         return ['lxc-console', '-n',  $vmid, '-t', 0];
1387     } elsif ($cmode eq 'tty') {
1388         return ['lxc-console', '-n',  $vmid];
1389     } elsif ($cmode eq 'shell') {
1390         return ['lxc-attach', '--clear-env', '-n', $vmid];
1391     } else {
1392         die "internal error";
1393     }
1394 }
1395
1396 sub get_primary_ips {
1397     my ($conf) = @_;
1398
1399     # return data from net0
1400
1401     return undef if !defined($conf->{net0});
1402     my $net = parse_lxc_network($conf->{net0});
1403
1404     my $ipv4 = $net->{ip};
1405     if ($ipv4) {
1406         if ($ipv4 =~ /^(dhcp|manual)$/) {
1407             $ipv4 = undef
1408         } else {
1409             $ipv4 =~ s!/\d+$!!;
1410         }
1411     }
1412     my $ipv6 = $net->{ip6};
1413     if ($ipv6) {
1414         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1415             $ipv6 = undef;
1416         } else {
1417             $ipv6 =~ s!/\d+$!!;
1418         }
1419     }
1420
1421     return ($ipv4, $ipv6);
1422 }
1423
1424 sub delete_mountpoint_volume {
1425     my ($storage_cfg, $vmid, $volume) = @_;
1426
1427     return if classify_mountpoint($volume) ne 'volume';
1428
1429     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1430     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1431 }
1432
1433 sub destroy_lxc_container {
1434     my ($storage_cfg, $vmid, $conf) = @_;
1435
1436     foreach_mountpoint($conf, sub {
1437         my ($ms, $mountpoint) = @_;
1438         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1439     });
1440
1441     rmdir "/var/lib/lxc/$vmid/rootfs";
1442     unlink "/var/lib/lxc/$vmid/config";
1443     rmdir "/var/lib/lxc/$vmid";
1444     destroy_config($vmid);
1445
1446     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1447     #PVE::Tools::run_command($cmd);
1448 }
1449
1450 sub vm_stop_cleanup {
1451     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1452
1453     eval {
1454         if (!$keepActive) {
1455
1456             my $vollist = get_vm_volumes($conf);
1457             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1458         }
1459     };
1460     warn $@ if $@; # avoid errors - just warn
1461 }
1462
1463 my $safe_num_ne = sub {
1464     my ($a, $b) = @_;
1465
1466     return 0 if !defined($a) && !defined($b);
1467     return 1 if !defined($a);
1468     return 1 if !defined($b);
1469
1470     return $a != $b;
1471 };
1472
1473 my $safe_string_ne = sub {
1474     my ($a, $b) = @_;
1475
1476     return 0 if !defined($a) && !defined($b);
1477     return 1 if !defined($a);
1478     return 1 if !defined($b);
1479
1480     return $a ne $b;
1481 };
1482
1483 sub update_net {
1484     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1485
1486     if ($newnet->{type} ne 'veth') {
1487         # for when there are physical interfaces
1488         die "cannot update interface of type $newnet->{type}";
1489     }
1490
1491     my $veth = "veth${vmid}i${netid}";
1492     my $eth = $newnet->{name};
1493
1494     if (my $oldnetcfg = $conf->{$opt}) {
1495         my $oldnet = parse_lxc_network($oldnetcfg);
1496
1497         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1498             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1499
1500             PVE::Network::veth_delete($veth);
1501             delete $conf->{$opt};
1502             write_config($vmid, $conf);
1503
1504             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1505
1506         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1507                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1508                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1509
1510                 if ($oldnet->{bridge}) {
1511                     PVE::Network::tap_unplug($veth);
1512                     foreach (qw(bridge tag firewall)) {
1513                         delete $oldnet->{$_};
1514                     }
1515                     $conf->{$opt} = print_lxc_network($oldnet);
1516                     write_config($vmid, $conf);
1517                 }
1518
1519                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1520                 foreach (qw(bridge tag firewall)) {
1521                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1522                 }
1523                 $conf->{$opt} = print_lxc_network($oldnet);
1524                 write_config($vmid, $conf);
1525         }
1526     } else {
1527         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1528     }
1529
1530     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1531 }
1532
1533 sub hotplug_net {
1534     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1535
1536     my $veth = "veth${vmid}i${netid}";
1537     my $vethpeer = $veth . "p";
1538     my $eth = $newnet->{name};
1539
1540     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1541     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1542
1543     # attach peer in container
1544     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1545     PVE::Tools::run_command($cmd);
1546
1547     # link up peer in container
1548     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1549     PVE::Tools::run_command($cmd);
1550
1551     my $done = { type => 'veth' };
1552     foreach (qw(bridge tag firewall hwaddr name)) {
1553         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1554     }
1555     $conf->{$opt} = print_lxc_network($done);
1556
1557     write_config($vmid, $conf);
1558 }
1559
1560 sub update_ipconfig {
1561     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1562
1563     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1564
1565     my $optdata = parse_lxc_network($conf->{$opt});
1566     my $deleted = [];
1567     my $added = [];
1568     my $nscmd = sub {
1569         my $cmdargs = shift;
1570         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1571     };
1572     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1573
1574     my $change_ip_config = sub {
1575         my ($ipversion) = @_;
1576
1577         my $family_opt = "-$ipversion";
1578         my $suffix = $ipversion == 4 ? '' : $ipversion;
1579         my $gw= "gw$suffix";
1580         my $ip= "ip$suffix";
1581
1582         my $newip = $newnet->{$ip};
1583         my $newgw = $newnet->{$gw};
1584         my $oldip = $optdata->{$ip};
1585
1586         my $change_ip = &$safe_string_ne($oldip, $newip);
1587         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1588
1589         return if !$change_ip && !$change_gw;
1590
1591         # step 1: add new IP, if this fails we cancel
1592         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1593         if ($change_ip && $is_real_ip) {
1594             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1595             if (my $err = $@) {
1596                 warn $err;
1597                 return;
1598             }
1599         }
1600
1601         # step 2: replace gateway
1602         #   If this fails we delete the added IP and cancel.
1603         #   If it succeeds we save the config and delete the old IP, ignoring
1604         #   errors. The config is then saved.
1605         # Note: 'ip route replace' can add
1606         if ($change_gw) {
1607             if ($newgw) {
1608                 eval {
1609                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1610                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1611                     }
1612                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1613                 };
1614                 if (my $err = $@) {
1615                     warn $err;
1616                     # the route was not replaced, the old IP is still available
1617                     # rollback (delete new IP) and cancel
1618                     if ($change_ip) {
1619                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1620                         warn $@ if $@; # no need to die here
1621                     }
1622                     return;
1623                 }
1624             } else {
1625                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1626                 # if the route was not deleted, the guest might have deleted it manually
1627                 # warn and continue
1628                 warn $@ if $@;
1629             }
1630         }
1631
1632         # from this point on we save the configuration
1633         # step 3: delete old IP ignoring errors
1634         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1635             # We need to enable promote_secondaries, otherwise our newly added
1636             # address will be removed along with the old one.
1637             my $promote = 0;
1638             eval {
1639                 if ($ipversion == 4) {
1640                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1641                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1642                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1643                 }
1644                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1645             };
1646             warn $@ if $@; # no need to die here
1647
1648             if ($ipversion == 4) {
1649                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1650             }
1651         }
1652
1653         foreach my $property ($ip, $gw) {
1654             if ($newnet->{$property}) {
1655                 $optdata->{$property} = $newnet->{$property};
1656             } else {
1657                 delete $optdata->{$property};
1658             }
1659         }
1660         $conf->{$opt} = print_lxc_network($optdata);
1661         write_config($vmid, $conf);
1662         $lxc_setup->setup_network($conf);
1663     };
1664
1665     &$change_ip_config(4);
1666     &$change_ip_config(6);
1667
1668 }
1669
1670 # Internal snapshots
1671
1672 # NOTE: Snapshot create/delete involves several non-atomic
1673 # action, and can take a long time.
1674 # So we try to avoid locking the file and use 'lock' variable
1675 # inside the config file instead.
1676
1677 my $snapshot_copy_config = sub {
1678     my ($source, $dest) = @_;
1679
1680     foreach my $k (keys %$source) {
1681         next if $k eq 'snapshots';
1682         next if $k eq 'snapstate';
1683         next if $k eq 'snaptime';
1684         next if $k eq 'vmstate';
1685         next if $k eq 'lock';
1686         next if $k eq 'digest';
1687         next if $k eq 'description';
1688
1689         $dest->{$k} = $source->{$k};
1690     }
1691 };
1692
1693 my $snapshot_prepare = sub {
1694     my ($vmid, $snapname, $comment) = @_;
1695
1696     my $snap;
1697
1698     my $updatefn =  sub {
1699
1700         my $conf = load_config($vmid);
1701
1702         die "you can't take a snapshot if it's a template\n"
1703             if is_template($conf);
1704
1705         check_lock($conf);
1706
1707         $conf->{lock} = 'snapshot';
1708
1709         die "snapshot name '$snapname' already used\n"
1710             if defined($conf->{snapshots}->{$snapname});
1711
1712         my $storecfg = PVE::Storage::config();
1713         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1714
1715         $snap = $conf->{snapshots}->{$snapname} = {};
1716
1717         &$snapshot_copy_config($conf, $snap);
1718
1719         $snap->{'snapstate'} = "prepare";
1720         $snap->{'snaptime'} = time();
1721         $snap->{'description'} = $comment if $comment;
1722         $conf->{snapshots}->{$snapname} = $snap;
1723
1724         write_config($vmid, $conf);
1725     };
1726
1727     lock_container($vmid, 10, $updatefn);
1728
1729     return $snap;
1730 };
1731
1732 my $snapshot_commit = sub {
1733     my ($vmid, $snapname) = @_;
1734
1735     my $updatefn = sub {
1736
1737         my $conf = load_config($vmid);
1738
1739         die "missing snapshot lock\n"
1740             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1741
1742         die "snapshot '$snapname' does not exist\n"
1743             if !defined($conf->{snapshots}->{$snapname});
1744
1745         die "wrong snapshot state\n"
1746             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1747                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1748
1749         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1750         delete $conf->{lock};
1751         $conf->{parent} = $snapname;
1752
1753         write_config($vmid, $conf);
1754     };
1755
1756     lock_container($vmid, 10 ,$updatefn);
1757 };
1758
1759 sub has_feature {
1760     my ($feature, $conf, $storecfg, $snapname) = @_;
1761
1762     my $err;
1763
1764     foreach_mountpoint($conf, sub {
1765         my ($ms, $mountpoint) = @_;
1766
1767         return if $err; # skip further test
1768
1769         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1770
1771         # TODO: implement support for mountpoints
1772         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1773             if $ms ne 'rootfs';
1774     });
1775
1776     return $err ? 0 : 1;
1777 }
1778
1779 sub snapshot_create {
1780     my ($vmid, $snapname, $comment) = @_;
1781
1782     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1783
1784     my $conf = load_config($vmid);
1785
1786     my $running = check_running($vmid);
1787
1788     my $unfreeze = 0;
1789
1790     eval {
1791         if ($running) {
1792             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1793             $unfreeze = 1;
1794             PVE::Tools::run_command(['/bin/sync']);
1795         };
1796
1797         my $storecfg = PVE::Storage::config();
1798         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1799         my $volid = $rootinfo->{volume};
1800
1801         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1802         &$snapshot_commit($vmid, $snapname);
1803     };
1804     my $err = $@;
1805
1806     if ($unfreeze) {
1807         eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
1808         warn $@ if $@;
1809     }
1810
1811     if ($err) {
1812         snapshot_delete($vmid, $snapname, 1);
1813         die "$err\n";
1814     }
1815 }
1816
1817 sub snapshot_delete {
1818     my ($vmid, $snapname, $force) = @_;
1819
1820     my $snap;
1821
1822     my $conf;
1823
1824     my $updatefn =  sub {
1825
1826         $conf = load_config($vmid);
1827
1828         die "you can't delete a snapshot if vm is a template\n"
1829             if is_template($conf);
1830
1831         $snap = $conf->{snapshots}->{$snapname};
1832
1833         check_lock($conf);
1834
1835         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1836
1837         $snap->{snapstate} = 'delete';
1838
1839         write_config($vmid, $conf);
1840     };
1841
1842     lock_container($vmid, 10, $updatefn);
1843
1844     my $storecfg = PVE::Storage::config();
1845
1846     my $del_snap =  sub {
1847
1848         check_lock($conf);
1849
1850         if ($conf->{parent} eq $snapname) {
1851             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1852                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1853             } else {
1854                 delete $conf->{parent};
1855             }
1856         }
1857
1858         delete $conf->{snapshots}->{$snapname};
1859
1860         write_config($vmid, $conf);
1861     };
1862
1863     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1864     my $rootinfo = parse_ct_mountpoint($rootfs);
1865     my $volid = $rootinfo->{volume};
1866
1867     eval {
1868         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1869     };
1870     my $err = $@;
1871
1872     if(!$err || ($err && $force)) {
1873         lock_container($vmid, 10, $del_snap);
1874         if ($err) {
1875             die "Can't delete snapshot: $vmid $snapname $err\n";
1876         }
1877     }
1878 }
1879
1880 sub snapshot_rollback {
1881     my ($vmid, $snapname) = @_;
1882
1883     my $storecfg = PVE::Storage::config();
1884
1885     my $conf = load_config($vmid);
1886
1887     die "you can't rollback if vm is a template\n" if is_template($conf);
1888
1889     my $snap = $conf->{snapshots}->{$snapname};
1890
1891     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1892
1893     my $rootfs = $snap->{rootfs};
1894     my $rootinfo = parse_ct_mountpoint($rootfs);
1895     my $volid = $rootinfo->{volume};
1896
1897     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1898
1899     my $updatefn = sub {
1900
1901         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1902             if $snap->{snapstate};
1903
1904         check_lock($conf);
1905
1906         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1907
1908         die "unable to rollback vm $vmid: vm is running\n"
1909             if check_running($vmid);
1910
1911         $conf->{lock} = 'rollback';
1912
1913         my $forcemachine;
1914
1915         # copy snapshot config to current config
1916
1917         my $tmp_conf = $conf;
1918         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1919         $conf->{snapshots} = $tmp_conf->{snapshots};
1920         delete $conf->{snaptime};
1921         delete $conf->{snapname};
1922         $conf->{parent} = $snapname;
1923
1924         write_config($vmid, $conf);
1925     };
1926
1927     my $unlockfn = sub {
1928         delete $conf->{lock};
1929         write_config($vmid, $conf);
1930     };
1931
1932     lock_container($vmid, 10, $updatefn);
1933
1934     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1935
1936     lock_container($vmid, 5, $unlockfn);
1937 }
1938
1939 sub template_create {
1940     my ($vmid, $conf) = @_;
1941
1942     my $storecfg = PVE::Storage::config();
1943
1944     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1945     my $volid = $rootinfo->{volume};
1946
1947     die "Template feature is not available for '$volid'\n"
1948         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1949
1950     PVE::Storage::activate_volumes($storecfg, [$volid]);
1951
1952     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1953     $rootinfo->{volume} = $template_volid;
1954     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1955
1956     write_config($vmid, $conf);
1957 }
1958
1959 sub is_template {
1960     my ($conf) = @_;
1961
1962     return 1 if defined $conf->{template} && $conf->{template} == 1;
1963 }
1964
1965 sub mountpoint_names {
1966     my ($reverse) = @_;
1967
1968     my @names = ('rootfs');
1969
1970     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1971         push @names, "mp$i";
1972     }
1973
1974     return $reverse ? reverse @names : @names;
1975 }
1976
1977 # The container might have *different* symlinks than the host. realpath/abs_path
1978 # use the actual filesystem to resolve links.
1979 sub sanitize_mountpoint {
1980     my ($mp) = @_;
1981     $mp = '/' . $mp; # we always start with a slash
1982     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1983     $mp =~ s@/\./@@g; # collapse /./
1984     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1985     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1986     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1987     return $mp;
1988 }
1989
1990 sub foreach_mountpoint_full {
1991     my ($conf, $reverse, $func) = @_;
1992
1993     foreach my $key (mountpoint_names($reverse)) {
1994         my $value = $conf->{$key};
1995         next if !defined($value);
1996         my $mountpoint = parse_ct_mountpoint($value, 1);
1997         next if !defined($mountpoint);
1998
1999         # just to be sure: rootfs is /
2000         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
2001         $mountpoint->{mp} = sanitize_mountpoint($path);
2002
2003         $path = $mountpoint->{volume};
2004         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
2005
2006         &$func($key, $mountpoint);
2007     }
2008 }
2009
2010 sub foreach_mountpoint {
2011     my ($conf, $func) = @_;
2012
2013     foreach_mountpoint_full($conf, 0, $func);
2014 }
2015
2016 sub foreach_mountpoint_reverse {
2017     my ($conf, $func) = @_;
2018
2019     foreach_mountpoint_full($conf, 1, $func);
2020 }
2021
2022 sub check_ct_modify_config_perm {
2023     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2024
2025     return 1 if $authuser ne 'root@pam';
2026
2027     foreach my $opt (@$key_list) {
2028
2029         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2030             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2031         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2032             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2033         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2034             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2035         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2036                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2037             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2038         } else {
2039             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2040         }
2041     }
2042
2043     return 1;
2044 }
2045
2046 sub umount_all {
2047     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2048
2049     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2050     my $volid_list = get_vm_volumes($conf);
2051
2052     foreach_mountpoint_reverse($conf, sub {
2053         my ($ms, $mountpoint) = @_;
2054
2055         my $volid = $mountpoint->{volume};
2056         my $mount = $mountpoint->{mp};
2057
2058         return if !$volid || !$mount;
2059
2060         my $mount_path = "$rootdir/$mount";
2061         $mount_path =~ s!/+!/!g;
2062
2063         return if !PVE::ProcFSTools::is_mounted($mount_path);
2064
2065         eval {
2066             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2067         };
2068         if (my $err = $@) {
2069             if ($noerr) {
2070                 warn $err;
2071             } else {
2072                 die $err;
2073             }
2074         }
2075     });
2076 }
2077
2078 sub mount_all {
2079     my ($vmid, $storage_cfg, $conf) = @_;
2080
2081     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2082     File::Path::make_path($rootdir);
2083
2084     my $volid_list = get_vm_volumes($conf);
2085     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2086
2087     eval {
2088         foreach_mountpoint($conf, sub {
2089             my ($ms, $mountpoint) = @_;
2090
2091             my $volid = $mountpoint->{volume};
2092             my $mount = $mountpoint->{mp};
2093
2094             return if !$volid || !$mount;
2095
2096             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2097             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2098                 PVE::Storage::parse_volname($storage_cfg, $volid);
2099
2100             die "unable to mount base volume - internal error" if $isBase;
2101
2102             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2103         });
2104     };
2105     if (my $err = $@) {
2106         warn "mounting container failed - $err";
2107         umount_all($vmid, $storage_cfg, $conf, 1);
2108     }
2109
2110     return $rootdir;
2111 }
2112
2113
2114 sub mountpoint_mount_path {
2115     my ($mountpoint, $storage_cfg, $snapname) = @_;
2116
2117     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2118 }
2119
2120 my $check_mount_path = sub {
2121     my ($path) = @_;
2122     $path = File::Spec->canonpath($path);
2123     my $real = Cwd::realpath($path);
2124     if ($real ne $path) {
2125         die "mount path modified by symlink: $path != $real";
2126     }
2127 };
2128
2129 sub query_loopdev {
2130     my ($path) = @_;
2131     my $found;
2132     my $parser = sub {
2133         my $line = shift;
2134         if ($line =~ m@^(/dev/loop\d+):@) {
2135             $found = $1;
2136         }
2137     };
2138     my $cmd = ['losetup', '--associated', $path];
2139     PVE::Tools::run_command($cmd, outfunc => $parser);
2140     return $found;
2141 }
2142
2143 # use $rootdir = undef to just return the corresponding mount path
2144 sub mountpoint_mount {
2145     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2146
2147     my $volid = $mountpoint->{volume};
2148     my $mount = $mountpoint->{mp};
2149     my $type = $mountpoint->{type};
2150
2151     return if !$volid || !$mount;
2152
2153     my $mount_path;
2154
2155     if (defined($rootdir)) {
2156         $rootdir =~ s!/+$!!;
2157         $mount_path = "$rootdir/$mount";
2158         $mount_path =~ s!/+!/!g;
2159         &$check_mount_path($mount_path);
2160         File::Path::mkpath($mount_path);
2161     }
2162
2163     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2164
2165     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2166
2167     if ($storage) {
2168
2169         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2170         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2171
2172         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2173             PVE::Storage::parse_volname($storage_cfg, $volid);
2174
2175         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2176
2177         if ($format eq 'subvol') {
2178             if ($mount_path) {
2179                 if ($snapname) {
2180                     if ($scfg->{type} eq 'zfspool') {
2181                         my $path_arg = $path;
2182                         $path_arg =~ s!^/+!!;
2183                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2184                     } else {
2185                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2186                     }
2187                 } else {
2188                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2189                 }
2190             }
2191             return wantarray ? ($path, 0) : $path;
2192         } elsif ($format eq 'raw' || $format eq 'iso') {
2193             my $use_loopdev = 0;
2194             my @extra_opts;
2195             if ($scfg->{path}) {
2196                 push @extra_opts, '-o', 'loop';
2197                 $use_loopdev = 1;
2198             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2199                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2200                 # do nothing
2201             } else {
2202                 die "unsupported storage type '$scfg->{type}'\n";
2203             }
2204             if ($mount_path) {
2205                 if ($format eq 'iso') {
2206                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2207                 } elsif ($isBase || defined($snapname)) {
2208                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2209                 } else {
2210                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2211                 }
2212             }
2213             return wantarray ? ($path, $use_loopdev) : $path;
2214         } else {
2215             die "unsupported image format '$format'\n";
2216         }
2217     } elsif ($type eq 'device') {
2218         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2219         return wantarray ? ($volid, 0) : $volid;
2220     } elsif ($type eq 'bind' && -d $volid) {
2221         &$check_mount_path($volid);
2222         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2223         return wantarray ? ($volid, 0) : $volid;
2224     }
2225
2226     die "unsupported storage";
2227 }
2228
2229 sub get_vm_volumes {
2230     my ($conf, $excludes) = @_;
2231
2232     my $vollist = [];
2233
2234     foreach_mountpoint($conf, sub {
2235         my ($ms, $mountpoint) = @_;
2236
2237         return if $excludes && $ms eq $excludes;
2238
2239         my $volid = $mountpoint->{volume};
2240
2241         return if !$volid || $mountpoint->{type} ne 'volume';
2242
2243         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2244         return if !$sid;
2245
2246         push @$vollist, $volid;
2247     });
2248
2249     return $vollist;
2250 }
2251
2252 sub mkfs {
2253     my ($dev, $rootuid, $rootgid) = @_;
2254
2255     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2256                              '-E', "root_owner=$rootuid:$rootgid",
2257                              $dev]);
2258 }
2259
2260 sub format_disk {
2261     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2262
2263     if ($volid =~ m!^/dev/.+!) {
2264         mkfs($volid);
2265         return;
2266     }
2267
2268     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2269
2270     die "cannot format volume '$volid' with no storage\n" if !$storage;
2271
2272     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2273
2274     my $path = PVE::Storage::path($storage_cfg, $volid);
2275
2276     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2277         PVE::Storage::parse_volname($storage_cfg, $volid);
2278
2279     die "cannot format volume '$volid' (format == $format)\n"
2280         if $format ne 'raw';
2281
2282     mkfs($path, $rootuid, $rootgid);
2283 }
2284
2285 sub destroy_disks {
2286     my ($storecfg, $vollist) = @_;
2287
2288     foreach my $volid (@$vollist) {
2289         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2290         warn $@ if $@;
2291     }
2292 }
2293
2294 sub create_disks {
2295     my ($storecfg, $vmid, $settings, $conf) = @_;
2296
2297     my $vollist = [];
2298
2299     eval {
2300         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2301         my $chown_vollist = [];
2302
2303         foreach_mountpoint($settings, sub {
2304             my ($ms, $mountpoint) = @_;
2305
2306             my $volid = $mountpoint->{volume};
2307             my $mp = $mountpoint->{mp};
2308
2309             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2310
2311             return if !$storage;
2312
2313             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2314                 my ($storeid, $size_gb) = ($1, $2);
2315
2316                 my $size_kb = int(${size_gb}*1024) * 1024;
2317
2318                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2319                 # fixme: use better naming ct-$vmid-disk-X.raw?
2320
2321                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2322                     if ($size_kb > 0) {
2323                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2324                                                            undef, $size_kb);
2325                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2326                     } else {
2327                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2328                                                            undef, 0);
2329                         push @$chown_vollist, $volid;
2330                     }
2331                 } elsif ($scfg->{type} eq 'zfspool') {
2332
2333                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2334                                                        undef, $size_kb);
2335                     push @$chown_vollist, $volid;
2336                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2337
2338                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2339                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2340
2341                 } elsif ($scfg->{type} eq 'rbd') {
2342
2343                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2344                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2345                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2346                 } else {
2347                     die "unable to create containers on storage type '$scfg->{type}'\n";
2348                 }
2349                 push @$vollist, $volid;
2350                 $mountpoint->{volume} = $volid;
2351                 $mountpoint->{size} = $size_kb * 1024;
2352                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2353             } else {
2354                 # use specified/existing volid
2355             }
2356         });
2357
2358         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2359         foreach my $volid (@$chown_vollist) {
2360             my $path = PVE::Storage::path($storecfg, $volid, undef);
2361             chown($rootuid, $rootgid, $path);
2362         }
2363         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2364     };
2365     # free allocated images on error
2366     if (my $err = $@) {
2367         destroy_disks($storecfg, $vollist);
2368         die $err;
2369     }
2370     return $vollist;
2371 }
2372
2373 # bash completion helper
2374
2375 sub complete_os_templates {
2376     my ($cmdname, $pname, $cvalue) = @_;
2377
2378     my $cfg = PVE::Storage::config();
2379
2380     my $storeid;
2381
2382     if ($cvalue =~ m/^([^:]+):/) {
2383         $storeid = $1;
2384     }
2385
2386     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2387     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2388
2389     my $res = [];
2390     foreach my $id (keys %$data) {
2391         foreach my $item (@{$data->{$id}}) {
2392             push @$res, $item->{volid} if defined($item->{volid});
2393         }
2394     }
2395
2396     return $res;
2397 }
2398
2399 my $complete_ctid_full = sub {
2400     my ($running) = @_;
2401
2402     my $idlist = vmstatus();
2403
2404     my $active_hash = list_active_containers();
2405
2406     my $res = [];
2407
2408     foreach my $id (keys %$idlist) {
2409         my $d = $idlist->{$id};
2410         if (defined($running)) {
2411             next if $d->{template};
2412             next if $running && !$active_hash->{$id};
2413             next if !$running && $active_hash->{$id};
2414         }
2415         push @$res, $id;
2416
2417     }
2418     return $res;
2419 };
2420
2421 sub complete_ctid {
2422     return &$complete_ctid_full();
2423 }
2424
2425 sub complete_ctid_stopped {
2426     return &$complete_ctid_full(0);
2427 }
2428
2429 sub complete_ctid_running {
2430     return &$complete_ctid_full(1);
2431 }
2432
2433 sub parse_id_maps {
2434     my ($conf) = @_;
2435
2436     my $id_map = [];
2437     my $rootuid = 0;
2438     my $rootgid = 0;
2439
2440     my $lxc = $conf->{lxc};
2441     foreach my $entry (@$lxc) {
2442         my ($key, $value) = @$entry;
2443         next if $key ne 'lxc.id_map';
2444         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2445             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2446             push @$id_map, [$type, $ct, $host, $length];
2447             if ($ct == 0) {
2448                 $rootuid = $host if $type eq 'u';
2449                 $rootgid = $host if $type eq 'g';
2450             }
2451         } else {
2452             die "failed to parse id_map: $value\n";
2453         }
2454     }
2455
2456     if (!@$id_map && $conf->{unprivileged}) {
2457         # Should we read them from /etc/subuid?
2458         $id_map = [ ['u', '0', '100000', '65536'],
2459                     ['g', '0', '100000', '65536'] ];
2460         $rootuid = $rootgid = 100000;
2461     }
2462
2463     return ($id_map, $rootuid, $rootgid);
2464 }
2465
2466 sub userns_command {
2467     my ($id_map) = @_;
2468     if (@$id_map) {
2469         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2470     }
2471     return [];
2472 }
2473
2474 1;