src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  30
  31 my $rootfs_desc = {
  32     volume => {
  33         type => 'string',
  34         default_key => 1,
  35         format_description => 'volume',
  36         description => 'Volume, device or directory to mount into the container.',
  37     },
  38     backup => {
  39         type => 'boolean',
  40         format_description => '[1|0]',
  41         description => 'Whether to include the mountpoint in backups.',
  42         optional => 1,
  43     },
  44     size => {
  45         type => 'string',
  46         format => 'disk-size',
  47         format_description => 'DiskSize',
  48         description => 'Volume size (read only value).',
  49         optional => 1,
  50     },
  51 };
  52
  53 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  54     type => 'string', format => $rootfs_desc,
  55     description => "Use volume as container root.",
  56     optional => 1,
  57 });
  58
  59 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  60     description => "The name of the snapshot.",
  61     type => 'string', format => 'pve-configid',
  62     maxLength => 40,
  63 });
  64
  65 my $confdesc = {
  66     lock => {
  67         optional => 1,
  68         type => 'string',
  69         description => "Lock/unlock the VM.",
  70         enum => [qw(migrate backup snapshot rollback)],
  71     },
  72     onboot => {
  73         optional => 1,
  74         type => 'boolean',
  75         description => "Specifies whether a VM will be started during system bootup.",
  76         default => 0,
  77     },
  78     startup => get_standard_option('pve-startup-order'),
  79     template => {
  80         optional => 1,
  81         type => 'boolean',
  82         description => "Enable/disable Template.",
  83         default => 0,
  84     },
  85     arch => {
  86         optional => 1,
  87         type => 'string',
  88         enum => ['amd64', 'i386'],
  89         description => "OS architecture type.",
  90         default => 'amd64',
  91     },
  92     ostype => {
  93         optional => 1,
  94         type => 'string',
  95         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  96         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  97     },
  98     console => {
  99         optional => 1,
 100         type => 'boolean',
 101         description => "Attach a console device (/dev/console) to the container.",
 102         default => 1,
 103     },
 104     tty => {
 105         optional => 1,
 106         type => 'integer',
 107         description => "Specify the number of tty available to the container",
 108         minimum => 0,
 109         maximum => 6,
 110         default => 2,
 111     },
 112     cpulimit => {
 113         optional => 1,
 114         type => 'number',
 115         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 116         minimum => 0,
 117         maximum => 128,
 118         default => 0,
 119     },
 120     cpuunits => {
 121         optional => 1,
 122         type => 'integer',
 123         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 124         minimum => 0,
 125         maximum => 500000,
 126         default => 1024,
 127     },
 128     memory => {
 129         optional => 1,
 130         type => 'integer',
 131         description => "Amount of RAM for the VM in MB.",
 132         minimum => 16,
 133         default => 512,
 134     },
 135     swap => {
 136         optional => 1,
 137         type => 'integer',
 138         description => "Amount of SWAP for the VM in MB.",
 139         minimum => 0,
 140         default => 512,
 141     },
 142     hostname => {
 143         optional => 1,
 144         description => "Set a host name for the container.",
 145         type => 'string', format => 'dns-name',
 146         maxLength => 255,
 147     },
 148     description => {
 149         optional => 1,
 150         type => 'string',
 151         description => "Container description. Only used on the configuration web interface.",
 152     },
 153     searchdomain => {
 154         optional => 1,
 155         type => 'string', format => 'dns-name-list',
 156         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 157     },
 158     nameserver => {
 159         optional => 1,
 160         type => 'string', format => 'address-list',
 161         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 162     },
 163     rootfs => get_standard_option('pve-ct-rootfs'),
 164     parent => {
 165         optional => 1,
 166         type => 'string', format => 'pve-configid',
 167         maxLength => 40,
 168         description => "Parent snapshot name. This is used internally, and should not be modified.",
 169     },
 170     snaptime => {
 171         optional => 1,
 172         description => "Timestamp for snapshots.",
 173         type => 'integer',
 174         minimum => 0,
 175     },
 176     cmode => {
 177         optional => 1,
 178         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 179         type => 'string',
 180         enum => ['shell', 'console', 'tty'],
 181         default => 'tty',
 182     },
 183     protection => {
 184         optional => 1,
 185         type => 'boolean',
 186         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 187         default => 0,
 188     },
 189 };
 190
 191 my $valid_lxc_conf_keys = {
 192     'lxc.include' => 1,
 193     'lxc.arch' => 1,
 194     'lxc.utsname' => 1,
 195     'lxc.haltsignal' => 1,
 196     'lxc.rebootsignal' => 1,
 197     'lxc.stopsignal' => 1,
 198     'lxc.init_cmd' => 1,
 199     'lxc.network.type' => 1,
 200     'lxc.network.flags' => 1,
 201     'lxc.network.link' => 1,
 202     'lxc.network.mtu' => 1,
 203     'lxc.network.name' => 1,
 204     'lxc.network.hwaddr' => 1,
 205     'lxc.network.ipv4' => 1,
 206     'lxc.network.ipv4.gateway' => 1,
 207     'lxc.network.ipv6' => 1,
 208     'lxc.network.ipv6.gateway' => 1,
 209     'lxc.network.script.up' => 1,
 210     'lxc.network.script.down' => 1,
 211     'lxc.pts' => 1,
 212     'lxc.console.logfile' => 1,
 213     'lxc.console' => 1,
 214     'lxc.tty' => 1,
 215     'lxc.devttydir' => 1,
 216     'lxc.hook.autodev' => 1,
 217     'lxc.autodev' => 1,
 218     'lxc.kmsg' => 1,
 219     'lxc.mount' => 1,
 220     'lxc.mount.entry' => 1,
 221     'lxc.mount.auto' => 1,
 222     'lxc.rootfs' => 1,
 223     'lxc.rootfs.mount' => 1,
 224     'lxc.rootfs.options' => 1,
 225     # lxc.cgroup.*
 226     'lxc.cap.drop' => 1,
 227     'lxc.cap.keep' => 1,
 228     'lxc.aa_profile' => 1,
 229     'lxc.aa_allow_incomplete' => 1,
 230     'lxc.se_context' => 1,
 231     'lxc.seccomp' => 1,
 232     'lxc.id_map' => 1,
 233     'lxc.hook.pre-start' => 1,
 234     'lxc.hook.pre-mount' => 1,
 235     'lxc.hook.mount' => 1,
 236     'lxc.hook.start' => 1,
 237     'lxc.hook.stop' => 1,
 238     'lxc.hook.post-stop' => 1,
 239     'lxc.hook.clone' => 1,
 240     'lxc.hook.destroy' => 1,
 241     'lxc.loglevel' => 1,
 242     'lxc.logfile' => 1,
 243     'lxc.start.auto' => 1,
 244     'lxc.start.delay' => 1,
 245     'lxc.start.order' => 1,
 246     'lxc.group' => 1,
 247     'lxc.environment' => 1,
 248     'lxc.' => 1,
 249     'lxc.' => 1,
 250     'lxc.' => 1,
 251     'lxc.' => 1,
 252 };
 253
 254 my $netconf_desc = {
 255     type => {
 256         type => 'string',
 257         optional => 1,
 258         description => "Network interface type.",
 259         enum => [qw(veth)],
 260     },
 261     name => {
 262         type => 'string',
 263         format_description => 'String',
 264         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 265         pattern => '[-_.\w\d]+',
 266     },
 267     bridge => {
 268         type => 'string',
 269         format_description => 'vmbr<Number>',
 270         description => 'Bridge to attach the network device to.',
 271         pattern => '[-_.\w\d]+',
 272         optional => 1,
 273     },
 274     hwaddr => {
 275         type => 'string',
 276         format_description => 'MAC',
 277         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 278         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 279         optional => 1,
 280     },
 281     mtu => {
 282         type => 'integer',
 283         format_description => 'Number',
 284         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 285         minimum => 64, # minimum ethernet frame is 64 bytes
 286         optional => 1,
 287     },
 288     ip => {
 289         type => 'string',
 290         format => 'pve-ipv4-config',
 291         format_description => 'IPv4Format/CIDR',
 292         description => 'IPv4 address in CIDR format.',
 293         optional => 1,
 294     },
 295     gw => {
 296         type => 'string',
 297         format => 'ipv4',
 298         format_description => 'GatewayIPv4',
 299         description => 'Default gateway for IPv4 traffic.',
 300         optional => 1,
 301     },
 302     ip6 => {
 303         type => 'string',
 304         format => 'pve-ipv6-config',
 305         format_description => 'IPv6Format/CIDR',
 306         description => 'IPv6 address in CIDR format.',
 307         optional => 1,
 308     },
 309     gw6 => {
 310         type => 'string',
 311         format => 'ipv6',
 312         format_description => 'GatewayIPv6',
 313         description => 'Default gateway for IPv6 traffic.',
 314         optional => 1,
 315     },
 316     firewall => {
 317         type => 'boolean',
 318         format_description => '[1|0]',
 319         description => "Controls whether this interface's firewall rules should be used.",
 320         optional => 1,
 321     },
 322     tag => {
 323         type => 'integer',
 324         format_description => 'VlanNo',
 325         minimum => '2',
 326         maximum => '4094',
 327         description => "VLAN tag foro this interface.",
 328         optional => 1,
 329     },
 330 };
 331 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 332
 333 my $MAX_LXC_NETWORKS = 10;
 334 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 335     $confdesc->{"net$i"} = {
 336         optional => 1,
 337         type => 'string', format => $netconf_desc,
 338         description => "Specifies network interfaces for the container.",
 339     };
 340 }
 341
 342 my $mp_desc = {
 343     %$rootfs_desc,
 344     mp => {
 345         type => 'string',
 346         format_description => 'Path',
 347         description => 'Path to the mountpoint as seen from inside the container.',
 348         optional => 1,
 349     },
 350 };
 351 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 352
 353 my $unuseddesc = {
 354     optional => 1,
 355     type => 'string', format => 'pve-volume-id',
 356     description => "Reference to unused volumes.",
 357 };
 358
 359 my $MAX_MOUNT_POINTS = 10;
 360 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 361     $confdesc->{"mp$i"} = {
 362         optional => 1,
 363         type => 'string', format => $mp_desc,
 364         description => "Use volume as container mount point (experimental feature).",
 365         optional => 1,
 366     };
 367 }
 368
 369 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 370 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 371     $confdesc->{"unused$i"} = $unuseddesc;
 372 }
 373
 374 sub write_pct_config {
 375     my ($filename, $conf) = @_;
 376
 377     delete $conf->{snapstate}; # just to be sure
 378
 379     my $generate_raw_config = sub {
 380         my ($conf) = @_;
 381
 382         my $raw = '';
 383
 384         # add description as comment to top of file
 385         my $descr = $conf->{description} || '';
 386         foreach my $cl (split(/\n/, $descr)) {
 387             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 388         }
 389
 390         foreach my $key (sort keys %$conf) {
 391             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 392                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 393             my $value = $conf->{$key};
 394             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 395             $raw .= "$key: $value\n";
 396         }
 397
 398         if (my $lxcconf = $conf->{lxc}) {
 399             foreach my $entry (@$lxcconf) {
 400                 my ($k, $v) = @$entry;
 401                 $raw .= "$k: $v\n";
 402             }
 403         }
 404
 405         return $raw;
 406     };
 407
 408     my $raw = &$generate_raw_config($conf);
 409
 410     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 411         $raw .= "\n[$snapname]\n";
 412         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 413     }
 414
 415     return $raw;
 416 }
 417
 418 sub check_type {
 419     my ($key, $value) = @_;
 420
 421     die "unknown setting '$key'\n" if !$confdesc->{$key};
 422
 423     my $type = $confdesc->{$key}->{type};
 424
 425     if (!defined($value)) {
 426         die "got undefined value\n";
 427     }
 428
 429     if ($value =~ m/[\n\r]/) {
 430         die "property contains a line feed\n";
 431     }
 432
 433     if ($type eq 'boolean') {
 434         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 435         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 436         die "type check ('boolean') failed - got '$value'\n";
 437     } elsif ($type eq 'integer') {
 438         return int($1) if $value =~ m/^(\d+)$/;
 439         die "type check ('integer') failed - got '$value'\n";
 440     } elsif ($type eq 'number') {
 441         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 442         die "type check ('number') failed - got '$value'\n";
 443     } elsif ($type eq 'string') {
 444         if (my $fmt = $confdesc->{$key}->{format}) {
 445             PVE::JSONSchema::check_format($fmt, $value);
 446             return $value;
 447         }
 448         return $value;
 449     } else {
 450         die "internal error"
 451     }
 452 }
 453
 454 sub parse_pct_config {
 455     my ($filename, $raw) = @_;
 456
 457     return undef if !defined($raw);
 458
 459     my $res = {
 460         digest => Digest::SHA::sha1_hex($raw),
 461         snapshots => {},
 462     };
 463
 464     $filename =~ m|/lxc/(\d+).conf$|
 465         || die "got strange filename '$filename'";
 466
 467     my $vmid = $1;
 468
 469     my $conf = $res;
 470     my $descr = '';
 471     my $section = '';
 472
 473     my @lines = split(/\n/, $raw);
 474     foreach my $line (@lines) {
 475         next if $line =~ m/^\s*$/;
 476
 477         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 478             $section = $1;
 479             $conf->{description} = $descr if $descr;
 480             $descr = '';
 481             $conf = $res->{snapshots}->{$section} = {};
 482             next;
 483         }
 484
 485         if ($line =~ m/^\#(.*)\s*$/) {
 486             $descr .= PVE::Tools::decode_text($1) . "\n";
 487             next;
 488         }
 489
 490         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 491             my $key = $1;
 492             my $value = $3;
 493             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 494                 push @{$conf->{lxc}}, [$key, $value];
 495             } else {
 496                 warn "vm $vmid - unable to parse config: $line\n";
 497             }
 498         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 499             $descr .= PVE::Tools::decode_text($2);
 500         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 501             $conf->{snapstate} = $1;
 502         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 503             my $key = $1;
 504             my $value = $2;
 505             eval { $value = check_type($key, $value); };
 506             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 507             $conf->{$key} = $value;
 508         } else {
 509             warn "vm $vmid - unable to parse config: $line\n";
 510         }
 511     }
 512
 513     $conf->{description} = $descr if $descr;
 514
 515     delete $res->{snapstate}; # just to be sure
 516
 517     return $res;
 518 }
 519
 520 sub config_list {
 521     my $vmlist = PVE::Cluster::get_vmlist();
 522     my $res = {};
 523     return $res if !$vmlist || !$vmlist->{ids};
 524     my $ids = $vmlist->{ids};
 525
 526     foreach my $vmid (keys %$ids) {
 527         next if !$vmid; # skip CT0
 528         my $d = $ids->{$vmid};
 529         next if !$d->{node} || $d->{node} ne $nodename;
 530         next if !$d->{type} || $d->{type} ne 'lxc';
 531         $res->{$vmid}->{type} = 'lxc';
 532     }
 533     return $res;
 534 }
 535
 536 sub cfs_config_path {
 537     my ($vmid, $node) = @_;
 538
 539     $node = $nodename if !$node;
 540     return "nodes/$node/lxc/$vmid.conf";
 541 }
 542
 543 sub config_file {
 544     my ($vmid, $node) = @_;
 545
 546     my $cfspath = cfs_config_path($vmid, $node);
 547     return "/etc/pve/$cfspath";
 548 }
 549
 550 sub load_config {
 551     my ($vmid, $node) = @_;
 552
 553     $node = $nodename if !$node;
 554     my $cfspath = cfs_config_path($vmid, $node);
 555
 556     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 557     die "container $vmid does not exists\n" if !defined($conf);
 558
 559     return $conf;
 560 }
 561
 562 sub create_config {
 563     my ($vmid, $conf) = @_;
 564
 565     my $dir = "/etc/pve/nodes/$nodename/lxc";
 566     mkdir $dir;
 567
 568     write_config($vmid, $conf);
 569 }
 570
 571 sub destroy_config {
 572     my ($vmid) = @_;
 573
 574     unlink config_file($vmid, $nodename);
 575 }
 576
 577 sub write_config {
 578     my ($vmid, $conf) = @_;
 579
 580     my $cfspath = cfs_config_path($vmid);
 581
 582     PVE::Cluster::cfs_write_file($cfspath, $conf);
 583 }
 584
 585 # flock: we use one file handle per process, so lock file
 586 # can be called multiple times and succeeds for the same process.
 587
 588 my $lock_handles =  {};
 589 my $lockdir = "/run/lock/lxc";
 590
 591 sub lock_filename {
 592     my ($vmid) = @_;
 593
 594     return "$lockdir/pve-config-${vmid}.lock";
 595 }
 596
 597 sub lock_aquire {
 598     my ($vmid, $timeout) = @_;
 599
 600     $timeout = 10 if !$timeout;
 601     my $mode = LOCK_EX;
 602
 603     my $filename = lock_filename($vmid);
 604
 605     mkdir $lockdir if !-d $lockdir;
 606
 607     my $lock_func = sub {
 608         if (!$lock_handles->{$$}->{$filename}) {
 609             my $fh = new IO::File(">>$filename") ||
 610                 die "can't open file - $!\n";
 611             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 612         }
 613
 614         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 615             print STDERR "trying to aquire lock...";
 616             my $success;
 617             while(1) {
 618                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 619                 # try again on EINTR (see bug #273)
 620                 if ($success || ($! != EINTR)) {
 621                     last;
 622                 }
 623             }
 624             if (!$success) {
 625                 print STDERR " failed\n";
 626                 die "can't aquire lock - $!\n";
 627             }
 628
 629             print STDERR " OK\n";
 630         }
 631
 632         $lock_handles->{$$}->{$filename}->{refcount}++;
 633     };
 634
 635     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 636     my $err = $@;
 637     if ($err) {
 638         die "can't lock file '$filename' - $err";
 639     }
 640 }
 641
 642 sub lock_release {
 643     my ($vmid) = @_;
 644
 645     my $filename = lock_filename($vmid);
 646
 647     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 648         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 649         if ($refcount <= 0) {
 650             $lock_handles->{$$}->{$filename} = undef;
 651             close ($fh);
 652         }
 653     }
 654 }
 655
 656 sub lock_container {
 657     my ($vmid, $timeout, $code, @param) = @_;
 658
 659     my $res;
 660
 661     lock_aquire($vmid, $timeout);
 662     eval { $res = &$code(@param) };
 663     my $err = $@;
 664     lock_release($vmid);
 665
 666     die $err if $err;
 667
 668     return $res;
 669 }
 670
 671 sub option_exists {
 672     my ($name) = @_;
 673
 674     return defined($confdesc->{$name});
 675 }
 676
 677 # add JSON properties for create and set function
 678 sub json_config_properties {
 679     my $prop = shift;
 680
 681     foreach my $opt (keys %$confdesc) {
 682         next if $opt eq 'parent' || $opt eq 'snaptime';
 683         next if $prop->{$opt};
 684         $prop->{$opt} = $confdesc->{$opt};
 685     }
 686
 687     return $prop;
 688 }
 689
 690 sub json_config_properties_no_rootfs {
 691     my $prop = shift;
 692
 693     foreach my $opt (keys %$confdesc) {
 694         next if $prop->{$opt};
 695         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 696         $prop->{$opt} = $confdesc->{$opt};
 697     }
 698
 699     return $prop;
 700 }
 701
 702 # container status helpers
 703
 704 sub list_active_containers {
 705
 706     my $filename = "/proc/net/unix";
 707
 708     # similar test is used by lcxcontainers.c: list_active_containers
 709     my $res = {};
 710
 711     my $fh = IO::File->new ($filename, "r");
 712     return $res if !$fh;
 713
 714     while (defined(my $line = <$fh>)) {
 715         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 716             my $path = $1;
 717             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 718                 $res->{$1} = 1;
 719             }
 720         }
 721     }
 722
 723     close($fh);
 724
 725     return $res;
 726 }
 727
 728 # warning: this is slow
 729 sub check_running {
 730     my ($vmid) = @_;
 731
 732     my $active_hash = list_active_containers();
 733
 734     return 1 if defined($active_hash->{$vmid});
 735
 736     return undef;
 737 }
 738
 739 sub get_container_disk_usage {
 740     my ($vmid) = @_;
 741
 742     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 743
 744     my $res = {
 745         total => 0,
 746         used => 0,
 747         avail => 0,
 748     };
 749
 750     my $parser = sub {
 751         my $line = shift;
 752         if (my ($fsid, $total, $used, $avail) = $line =~
 753             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 754             $res = {
 755                 total => $total,
 756                 used => $used,
 757                 avail => $avail,
 758             };
 759         }
 760     };
 761     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 762     warn $@ if $@;
 763
 764     return $res;
 765 }
 766
 767 my $last_proc_vmid_stat;
 768
 769 my $parse_cpuacct_stat = sub {
 770     my ($vmid) = @_;
 771
 772     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 773
 774     my $stat = {};
 775
 776     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 777
 778         $stat->{utime} = $1;
 779         $stat->{stime} = $2;
 780
 781     }
 782
 783     return $stat;
 784 };
 785
 786 sub vmstatus {
 787     my ($opt_vmid) = @_;
 788
 789     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 790
 791     my $active_hash = list_active_containers();
 792
 793     my $cpucount = $cpuinfo->{cpus} || 1;
 794
 795     my $cdtime = gettimeofday;
 796
 797     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 798
 799     foreach my $vmid (keys %$list) {
 800         my $d = $list->{$vmid};
 801
 802         my $running = defined($active_hash->{$vmid});
 803
 804         $d->{status} = $running ? 'running' : 'stopped';
 805
 806         my $cfspath = cfs_config_path($vmid);
 807         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 808
 809         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 810         $d->{name} =~ s/[\s]//g;
 811
 812         $d->{cpus} = $conf->{cpulimit} // 0;
 813
 814         if ($running) {
 815             my $res = get_container_disk_usage($vmid);
 816             $d->{disk} = $res->{used};
 817             $d->{maxdisk} = $res->{total};
 818         } else {
 819             $d->{disk} = 0;
 820             # use 4GB by default ??
 821             if (my $rootfs = $conf->{rootfs}) {
 822                 my $rootinfo = parse_ct_mountpoint($rootfs);
 823                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 824             } else {
 825                 $d->{maxdisk} = 4*1024*1024*1024;
 826             }
 827         }
 828
 829         $d->{mem} = 0;
 830         $d->{swap} = 0;
 831         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 832         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 833
 834         $d->{uptime} = 0;
 835         $d->{cpu} = 0;
 836
 837         $d->{netout} = 0;
 838         $d->{netin} = 0;
 839
 840         $d->{diskread} = 0;
 841         $d->{diskwrite} = 0;
 842
 843         $d->{template} = is_template($conf);
 844     }
 845
 846     foreach my $vmid (keys %$list) {
 847         my $d = $list->{$vmid};
 848         next if $d->{status} ne 'running';
 849
 850         my $pid = find_lxc_pid($vmid);
 851         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 852         $d->{uptime} = time - $ctime; # the method lxcfs uses
 853
 854         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 855         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 856
 857         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 858         my @bytes = split(/\n/, $blkio_bytes);
 859         foreach my $byte (@bytes) {
 860             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 861                 $d->{diskread} = $2 if $key eq 'Read';
 862                 $d->{diskwrite} = $2 if $key eq 'Write';
 863             }
 864         }
 865
 866         my $pstat = &$parse_cpuacct_stat($vmid);
 867
 868         my $used = $pstat->{utime} + $pstat->{stime};
 869
 870         my $old = $last_proc_vmid_stat->{$vmid};
 871         if (!$old) {
 872             $last_proc_vmid_stat->{$vmid} = {
 873                 time => $cdtime,
 874                 used => $used,
 875                 cpu => 0,
 876             };
 877             next;
 878         }
 879
 880         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 881
 882         if ($dtime > 1000) {
 883             my $dutime = $used -  $old->{used};
 884
 885             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 886             $last_proc_vmid_stat->{$vmid} = {
 887                 time => $cdtime,
 888                 used => $used,
 889                 cpu => $d->{cpu},
 890             };
 891         } else {
 892             $d->{cpu} = $old->{cpu};
 893         }
 894     }
 895
 896     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 897
 898     foreach my $dev (keys %$netdev) {
 899         next if $dev !~ m/^veth([1-9]\d*)i/;
 900         my $vmid = $1;
 901         my $d = $list->{$vmid};
 902
 903         next if !$d;
 904
 905         $d->{netout} += $netdev->{$dev}->{receive};
 906         $d->{netin} += $netdev->{$dev}->{transmit};
 907
 908     }
 909
 910     return $list;
 911 }
 912
 913 sub parse_ct_mountpoint {
 914     my ($data, $noerr) = @_;
 915
 916     $data //= '';
 917
 918     my $res;
 919     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 920     if ($@) {
 921         return undef if $noerr;
 922         die $@;
 923     }
 924
 925     if (defined(my $size = $res->{size})) {
 926         $size = PVE::JSONSchema::parse_size($size);
 927         if (!defined($size)) {
 928             return undef if $noerr;
 929             die "invalid size: $size\n";
 930         }
 931         $res->{size} = $size;
 932     }
 933
 934     return $res;
 935 }
 936
 937 sub print_ct_mountpoint {
 938     my ($info, $nomp) = @_;
 939     my $skip = $nomp ? ['mp'] : [];
 940     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 941 }
 942
 943 sub print_lxc_network {
 944     my $net = shift;
 945     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 946 }
 947
 948 sub parse_lxc_network {
 949     my ($data) = @_;
 950
 951     my $res = {};
 952
 953     return $res if !$data;
 954
 955     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 956
 957     $res->{type} = 'veth';
 958     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 959
 960     return $res;
 961 }
 962
 963 sub read_cgroup_value {
 964     my ($group, $vmid, $name, $full) = @_;
 965
 966     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 967
 968     return PVE::Tools::file_get_contents($path) if $full;
 969
 970     return PVE::Tools::file_read_firstline($path);
 971 }
 972
 973 sub write_cgroup_value {
 974    my ($group, $vmid, $name, $value) = @_;
 975
 976    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 977    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 978
 979 }
 980
 981 sub find_lxc_console_pids {
 982
 983     my $res = {};
 984
 985     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 986         my ($pid) = @_;
 987
 988         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 989         return if !$cmdline;
 990
 991         my @args = split(/\0/, $cmdline);
 992
 993         # serach for lxc-console -n <vmid>
 994         return if scalar(@args) != 3;
 995         return if $args[1] ne '-n';
 996         return if $args[2] !~ m/^\d+$/;
 997         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 998
 999         my $vmid = $args[2];
1000
1001         push @{$res->{$vmid}}, $pid;
1002     });
1003
1004     return $res;
1005 }
1006
1007 sub find_lxc_pid {
1008     my ($vmid) = @_;
1009
1010     my $pid = undef;
1011     my $parser = sub {
1012         my $line = shift;
1013         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1014     };
1015     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1016
1017     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1018
1019     return $pid;
1020 }
1021
1022 # Note: we cannot use Net:IP, because that only allows strict
1023 # CIDR networks
1024 sub parse_ipv4_cidr {
1025     my ($cidr, $noerr) = @_;
1026
1027     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1028         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1029     }
1030
1031     return undef if $noerr;
1032
1033     die "unable to parse ipv4 address/mask\n";
1034 }
1035
1036 sub check_lock {
1037     my ($conf) = @_;
1038
1039     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1040 }
1041
1042 sub check_protection {
1043     my ($vm_conf, $err_msg) = @_;
1044
1045     if ($vm_conf->{protection}) {
1046         die "$err_msg - protection mode enabled\n";
1047     }
1048 }
1049
1050 sub update_lxc_config {
1051     my ($storage_cfg, $vmid, $conf) = @_;
1052
1053     my $dir = "/var/lib/lxc/$vmid";
1054
1055     if ($conf->{template}) {
1056
1057         unlink "$dir/config";
1058
1059         return;
1060     }
1061
1062     my $raw = '';
1063
1064     die "missing 'arch' - internal error" if !$conf->{arch};
1065     $raw .= "lxc.arch = $conf->{arch}\n";
1066
1067     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1068     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1069         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1070     } else {
1071         die "implement me";
1072     }
1073
1074     if (!has_dev_console($conf)) {
1075         $raw .= "lxc.console = none\n";
1076         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1077     }
1078
1079     my $ttycount = get_tty_count($conf);
1080     $raw .= "lxc.tty = $ttycount\n";
1081
1082     # some init scripts expects a linux terminal (turnkey).
1083     $raw .= "lxc.environment = TERM=linux\n";
1084
1085     my $utsname = $conf->{hostname} || "CT$vmid";
1086     $raw .= "lxc.utsname = $utsname\n";
1087
1088     my $memory = $conf->{memory} || 512;
1089     my $swap = $conf->{swap} // 0;
1090
1091     my $lxcmem = int($memory*1024*1024);
1092     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1093
1094     my $lxcswap = int(($memory + $swap)*1024*1024);
1095     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1096
1097     if (my $cpulimit = $conf->{cpulimit}) {
1098         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1099         my $value = int(100000*$cpulimit);
1100         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1101     }
1102
1103     my $shares = $conf->{cpuunits} || 1024;
1104     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1105
1106     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1107     $mountpoint->{mp} = '/';
1108
1109     my ($path, $use_loopdev) = mountpoint_mount_path($mountpoint, $storage_cfg);
1110     $path = "loop:$path" if $use_loopdev;
1111
1112     $raw .= "lxc.rootfs = $path\n";
1113
1114     my $netcount = 0;
1115     foreach my $k (keys %$conf) {
1116         next if $k !~ m/^net(\d+)$/;
1117         my $ind = $1;
1118         my $d = parse_lxc_network($conf->{$k});
1119         $netcount++;
1120         $raw .= "lxc.network.type = veth\n";
1121         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1122         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1123         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1124         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1125     }
1126
1127     if (my $lxcconf = $conf->{lxc}) {
1128         foreach my $entry (@$lxcconf) {
1129             my ($k, $v) = @$entry;
1130             $netcount++ if $k eq 'lxc.network.type';
1131             $raw .= "$k = $v\n";
1132         }
1133     }
1134
1135     $raw .= "lxc.network.type = empty\n" if !$netcount;
1136
1137     File::Path::mkpath("$dir/rootfs");
1138
1139     PVE::Tools::file_set_contents("$dir/config", $raw);
1140 }
1141
1142 # verify and cleanup nameserver list (replace \0 with ' ')
1143 sub verify_nameserver_list {
1144     my ($nameserver_list) = @_;
1145
1146     my @list = ();
1147     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1148         PVE::JSONSchema::pve_verify_ip($server);
1149         push @list, $server;
1150     }
1151
1152     return join(' ', @list);
1153 }
1154
1155 sub verify_searchdomain_list {
1156     my ($searchdomain_list) = @_;
1157
1158     my @list = ();
1159     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1160         # todo: should we add checks for valid dns domains?
1161         push @list, $server;
1162     }
1163
1164     return join(' ', @list);
1165 }
1166
1167 sub add_unused_volume {
1168     my ($config, $volid) = @_;
1169
1170     # skip bind mounts and block devices
1171     return if $volid =~ m|^/|;
1172
1173     my $key;
1174     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1175         my $test = "unused$ind";
1176         if (my $vid = $config->{$test}) {
1177             return if $vid eq $volid; # do not add duplicates
1178         } else {
1179             $key = $test;
1180         }
1181     }
1182
1183     die "To many unused volume - please delete them first.\n" if !$key;
1184
1185     $config->{$key} = $volid;
1186
1187     return $key;
1188 }
1189
1190 sub update_pct_config {
1191     my ($vmid, $conf, $running, $param, $delete) = @_;
1192
1193     my @nohotplug;
1194
1195     my $new_disks = 0;
1196     my @deleted_volumes;
1197
1198     my $rootdir;
1199     if ($running) {
1200         my $pid = find_lxc_pid($vmid);
1201         $rootdir = "/proc/$pid/root";
1202     }
1203
1204     my $hotplug_error = sub {
1205         if ($running) {
1206             push @nohotplug, @_;
1207             return 1;
1208         } else {
1209             return 0;
1210         }
1211     };
1212
1213     if (defined($delete)) {
1214         foreach my $opt (@$delete) {
1215             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1216                 die "unable to delete required option '$opt'\n";
1217             } elsif ($opt eq 'swap') {
1218                 delete $conf->{$opt};
1219                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1220             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1221                 delete $conf->{$opt};
1222             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1223                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1224                 next if $hotplug_error->($opt);
1225                 delete $conf->{$opt};
1226             } elsif ($opt =~ m/^net(\d)$/) {
1227                 delete $conf->{$opt};
1228                 next if !$running;
1229                 my $netid = $1;
1230                 PVE::Network::veth_delete("veth${vmid}i$netid");
1231             } elsif ($opt eq 'protection') {
1232                 delete $conf->{$opt};
1233             } elsif ($opt =~ m/^unused(\d+)$/) {
1234                 next if $hotplug_error->($opt);
1235                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1236                 push @deleted_volumes, $conf->{$opt};
1237                 delete $conf->{$opt};
1238             } elsif ($opt =~ m/^mp(\d+)$/) {
1239                 next if $hotplug_error->($opt);
1240                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1241                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1242                 add_unused_volume($conf, $mountpoint->{volume});
1243                 delete $conf->{$opt};
1244             } else {
1245                 die "implement me"
1246             }
1247             write_config($vmid, $conf) if $running;
1248         }
1249     }
1250
1251     # There's no separate swap size to configure, there's memory and "total"
1252     # memory (iow. memory+swap). This means we have to change them together.
1253     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1254     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1255     if (defined($wanted_memory) || defined($wanted_swap)) {
1256
1257         $wanted_memory //= ($conf->{memory} || 512);
1258         $wanted_swap //=  ($conf->{swap} || 0);
1259
1260         my $total = $wanted_memory + $wanted_swap;
1261         if ($running) {
1262             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1263             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1264         }
1265         $conf->{memory} = $wanted_memory;
1266         $conf->{swap} = $wanted_swap;
1267
1268         write_config($vmid, $conf) if $running;
1269     }
1270
1271     foreach my $opt (keys %$param) {
1272         my $value = $param->{$opt};
1273         if ($opt eq 'hostname') {
1274             $conf->{$opt} = $value;
1275         } elsif ($opt eq 'onboot') {
1276             $conf->{$opt} = $value ? 1 : 0;
1277         } elsif ($opt eq 'startup') {
1278             $conf->{$opt} = $value;
1279         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1280             next if $hotplug_error->($opt);
1281             $conf->{$opt} = $value;
1282         } elsif ($opt eq 'nameserver') {
1283             next if $hotplug_error->($opt);
1284             my $list = verify_nameserver_list($value);
1285             $conf->{$opt} = $list;
1286         } elsif ($opt eq 'searchdomain') {
1287             next if $hotplug_error->($opt);
1288             my $list = verify_searchdomain_list($value);
1289             $conf->{$opt} = $list;
1290         } elsif ($opt eq 'cpulimit') {
1291             next if $hotplug_error->($opt); # FIXME: hotplug
1292             $conf->{$opt} = $value;
1293         } elsif ($opt eq 'cpuunits') {
1294             $conf->{$opt} = $value;
1295             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1296         } elsif ($opt eq 'description') {
1297             $conf->{$opt} = PVE::Tools::encode_text($value);
1298         } elsif ($opt =~ m/^net(\d+)$/) {
1299             my $netid = $1;
1300             my $net = parse_lxc_network($value);
1301             if (!$running) {
1302                 $conf->{$opt} = print_lxc_network($net);
1303             } else {
1304                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1305             }
1306         } elsif ($opt eq 'protection') {
1307             $conf->{$opt} = $value ? 1 : 0;
1308         } elsif ($opt =~ m/^mp(\d+)$/) {
1309             next if $hotplug_error->($opt);
1310             check_protection($conf, "can't update CT $vmid drive '$opt'");
1311             $conf->{$opt} = $value;
1312             $new_disks = 1;
1313         } elsif ($opt eq 'rootfs') {
1314             check_protection($conf, "can't update CT $vmid drive '$opt'");
1315             die "implement me: $opt";
1316         } else {
1317             die "implement me: $opt";
1318         }
1319         write_config($vmid, $conf) if $running;
1320     }
1321
1322     if (@deleted_volumes) {
1323         my $storage_cfg = PVE::Storage::config();
1324         foreach my $volume (@deleted_volumes) {
1325             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1326         }
1327     }
1328
1329     if ($new_disks) {
1330         my $storage_cfg = PVE::Storage::config();
1331         create_disks($storage_cfg, $vmid, $conf, $conf);
1332     }
1333
1334     # This should be the last thing we do here
1335     if ($running && scalar(@nohotplug)) {
1336         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1337     }
1338 }
1339
1340 sub has_dev_console {
1341     my ($conf) = @_;
1342
1343     return !(defined($conf->{console}) && !$conf->{console});
1344 }
1345
1346 sub get_tty_count {
1347     my ($conf) = @_;
1348
1349     return $conf->{tty} // $confdesc->{tty}->{default};
1350 }
1351
1352 sub get_cmode {
1353     my ($conf) = @_;
1354
1355     return $conf->{cmode} // $confdesc->{cmode}->{default};
1356 }
1357
1358 sub get_console_command {
1359     my ($vmid, $conf) = @_;
1360
1361     my $cmode = get_cmode($conf);
1362
1363     if ($cmode eq 'console') {
1364         return ['lxc-console', '-n',  $vmid, '-t', 0];
1365     } elsif ($cmode eq 'tty') {
1366         return ['lxc-console', '-n',  $vmid];
1367     } elsif ($cmode eq 'shell') {
1368         return ['lxc-attach', '--clear-env', '-n', $vmid];
1369     } else {
1370         die "internal error";
1371     }
1372 }
1373
1374 sub get_primary_ips {
1375     my ($conf) = @_;
1376
1377     # return data from net0
1378
1379     return undef if !defined($conf->{net0});
1380     my $net = parse_lxc_network($conf->{net0});
1381
1382     my $ipv4 = $net->{ip};
1383     if ($ipv4) {
1384         if ($ipv4 =~ /^(dhcp|manual)$/) {
1385             $ipv4 = undef
1386         } else {
1387             $ipv4 =~ s!/\d+$!!;
1388         }
1389     }
1390     my $ipv6 = $net->{ip6};
1391     if ($ipv6) {
1392         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1393             $ipv6 = undef;
1394         } else {
1395             $ipv6 =~ s!/\d+$!!;
1396         }
1397     }
1398
1399     return ($ipv4, $ipv6);
1400 }
1401
1402 sub delete_mountpoint_volume {
1403     my ($storage_cfg, $vmid, $volume) = @_;
1404
1405     # skip bind mounts and block devices
1406     if ($volume =~ m|^/|) {
1407             return;
1408     }
1409
1410     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1411     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1412 }
1413
1414 sub destroy_lxc_container {
1415     my ($storage_cfg, $vmid, $conf) = @_;
1416
1417     foreach_mountpoint($conf, sub {
1418         my ($ms, $mountpoint) = @_;
1419         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1420     });
1421
1422     rmdir "/var/lib/lxc/$vmid/rootfs";
1423     unlink "/var/lib/lxc/$vmid/config";
1424     rmdir "/var/lib/lxc/$vmid";
1425     destroy_config($vmid);
1426
1427     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1428     #PVE::Tools::run_command($cmd);
1429 }
1430
1431 sub vm_stop_cleanup {
1432     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1433
1434     eval {
1435         if (!$keepActive) {
1436
1437             my $vollist = get_vm_volumes($conf);
1438             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1439         }
1440     };
1441     warn $@ if $@; # avoid errors - just warn
1442 }
1443
1444 my $safe_num_ne = sub {
1445     my ($a, $b) = @_;
1446
1447     return 0 if !defined($a) && !defined($b);
1448     return 1 if !defined($a);
1449     return 1 if !defined($b);
1450
1451     return $a != $b;
1452 };
1453
1454 my $safe_string_ne = sub {
1455     my ($a, $b) = @_;
1456
1457     return 0 if !defined($a) && !defined($b);
1458     return 1 if !defined($a);
1459     return 1 if !defined($b);
1460
1461     return $a ne $b;
1462 };
1463
1464 sub update_net {
1465     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1466
1467     if ($newnet->{type} ne 'veth') {
1468         # for when there are physical interfaces
1469         die "cannot update interface of type $newnet->{type}";
1470     }
1471
1472     my $veth = "veth${vmid}i${netid}";
1473     my $eth = $newnet->{name};
1474
1475     if (my $oldnetcfg = $conf->{$opt}) {
1476         my $oldnet = parse_lxc_network($oldnetcfg);
1477
1478         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1479             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1480
1481             PVE::Network::veth_delete($veth);
1482             delete $conf->{$opt};
1483             write_config($vmid, $conf);
1484
1485             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1486
1487         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1488                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1489                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1490
1491                 if ($oldnet->{bridge}) {
1492                     PVE::Network::tap_unplug($veth);
1493                     foreach (qw(bridge tag firewall)) {
1494                         delete $oldnet->{$_};
1495                     }
1496                     $conf->{$opt} = print_lxc_network($oldnet);
1497                     write_config($vmid, $conf);
1498                 }
1499
1500                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1501                 foreach (qw(bridge tag firewall)) {
1502                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1503                 }
1504                 $conf->{$opt} = print_lxc_network($oldnet);
1505                 write_config($vmid, $conf);
1506         }
1507     } else {
1508         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1509     }
1510
1511     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1512 }
1513
1514 sub hotplug_net {
1515     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1516
1517     my $veth = "veth${vmid}i${netid}";
1518     my $vethpeer = $veth . "p";
1519     my $eth = $newnet->{name};
1520
1521     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1522     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1523
1524     # attach peer in container
1525     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1526     PVE::Tools::run_command($cmd);
1527
1528     # link up peer in container
1529     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1530     PVE::Tools::run_command($cmd);
1531
1532     my $done = { type => 'veth' };
1533     foreach (qw(bridge tag firewall hwaddr name)) {
1534         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1535     }
1536     $conf->{$opt} = print_lxc_network($done);
1537
1538     write_config($vmid, $conf);
1539 }
1540
1541 sub update_ipconfig {
1542     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1543
1544     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1545
1546     my $optdata = parse_lxc_network($conf->{$opt});
1547     my $deleted = [];
1548     my $added = [];
1549     my $nscmd = sub {
1550         my $cmdargs = shift;
1551         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1552     };
1553     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1554
1555     my $change_ip_config = sub {
1556         my ($ipversion) = @_;
1557
1558         my $family_opt = "-$ipversion";
1559         my $suffix = $ipversion == 4 ? '' : $ipversion;
1560         my $gw= "gw$suffix";
1561         my $ip= "ip$suffix";
1562
1563         my $newip = $newnet->{$ip};
1564         my $newgw = $newnet->{$gw};
1565         my $oldip = $optdata->{$ip};
1566
1567         my $change_ip = &$safe_string_ne($oldip, $newip);
1568         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1569
1570         return if !$change_ip && !$change_gw;
1571
1572         # step 1: add new IP, if this fails we cancel
1573         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1574         if ($change_ip && $is_real_ip) {
1575             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1576             if (my $err = $@) {
1577                 warn $err;
1578                 return;
1579             }
1580         }
1581
1582         # step 2: replace gateway
1583         #   If this fails we delete the added IP and cancel.
1584         #   If it succeeds we save the config and delete the old IP, ignoring
1585         #   errors. The config is then saved.
1586         # Note: 'ip route replace' can add
1587         if ($change_gw) {
1588             if ($newgw) {
1589                 eval {
1590                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1591                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1592                     }
1593                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1594                 };
1595                 if (my $err = $@) {
1596                     warn $err;
1597                     # the route was not replaced, the old IP is still available
1598                     # rollback (delete new IP) and cancel
1599                     if ($change_ip) {
1600                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1601                         warn $@ if $@; # no need to die here
1602                     }
1603                     return;
1604                 }
1605             } else {
1606                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1607                 # if the route was not deleted, the guest might have deleted it manually
1608                 # warn and continue
1609                 warn $@ if $@;
1610             }
1611         }
1612
1613         # from this point on we save the configuration
1614         # step 3: delete old IP ignoring errors
1615         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1616             # We need to enable promote_secondaries, otherwise our newly added
1617             # address will be removed along with the old one.
1618             my $promote = 0;
1619             eval {
1620                 if ($ipversion == 4) {
1621                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1622                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1623                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1624                 }
1625                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1626             };
1627             warn $@ if $@; # no need to die here
1628
1629             if ($ipversion == 4) {
1630                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1631             }
1632         }
1633
1634         foreach my $property ($ip, $gw) {
1635             if ($newnet->{$property}) {
1636                 $optdata->{$property} = $newnet->{$property};
1637             } else {
1638                 delete $optdata->{$property};
1639             }
1640         }
1641         $conf->{$opt} = print_lxc_network($optdata);
1642         write_config($vmid, $conf);
1643         $lxc_setup->setup_network($conf);
1644     };
1645
1646     &$change_ip_config(4);
1647     &$change_ip_config(6);
1648
1649 }
1650
1651 # Internal snapshots
1652
1653 # NOTE: Snapshot create/delete involves several non-atomic
1654 # action, and can take a long time.
1655 # So we try to avoid locking the file and use 'lock' variable
1656 # inside the config file instead.
1657
1658 my $snapshot_copy_config = sub {
1659     my ($source, $dest) = @_;
1660
1661     foreach my $k (keys %$source) {
1662         next if $k eq 'snapshots';
1663         next if $k eq 'snapstate';
1664         next if $k eq 'snaptime';
1665         next if $k eq 'vmstate';
1666         next if $k eq 'lock';
1667         next if $k eq 'digest';
1668         next if $k eq 'description';
1669
1670         $dest->{$k} = $source->{$k};
1671     }
1672 };
1673
1674 my $snapshot_prepare = sub {
1675     my ($vmid, $snapname, $comment) = @_;
1676
1677     my $snap;
1678
1679     my $updatefn =  sub {
1680
1681         my $conf = load_config($vmid);
1682
1683         die "you can't take a snapshot if it's a template\n"
1684             if is_template($conf);
1685
1686         check_lock($conf);
1687
1688         $conf->{lock} = 'snapshot';
1689
1690         die "snapshot name '$snapname' already used\n"
1691             if defined($conf->{snapshots}->{$snapname});
1692
1693         my $storecfg = PVE::Storage::config();
1694         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1695
1696         $snap = $conf->{snapshots}->{$snapname} = {};
1697
1698         &$snapshot_copy_config($conf, $snap);
1699
1700         $snap->{'snapstate'} = "prepare";
1701         $snap->{'snaptime'} = time();
1702         $snap->{'description'} = $comment if $comment;
1703         $conf->{snapshots}->{$snapname} = $snap;
1704
1705         write_config($vmid, $conf);
1706     };
1707
1708     lock_container($vmid, 10, $updatefn);
1709
1710     return $snap;
1711 };
1712
1713 my $snapshot_commit = sub {
1714     my ($vmid, $snapname) = @_;
1715
1716     my $updatefn = sub {
1717
1718         my $conf = load_config($vmid);
1719
1720         die "missing snapshot lock\n"
1721             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1722
1723         die "snapshot '$snapname' does not exist\n"
1724             if !defined($conf->{snapshots}->{$snapname});
1725
1726         die "wrong snapshot state\n"
1727             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1728                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1729
1730         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1731         delete $conf->{lock};
1732         $conf->{parent} = $snapname;
1733
1734         write_config($vmid, $conf);
1735     };
1736
1737     lock_container($vmid, 10 ,$updatefn);
1738 };
1739
1740 sub has_feature {
1741     my ($feature, $conf, $storecfg, $snapname) = @_;
1742
1743     my $err;
1744
1745     foreach_mountpoint($conf, sub {
1746         my ($ms, $mountpoint) = @_;
1747
1748         return if $err; # skip further test
1749
1750         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1751
1752         # TODO: implement support for mountpoints
1753         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1754             if $ms ne 'rootfs';
1755     });
1756
1757     return $err ? 0 : 1;
1758 }
1759
1760 sub snapshot_create {
1761     my ($vmid, $snapname, $comment) = @_;
1762
1763     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1764
1765     my $conf = load_config($vmid);
1766
1767     my $running = check_running($vmid);
1768     eval {
1769         if ($running) {
1770             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1771             PVE::Tools::run_command(['/bin/sync']);
1772         };
1773
1774         my $storecfg = PVE::Storage::config();
1775         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1776         my $volid = $rootinfo->{volume};
1777
1778         if ($running) {
1779             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1780         };
1781
1782         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1783         &$snapshot_commit($vmid, $snapname);
1784     };
1785     if(my $err = $@) {
1786         snapshot_delete($vmid, $snapname, 1);
1787         die "$err\n";
1788     }
1789 }
1790
1791 sub snapshot_delete {
1792     my ($vmid, $snapname, $force) = @_;
1793
1794     my $snap;
1795
1796     my $conf;
1797
1798     my $updatefn =  sub {
1799
1800         $conf = load_config($vmid);
1801
1802         die "you can't delete a snapshot if vm is a template\n"
1803             if is_template($conf);
1804
1805         $snap = $conf->{snapshots}->{$snapname};
1806
1807         check_lock($conf);
1808
1809         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1810
1811         $snap->{snapstate} = 'delete';
1812
1813         write_config($vmid, $conf);
1814     };
1815
1816     lock_container($vmid, 10, $updatefn);
1817
1818     my $storecfg = PVE::Storage::config();
1819
1820     my $del_snap =  sub {
1821
1822         check_lock($conf);
1823
1824         if ($conf->{parent} eq $snapname) {
1825             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1826                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1827             } else {
1828                 delete $conf->{parent};
1829             }
1830         }
1831
1832         delete $conf->{snapshots}->{$snapname};
1833
1834         write_config($vmid, $conf);
1835     };
1836
1837     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1838     my $rootinfo = parse_ct_mountpoint($rootfs);
1839     my $volid = $rootinfo->{volume};
1840
1841     eval {
1842         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1843     };
1844     my $err = $@;
1845
1846     if(!$err || ($err && $force)) {
1847         lock_container($vmid, 10, $del_snap);
1848         if ($err) {
1849             die "Can't delete snapshot: $vmid $snapname $err\n";
1850         }
1851     }
1852 }
1853
1854 sub snapshot_rollback {
1855     my ($vmid, $snapname) = @_;
1856
1857     my $storecfg = PVE::Storage::config();
1858
1859     my $conf = load_config($vmid);
1860
1861     die "you can't rollback if vm is a template\n" if is_template($conf);
1862
1863     my $snap = $conf->{snapshots}->{$snapname};
1864
1865     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1866
1867     my $rootfs = $snap->{rootfs};
1868     my $rootinfo = parse_ct_mountpoint($rootfs);
1869     my $volid = $rootinfo->{volume};
1870
1871     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1872
1873     my $updatefn = sub {
1874
1875         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1876             if $snap->{snapstate};
1877
1878         check_lock($conf);
1879
1880         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1881
1882         die "unable to rollback vm $vmid: vm is running\n"
1883             if check_running($vmid);
1884
1885         $conf->{lock} = 'rollback';
1886
1887         my $forcemachine;
1888
1889         # copy snapshot config to current config
1890
1891         my $tmp_conf = $conf;
1892         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1893         $conf->{snapshots} = $tmp_conf->{snapshots};
1894         delete $conf->{snaptime};
1895         delete $conf->{snapname};
1896         $conf->{parent} = $snapname;
1897
1898         write_config($vmid, $conf);
1899     };
1900
1901     my $unlockfn = sub {
1902         delete $conf->{lock};
1903         write_config($vmid, $conf);
1904     };
1905
1906     lock_container($vmid, 10, $updatefn);
1907
1908     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1909
1910     lock_container($vmid, 5, $unlockfn);
1911 }
1912
1913 sub template_create {
1914     my ($vmid, $conf) = @_;
1915
1916     my $storecfg = PVE::Storage::config();
1917
1918     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1919     my $volid = $rootinfo->{volume};
1920
1921     die "Template feature is not available for '$volid'\n"
1922         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1923
1924     PVE::Storage::activate_volumes($storecfg, [$volid]);
1925
1926     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1927     $rootinfo->{volume} = $template_volid;
1928     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1929
1930     write_config($vmid, $conf);
1931 }
1932
1933 sub is_template {
1934     my ($conf) = @_;
1935
1936     return 1 if defined $conf->{template} && $conf->{template} == 1;
1937 }
1938
1939 sub mountpoint_names {
1940     my ($reverse) = @_;
1941
1942     my @names = ('rootfs');
1943
1944     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1945         push @names, "mp$i";
1946     }
1947
1948     return $reverse ? reverse @names : @names;
1949 }
1950
1951 # The container might have *different* symlinks than the host. realpath/abs_path
1952 # use the actual filesystem to resolve links.
1953 sub sanitize_mountpoint {
1954     my ($mp) = @_;
1955     $mp = '/' . $mp; # we always start with a slash
1956     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1957     $mp =~ s@/\./@@g; # collapse /./
1958     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1959     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1960     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1961     return $mp;
1962 }
1963
1964 sub foreach_mountpoint_full {
1965     my ($conf, $reverse, $func) = @_;
1966
1967     foreach my $key (mountpoint_names($reverse)) {
1968         my $value = $conf->{$key};
1969         next if !defined($value);
1970         my $mountpoint = parse_ct_mountpoint($value, 1);
1971         next if !defined($mountpoint);
1972
1973         # just to be sure: rootfs is /
1974         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1975         $mountpoint->{mp} = sanitize_mountpoint($path);
1976
1977         $path = $mountpoint->{volume};
1978         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1979
1980         &$func($key, $mountpoint);
1981     }
1982 }
1983
1984 sub foreach_mountpoint {
1985     my ($conf, $func) = @_;
1986
1987     foreach_mountpoint_full($conf, 0, $func);
1988 }
1989
1990 sub foreach_mountpoint_reverse {
1991     my ($conf, $func) = @_;
1992
1993     foreach_mountpoint_full($conf, 1, $func);
1994 }
1995
1996 sub check_ct_modify_config_perm {
1997     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1998
1999     return 1 if $authuser ne 'root@pam';
2000
2001     foreach my $opt (@$key_list) {
2002
2003         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2004             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2005         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2006             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2007         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2008             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2009         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2010                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2011             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2012         } else {
2013             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2014         }
2015     }
2016
2017     return 1;
2018 }
2019
2020 sub umount_all {
2021     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2022
2023     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2024     my $volid_list = get_vm_volumes($conf);
2025
2026     foreach_mountpoint_reverse($conf, sub {
2027         my ($ms, $mountpoint) = @_;
2028
2029         my $volid = $mountpoint->{volume};
2030         my $mount = $mountpoint->{mp};
2031
2032         return if !$volid || !$mount;
2033
2034         my $mount_path = "$rootdir/$mount";
2035         $mount_path =~ s!/+!/!g;
2036
2037         return if !PVE::ProcFSTools::is_mounted($mount_path);
2038
2039         eval {
2040             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2041         };
2042         if (my $err = $@) {
2043             if ($noerr) {
2044                 warn $err;
2045             } else {
2046                 die $err;
2047             }
2048         }
2049     });
2050 }
2051
2052 sub mount_all {
2053     my ($vmid, $storage_cfg, $conf) = @_;
2054
2055     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2056     File::Path::make_path($rootdir);
2057
2058     my $volid_list = get_vm_volumes($conf);
2059     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2060
2061     eval {
2062         foreach_mountpoint($conf, sub {
2063             my ($ms, $mountpoint) = @_;
2064
2065             my $volid = $mountpoint->{volume};
2066             my $mount = $mountpoint->{mp};
2067
2068             return if !$volid || !$mount;
2069
2070             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2071             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2072                 PVE::Storage::parse_volname($storage_cfg, $volid);
2073
2074             die "unable to mount base volume - internal error" if $isBase;
2075
2076             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2077         });
2078     };
2079     if (my $err = $@) {
2080         warn "mounting container failed - $err";
2081         umount_all($vmid, $storage_cfg, $conf, 1);
2082     }
2083
2084     return $rootdir;
2085 }
2086
2087
2088 sub mountpoint_mount_path {
2089     my ($mountpoint, $storage_cfg, $snapname) = @_;
2090
2091     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2092 }
2093
2094 my $check_mount_path = sub {
2095     my ($path) = @_;
2096     $path = File::Spec->canonpath($path);
2097     my $real = Cwd::realpath($path);
2098     if ($real ne $path) {
2099         die "mount path modified by symlink: $path != $real";
2100     }
2101 };
2102
2103 # use $rootdir = undef to just return the corresponding mount path
2104 sub mountpoint_mount {
2105     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2106
2107     my $volid = $mountpoint->{volume};
2108     my $mount = $mountpoint->{mp};
2109
2110     return if !$volid || !$mount;
2111
2112     my $mount_path;
2113
2114     if (defined($rootdir)) {
2115         $rootdir =~ s!/+$!!;
2116         $mount_path = "$rootdir/$mount";
2117         $mount_path =~ s!/+!/!g;
2118         &$check_mount_path($mount_path);
2119         File::Path::mkpath($mount_path);
2120     }
2121
2122     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2123
2124     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2125
2126     if ($storage) {
2127
2128         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2129         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2130
2131         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2132             PVE::Storage::parse_volname($storage_cfg, $volid);
2133
2134         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2135
2136         if ($format eq 'subvol') {
2137             if ($mount_path) {
2138                 if ($snapname) {
2139                     if ($scfg->{type} eq 'zfspool') {
2140                         my $path_arg = $path;
2141                         $path_arg =~ s!^/+!!;
2142                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2143                     } else {
2144                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2145                     }
2146                 } else {
2147                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2148                 }
2149             }
2150             return wantarray ? ($path, 0) : $path;
2151         } elsif ($format eq 'raw' || $format eq 'iso') {
2152             my $use_loopdev = 0;
2153             my @extra_opts;
2154             if ($scfg->{path}) {
2155                 push @extra_opts, '-o', 'loop';
2156                 $use_loopdev = 1;
2157             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2158                 # do nothing
2159             } else {
2160                 die "unsupported storage type '$scfg->{type}'\n";
2161             }
2162             if ($mount_path) {
2163                 if ($format eq 'iso') {
2164                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2165                 } elsif ($isBase || defined($snapname)) {
2166                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2167                 } else {
2168                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2169                 }
2170             }
2171             return wantarray ? ($path, $use_loopdev) : $path;
2172         } else {
2173             die "unsupported image format '$format'\n";
2174         }
2175     } elsif ($volid =~ m|^/dev/.+|) {
2176         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2177         return wantarray ? ($volid, 0) : $volid;
2178     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2179         &$check_mount_path($volid);
2180         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2181         return wantarray ? ($volid, 0) : $volid;
2182     }
2183
2184     die "unsupported storage";
2185 }
2186
2187 sub get_vm_volumes {
2188     my ($conf, $excludes) = @_;
2189
2190     my $vollist = [];
2191
2192     foreach_mountpoint($conf, sub {
2193         my ($ms, $mountpoint) = @_;
2194
2195         return if $excludes && $ms eq $excludes;
2196
2197         my $volid = $mountpoint->{volume};
2198
2199         return if !$volid || $volid =~ m|^/|;
2200
2201         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2202         return if !$sid;
2203
2204         push @$vollist, $volid;
2205     });
2206
2207     return $vollist;
2208 }
2209
2210 sub mkfs {
2211     my ($dev) = @_;
2212
2213     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2214 }
2215
2216 sub format_disk {
2217     my ($storage_cfg, $volid) = @_;
2218
2219     if ($volid =~ m!^/dev/.+!) {
2220         mkfs($volid);
2221         return;
2222     }
2223
2224     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2225
2226     die "cannot format volume '$volid' with no storage\n" if !$storage;
2227
2228     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2229
2230     my $path = PVE::Storage::path($storage_cfg, $volid);
2231
2232     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2233         PVE::Storage::parse_volname($storage_cfg, $volid);
2234
2235     die "cannot format volume '$volid' (format == $format)\n"
2236         if $format ne 'raw';
2237
2238     mkfs($path);
2239 }
2240
2241 sub destroy_disks {
2242     my ($storecfg, $vollist) = @_;
2243
2244     foreach my $volid (@$vollist) {
2245         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2246         warn $@ if $@;
2247     }
2248 }
2249
2250 sub create_disks {
2251     my ($storecfg, $vmid, $settings, $conf) = @_;
2252
2253     my $vollist = [];
2254
2255     eval {
2256         foreach_mountpoint($settings, sub {
2257             my ($ms, $mountpoint) = @_;
2258
2259             my $volid = $mountpoint->{volume};
2260             my $mp = $mountpoint->{mp};
2261
2262             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2263
2264             return if !$storage;
2265
2266             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2267                 my ($storeid, $size_gb) = ($1, $2);
2268
2269                 my $size_kb = int(${size_gb}*1024) * 1024;
2270
2271                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2272                 # fixme: use better naming ct-$vmid-disk-X.raw?
2273
2274                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2275                     if ($size_kb > 0) {
2276                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2277                                                            undef, $size_kb);
2278                         format_disk($storecfg, $volid);
2279                     } else {
2280                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2281                                                            undef, 0);
2282                     }
2283                 } elsif ($scfg->{type} eq 'zfspool') {
2284
2285                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2286                                                        undef, $size_kb);
2287                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2288
2289                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2290                     format_disk($storecfg, $volid);
2291
2292                 } elsif ($scfg->{type} eq 'rbd') {
2293
2294                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2295                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2296                     format_disk($storecfg, $volid);
2297                 } else {
2298                     die "unable to create containers on storage type '$scfg->{type}'\n";
2299                 }
2300                 push @$vollist, $volid;
2301                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2302                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2303             } else {
2304                 # use specified/existing volid
2305             }
2306         });
2307     };
2308     # free allocated images on error
2309     if (my $err = $@) {
2310         destroy_disks($storecfg, $vollist);
2311         die $err;
2312     }
2313     return $vollist;
2314 }
2315
2316 # bash completion helper
2317
2318 sub complete_os_templates {
2319     my ($cmdname, $pname, $cvalue) = @_;
2320
2321     my $cfg = PVE::Storage::config();
2322
2323     my $storeid;
2324
2325     if ($cvalue =~ m/^([^:]+):/) {
2326         $storeid = $1;
2327     }
2328
2329     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2330     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2331
2332     my $res = [];
2333     foreach my $id (keys %$data) {
2334         foreach my $item (@{$data->{$id}}) {
2335             push @$res, $item->{volid} if defined($item->{volid});
2336         }
2337     }
2338
2339     return $res;
2340 }
2341
2342 my $complete_ctid_full = sub {
2343     my ($running) = @_;
2344
2345     my $idlist = vmstatus();
2346
2347     my $active_hash = list_active_containers();
2348
2349     my $res = [];
2350
2351     foreach my $id (keys %$idlist) {
2352         my $d = $idlist->{$id};
2353         if (defined($running)) {
2354             next if $d->{template};
2355             next if $running && !$active_hash->{$id};
2356             next if !$running && $active_hash->{$id};
2357         }
2358         push @$res, $id;
2359
2360     }
2361     return $res;
2362 };
2363
2364 sub complete_ctid {
2365     return &$complete_ctid_full();
2366 }
2367
2368 sub complete_ctid_stopped {
2369     return &$complete_ctid_full(0);
2370 }
2371
2372 sub complete_ctid_running {
2373     return &$complete_ctid_full(1);
2374 }
2375
2376 1;