src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--totals', '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format_description => 'volume',
  42         description => 'Volume, device or directory to mount into the container.',
  43     },
  44     backup => {
  45         type => 'boolean',
  46         format_description => '[1|0]',
  47         description => 'Whether to include the mountpoint in backups.',
  48         optional => 1,
  49     },
  50     size => {
  51         type => 'string',
  52         format => 'disk-size',
  53         format_description => 'DiskSize',
  54         description => 'Volume size (read only value).',
  55         optional => 1,
  56     },
  57 };
  58
  59 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  60     type => 'string', format => $rootfs_desc,
  61     description => "Use volume as container root.",
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 my $confdesc = {
  72     lock => {
  73         optional => 1,
  74         type => 'string',
  75         description => "Lock/unlock the VM.",
  76         enum => [qw(migrate backup snapshot rollback)],
  77     },
  78     onboot => {
  79         optional => 1,
  80         type => 'boolean',
  81         description => "Specifies whether a VM will be started during system bootup.",
  82         default => 0,
  83     },
  84     startup => get_standard_option('pve-startup-order'),
  85     template => {
  86         optional => 1,
  87         type => 'boolean',
  88         description => "Enable/disable Template.",
  89         default => 0,
  90     },
  91     arch => {
  92         optional => 1,
  93         type => 'string',
  94         enum => ['amd64', 'i386'],
  95         description => "OS architecture type.",
  96         default => 'amd64',
  97     },
  98     ostype => {
  99         optional => 1,
 100         type => 'string',
 101         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
 102         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 103     },
 104     console => {
 105         optional => 1,
 106         type => 'boolean',
 107         description => "Attach a console device (/dev/console) to the container.",
 108         default => 1,
 109     },
 110     tty => {
 111         optional => 1,
 112         type => 'integer',
 113         description => "Specify the number of tty available to the container",
 114         minimum => 0,
 115         maximum => 6,
 116         default => 2,
 117     },
 118     cpulimit => {
 119         optional => 1,
 120         type => 'number',
 121         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 122         minimum => 0,
 123         maximum => 128,
 124         default => 0,
 125     },
 126     cpuunits => {
 127         optional => 1,
 128         type => 'integer',
 129         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 130         minimum => 0,
 131         maximum => 500000,
 132         default => 1024,
 133     },
 134     memory => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Amount of RAM for the VM in MB.",
 138         minimum => 16,
 139         default => 512,
 140     },
 141     swap => {
 142         optional => 1,
 143         type => 'integer',
 144         description => "Amount of SWAP for the VM in MB.",
 145         minimum => 0,
 146         default => 512,
 147     },
 148     hostname => {
 149         optional => 1,
 150         description => "Set a host name for the container.",
 151         type => 'string', format => 'dns-name',
 152         maxLength => 255,
 153     },
 154     description => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Container description. Only used on the configuration web interface.",
 158     },
 159     searchdomain => {
 160         optional => 1,
 161         type => 'string', format => 'dns-name-list',
 162         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 163     },
 164     nameserver => {
 165         optional => 1,
 166         type => 'string', format => 'address-list',
 167         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 168     },
 169     rootfs => get_standard_option('pve-ct-rootfs'),
 170     parent => {
 171         optional => 1,
 172         type => 'string', format => 'pve-configid',
 173         maxLength => 40,
 174         description => "Parent snapshot name. This is used internally, and should not be modified.",
 175     },
 176     snaptime => {
 177         optional => 1,
 178         description => "Timestamp for snapshots.",
 179         type => 'integer',
 180         minimum => 0,
 181     },
 182     cmode => {
 183         optional => 1,
 184         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 185         type => 'string',
 186         enum => ['shell', 'console', 'tty'],
 187         default => 'tty',
 188     },
 189     protection => {
 190         optional => 1,
 191         type => 'boolean',
 192         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 193         default => 0,
 194     },
 195 };
 196
 197 my $valid_lxc_conf_keys = {
 198     'lxc.include' => 1,
 199     'lxc.arch' => 1,
 200     'lxc.utsname' => 1,
 201     'lxc.haltsignal' => 1,
 202     'lxc.rebootsignal' => 1,
 203     'lxc.stopsignal' => 1,
 204     'lxc.init_cmd' => 1,
 205     'lxc.network.type' => 1,
 206     'lxc.network.flags' => 1,
 207     'lxc.network.link' => 1,
 208     'lxc.network.mtu' => 1,
 209     'lxc.network.name' => 1,
 210     'lxc.network.hwaddr' => 1,
 211     'lxc.network.ipv4' => 1,
 212     'lxc.network.ipv4.gateway' => 1,
 213     'lxc.network.ipv6' => 1,
 214     'lxc.network.ipv6.gateway' => 1,
 215     'lxc.network.script.up' => 1,
 216     'lxc.network.script.down' => 1,
 217     'lxc.pts' => 1,
 218     'lxc.console.logfile' => 1,
 219     'lxc.console' => 1,
 220     'lxc.tty' => 1,
 221     'lxc.devttydir' => 1,
 222     'lxc.hook.autodev' => 1,
 223     'lxc.autodev' => 1,
 224     'lxc.kmsg' => 1,
 225     'lxc.mount' => 1,
 226     'lxc.mount.entry' => 1,
 227     'lxc.mount.auto' => 1,
 228     'lxc.rootfs' => 1,
 229     'lxc.rootfs.mount' => 1,
 230     'lxc.rootfs.options' => 1,
 231     # lxc.cgroup.*
 232     'lxc.cap.drop' => 1,
 233     'lxc.cap.keep' => 1,
 234     'lxc.aa_profile' => 1,
 235     'lxc.aa_allow_incomplete' => 1,
 236     'lxc.se_context' => 1,
 237     'lxc.seccomp' => 1,
 238     'lxc.id_map' => 1,
 239     'lxc.hook.pre-start' => 1,
 240     'lxc.hook.pre-mount' => 1,
 241     'lxc.hook.mount' => 1,
 242     'lxc.hook.start' => 1,
 243     'lxc.hook.stop' => 1,
 244     'lxc.hook.post-stop' => 1,
 245     'lxc.hook.clone' => 1,
 246     'lxc.hook.destroy' => 1,
 247     'lxc.loglevel' => 1,
 248     'lxc.logfile' => 1,
 249     'lxc.start.auto' => 1,
 250     'lxc.start.delay' => 1,
 251     'lxc.start.order' => 1,
 252     'lxc.group' => 1,
 253     'lxc.environment' => 1,
 254     'lxc.' => 1,
 255     'lxc.' => 1,
 256     'lxc.' => 1,
 257     'lxc.' => 1,
 258 };
 259
 260 my $netconf_desc = {
 261     type => {
 262         type => 'string',
 263         optional => 1,
 264         description => "Network interface type.",
 265         enum => [qw(veth)],
 266     },
 267     name => {
 268         type => 'string',
 269         format_description => 'String',
 270         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 271         pattern => '[-_.\w\d]+',
 272     },
 273     bridge => {
 274         type => 'string',
 275         format_description => 'vmbr<Number>',
 276         description => 'Bridge to attach the network device to.',
 277         pattern => '[-_.\w\d]+',
 278         optional => 1,
 279     },
 280     hwaddr => {
 281         type => 'string',
 282         format_description => 'MAC',
 283         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 284         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 285         optional => 1,
 286     },
 287     mtu => {
 288         type => 'integer',
 289         format_description => 'Number',
 290         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 291         minimum => 64, # minimum ethernet frame is 64 bytes
 292         optional => 1,
 293     },
 294     ip => {
 295         type => 'string',
 296         format => 'pve-ipv4-config',
 297         format_description => 'IPv4Format/CIDR',
 298         description => 'IPv4 address in CIDR format.',
 299         optional => 1,
 300     },
 301     gw => {
 302         type => 'string',
 303         format => 'ipv4',
 304         format_description => 'GatewayIPv4',
 305         description => 'Default gateway for IPv4 traffic.',
 306         optional => 1,
 307     },
 308     ip6 => {
 309         type => 'string',
 310         format => 'pve-ipv6-config',
 311         format_description => 'IPv6Format/CIDR',
 312         description => 'IPv6 address in CIDR format.',
 313         optional => 1,
 314     },
 315     gw6 => {
 316         type => 'string',
 317         format => 'ipv6',
 318         format_description => 'GatewayIPv6',
 319         description => 'Default gateway for IPv6 traffic.',
 320         optional => 1,
 321     },
 322     firewall => {
 323         type => 'boolean',
 324         format_description => '[1|0]',
 325         description => "Controls whether this interface's firewall rules should be used.",
 326         optional => 1,
 327     },
 328     tag => {
 329         type => 'integer',
 330         format_description => 'VlanNo',
 331         minimum => '2',
 332         maximum => '4094',
 333         description => "VLAN tag foro this interface.",
 334         optional => 1,
 335     },
 336 };
 337 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 338
 339 my $MAX_LXC_NETWORKS = 10;
 340 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 341     $confdesc->{"net$i"} = {
 342         optional => 1,
 343         type => 'string', format => $netconf_desc,
 344         description => "Specifies network interfaces for the container.",
 345     };
 346 }
 347
 348 my $mp_desc = {
 349     %$rootfs_desc,
 350     mp => {
 351         type => 'string',
 352         format_description => 'Path',
 353         description => 'Path to the mountpoint as seen from inside the container.',
 354         optional => 1,
 355     },
 356 };
 357 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 358
 359 my $unuseddesc = {
 360     optional => 1,
 361     type => 'string', format => 'pve-volume-id',
 362     description => "Reference to unused volumes.",
 363 };
 364
 365 my $MAX_MOUNT_POINTS = 10;
 366 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 367     $confdesc->{"mp$i"} = {
 368         optional => 1,
 369         type => 'string', format => $mp_desc,
 370         description => "Use volume as container mount point (experimental feature).",
 371         optional => 1,
 372     };
 373 }
 374
 375 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 376 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 377     $confdesc->{"unused$i"} = $unuseddesc;
 378 }
 379
 380 sub write_pct_config {
 381     my ($filename, $conf) = @_;
 382
 383     delete $conf->{snapstate}; # just to be sure
 384
 385     my $generate_raw_config = sub {
 386         my ($conf) = @_;
 387
 388         my $raw = '';
 389
 390         # add description as comment to top of file
 391         my $descr = $conf->{description} || '';
 392         foreach my $cl (split(/\n/, $descr)) {
 393             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 394         }
 395
 396         foreach my $key (sort keys %$conf) {
 397             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 398                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 399             my $value = $conf->{$key};
 400             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 401             $raw .= "$key: $value\n";
 402         }
 403
 404         if (my $lxcconf = $conf->{lxc}) {
 405             foreach my $entry (@$lxcconf) {
 406                 my ($k, $v) = @$entry;
 407                 $raw .= "$k: $v\n";
 408             }
 409         }
 410
 411         return $raw;
 412     };
 413
 414     my $raw = &$generate_raw_config($conf);
 415
 416     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 417         $raw .= "\n[$snapname]\n";
 418         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 419     }
 420
 421     return $raw;
 422 }
 423
 424 sub check_type {
 425     my ($key, $value) = @_;
 426
 427     die "unknown setting '$key'\n" if !$confdesc->{$key};
 428
 429     my $type = $confdesc->{$key}->{type};
 430
 431     if (!defined($value)) {
 432         die "got undefined value\n";
 433     }
 434
 435     if ($value =~ m/[\n\r]/) {
 436         die "property contains a line feed\n";
 437     }
 438
 439     if ($type eq 'boolean') {
 440         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 441         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 442         die "type check ('boolean') failed - got '$value'\n";
 443     } elsif ($type eq 'integer') {
 444         return int($1) if $value =~ m/^(\d+)$/;
 445         die "type check ('integer') failed - got '$value'\n";
 446     } elsif ($type eq 'number') {
 447         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 448         die "type check ('number') failed - got '$value'\n";
 449     } elsif ($type eq 'string') {
 450         if (my $fmt = $confdesc->{$key}->{format}) {
 451             PVE::JSONSchema::check_format($fmt, $value);
 452             return $value;
 453         }
 454         return $value;
 455     } else {
 456         die "internal error"
 457     }
 458 }
 459
 460 sub parse_pct_config {
 461     my ($filename, $raw) = @_;
 462
 463     return undef if !defined($raw);
 464
 465     my $res = {
 466         digest => Digest::SHA::sha1_hex($raw),
 467         snapshots => {},
 468     };
 469
 470     $filename =~ m|/lxc/(\d+).conf$|
 471         || die "got strange filename '$filename'";
 472
 473     my $vmid = $1;
 474
 475     my $conf = $res;
 476     my $descr = '';
 477     my $section = '';
 478
 479     my @lines = split(/\n/, $raw);
 480     foreach my $line (@lines) {
 481         next if $line =~ m/^\s*$/;
 482
 483         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 484             $section = $1;
 485             $conf->{description} = $descr if $descr;
 486             $descr = '';
 487             $conf = $res->{snapshots}->{$section} = {};
 488             next;
 489         }
 490
 491         if ($line =~ m/^\#(.*)\s*$/) {
 492             $descr .= PVE::Tools::decode_text($1) . "\n";
 493             next;
 494         }
 495
 496         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 497             my $key = $1;
 498             my $value = $3;
 499             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 500                 push @{$conf->{lxc}}, [$key, $value];
 501             } else {
 502                 warn "vm $vmid - unable to parse config: $line\n";
 503             }
 504         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 505             $descr .= PVE::Tools::decode_text($2);
 506         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 507             $conf->{snapstate} = $1;
 508         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 509             my $key = $1;
 510             my $value = $2;
 511             eval { $value = check_type($key, $value); };
 512             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 513             $conf->{$key} = $value;
 514         } else {
 515             warn "vm $vmid - unable to parse config: $line\n";
 516         }
 517     }
 518
 519     $conf->{description} = $descr if $descr;
 520
 521     delete $res->{snapstate}; # just to be sure
 522
 523     return $res;
 524 }
 525
 526 sub config_list {
 527     my $vmlist = PVE::Cluster::get_vmlist();
 528     my $res = {};
 529     return $res if !$vmlist || !$vmlist->{ids};
 530     my $ids = $vmlist->{ids};
 531
 532     foreach my $vmid (keys %$ids) {
 533         next if !$vmid; # skip CT0
 534         my $d = $ids->{$vmid};
 535         next if !$d->{node} || $d->{node} ne $nodename;
 536         next if !$d->{type} || $d->{type} ne 'lxc';
 537         $res->{$vmid}->{type} = 'lxc';
 538     }
 539     return $res;
 540 }
 541
 542 sub cfs_config_path {
 543     my ($vmid, $node) = @_;
 544
 545     $node = $nodename if !$node;
 546     return "nodes/$node/lxc/$vmid.conf";
 547 }
 548
 549 sub config_file {
 550     my ($vmid, $node) = @_;
 551
 552     my $cfspath = cfs_config_path($vmid, $node);
 553     return "/etc/pve/$cfspath";
 554 }
 555
 556 sub load_config {
 557     my ($vmid, $node) = @_;
 558
 559     $node = $nodename if !$node;
 560     my $cfspath = cfs_config_path($vmid, $node);
 561
 562     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 563     die "container $vmid does not exists\n" if !defined($conf);
 564
 565     return $conf;
 566 }
 567
 568 sub create_config {
 569     my ($vmid, $conf) = @_;
 570
 571     my $dir = "/etc/pve/nodes/$nodename/lxc";
 572     mkdir $dir;
 573
 574     write_config($vmid, $conf);
 575 }
 576
 577 sub destroy_config {
 578     my ($vmid) = @_;
 579
 580     unlink config_file($vmid, $nodename);
 581 }
 582
 583 sub write_config {
 584     my ($vmid, $conf) = @_;
 585
 586     my $cfspath = cfs_config_path($vmid);
 587
 588     PVE::Cluster::cfs_write_file($cfspath, $conf);
 589 }
 590
 591 # flock: we use one file handle per process, so lock file
 592 # can be called multiple times and succeeds for the same process.
 593
 594 my $lock_handles =  {};
 595 my $lockdir = "/run/lock/lxc";
 596
 597 sub lock_filename {
 598     my ($vmid) = @_;
 599
 600     return "$lockdir/pve-config-${vmid}.lock";
 601 }
 602
 603 sub lock_aquire {
 604     my ($vmid, $timeout) = @_;
 605
 606     $timeout = 10 if !$timeout;
 607     my $mode = LOCK_EX;
 608
 609     my $filename = lock_filename($vmid);
 610
 611     mkdir $lockdir if !-d $lockdir;
 612
 613     my $lock_func = sub {
 614         if (!$lock_handles->{$$}->{$filename}) {
 615             my $fh = new IO::File(">>$filename") ||
 616                 die "can't open file - $!\n";
 617             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 618         }
 619
 620         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 621             print STDERR "trying to aquire lock...";
 622             my $success;
 623             while(1) {
 624                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 625                 # try again on EINTR (see bug #273)
 626                 if ($success || ($! != EINTR)) {
 627                     last;
 628                 }
 629             }
 630             if (!$success) {
 631                 print STDERR " failed\n";
 632                 die "can't aquire lock - $!\n";
 633             }
 634
 635             print STDERR " OK\n";
 636         }
 637
 638         $lock_handles->{$$}->{$filename}->{refcount}++;
 639     };
 640
 641     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 642     my $err = $@;
 643     if ($err) {
 644         die "can't lock file '$filename' - $err";
 645     }
 646 }
 647
 648 sub lock_release {
 649     my ($vmid) = @_;
 650
 651     my $filename = lock_filename($vmid);
 652
 653     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 654         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 655         if ($refcount <= 0) {
 656             $lock_handles->{$$}->{$filename} = undef;
 657             close ($fh);
 658         }
 659     }
 660 }
 661
 662 sub lock_container {
 663     my ($vmid, $timeout, $code, @param) = @_;
 664
 665     my $res;
 666
 667     lock_aquire($vmid, $timeout);
 668     eval { $res = &$code(@param) };
 669     my $err = $@;
 670     lock_release($vmid);
 671
 672     die $err if $err;
 673
 674     return $res;
 675 }
 676
 677 sub option_exists {
 678     my ($name) = @_;
 679
 680     return defined($confdesc->{$name});
 681 }
 682
 683 # add JSON properties for create and set function
 684 sub json_config_properties {
 685     my $prop = shift;
 686
 687     foreach my $opt (keys %$confdesc) {
 688         next if $opt eq 'parent' || $opt eq 'snaptime';
 689         next if $prop->{$opt};
 690         $prop->{$opt} = $confdesc->{$opt};
 691     }
 692
 693     return $prop;
 694 }
 695
 696 sub json_config_properties_no_rootfs {
 697     my $prop = shift;
 698
 699     foreach my $opt (keys %$confdesc) {
 700         next if $prop->{$opt};
 701         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 702         $prop->{$opt} = $confdesc->{$opt};
 703     }
 704
 705     return $prop;
 706 }
 707
 708 # container status helpers
 709
 710 sub list_active_containers {
 711
 712     my $filename = "/proc/net/unix";
 713
 714     # similar test is used by lcxcontainers.c: list_active_containers
 715     my $res = {};
 716
 717     my $fh = IO::File->new ($filename, "r");
 718     return $res if !$fh;
 719
 720     while (defined(my $line = <$fh>)) {
 721         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 722             my $path = $1;
 723             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 724                 $res->{$1} = 1;
 725             }
 726         }
 727     }
 728
 729     close($fh);
 730
 731     return $res;
 732 }
 733
 734 # warning: this is slow
 735 sub check_running {
 736     my ($vmid) = @_;
 737
 738     my $active_hash = list_active_containers();
 739
 740     return 1 if defined($active_hash->{$vmid});
 741
 742     return undef;
 743 }
 744
 745 sub get_container_disk_usage {
 746     my ($vmid, $pid) = @_;
 747
 748     return PVE::Tools::df("/proc/$pid/root/", 1);
 749 }
 750
 751 my $last_proc_vmid_stat;
 752
 753 my $parse_cpuacct_stat = sub {
 754     my ($vmid) = @_;
 755
 756     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 757
 758     my $stat = {};
 759
 760     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 761
 762         $stat->{utime} = $1;
 763         $stat->{stime} = $2;
 764
 765     }
 766
 767     return $stat;
 768 };
 769
 770 sub vmstatus {
 771     my ($opt_vmid) = @_;
 772
 773     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 774
 775     my $active_hash = list_active_containers();
 776
 777     my $cpucount = $cpuinfo->{cpus} || 1;
 778
 779     my $cdtime = gettimeofday;
 780
 781     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 782
 783     foreach my $vmid (keys %$list) {
 784         my $d = $list->{$vmid};
 785
 786         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 787         warn $@ if $@; # ignore errors (consider them stopped)
 788
 789         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 790
 791         my $cfspath = cfs_config_path($vmid);
 792         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 793
 794         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 795         $d->{name} =~ s/[\s]//g;
 796
 797         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 798
 799         if ($d->{pid}) {
 800             my $res = get_container_disk_usage($vmid, $d->{pid});
 801             $d->{disk} = $res->{used};
 802             $d->{maxdisk} = $res->{total};
 803         } else {
 804             $d->{disk} = 0;
 805             # use 4GB by default ??
 806             if (my $rootfs = $conf->{rootfs}) {
 807                 my $rootinfo = parse_ct_mountpoint($rootfs);
 808                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 809             } else {
 810                 $d->{maxdisk} = 4*1024*1024*1024;
 811             }
 812         }
 813
 814         $d->{mem} = 0;
 815         $d->{swap} = 0;
 816         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 817         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 818
 819         $d->{uptime} = 0;
 820         $d->{cpu} = 0;
 821
 822         $d->{netout} = 0;
 823         $d->{netin} = 0;
 824
 825         $d->{diskread} = 0;
 826         $d->{diskwrite} = 0;
 827
 828         $d->{template} = is_template($conf);
 829     }
 830
 831     foreach my $vmid (keys %$list) {
 832         my $d = $list->{$vmid};
 833         my $pid = $d->{pid};
 834
 835         next if !$pid; # skip stopped CTs
 836
 837         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 838         $d->{uptime} = time - $ctime; # the method lxcfs uses
 839
 840         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 841         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 842
 843         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 844         my @bytes = split(/\n/, $blkio_bytes);
 845         foreach my $byte (@bytes) {
 846             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 847                 $d->{diskread} = $2 if $key eq 'Read';
 848                 $d->{diskwrite} = $2 if $key eq 'Write';
 849             }
 850         }
 851
 852         my $pstat = &$parse_cpuacct_stat($vmid);
 853
 854         my $used = $pstat->{utime} + $pstat->{stime};
 855
 856         my $old = $last_proc_vmid_stat->{$vmid};
 857         if (!$old) {
 858             $last_proc_vmid_stat->{$vmid} = {
 859                 time => $cdtime,
 860                 used => $used,
 861                 cpu => 0,
 862             };
 863             next;
 864         }
 865
 866         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 867
 868         if ($dtime > 1000) {
 869             my $dutime = $used -  $old->{used};
 870
 871             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 872             $last_proc_vmid_stat->{$vmid} = {
 873                 time => $cdtime,
 874                 used => $used,
 875                 cpu => $d->{cpu},
 876             };
 877         } else {
 878             $d->{cpu} = $old->{cpu};
 879         }
 880     }
 881
 882     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 883
 884     foreach my $dev (keys %$netdev) {
 885         next if $dev !~ m/^veth([1-9]\d*)i/;
 886         my $vmid = $1;
 887         my $d = $list->{$vmid};
 888
 889         next if !$d;
 890
 891         $d->{netout} += $netdev->{$dev}->{receive};
 892         $d->{netin} += $netdev->{$dev}->{transmit};
 893
 894     }
 895
 896     return $list;
 897 }
 898
 899 sub parse_ct_mountpoint {
 900     my ($data, $noerr) = @_;
 901
 902     $data //= '';
 903
 904     my $res;
 905     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 906     if ($@) {
 907         return undef if $noerr;
 908         die $@;
 909     }
 910
 911     if (defined(my $size = $res->{size})) {
 912         $size = PVE::JSONSchema::parse_size($size);
 913         if (!defined($size)) {
 914             return undef if $noerr;
 915             die "invalid size: $size\n";
 916         }
 917         $res->{size} = $size;
 918     }
 919
 920     return $res;
 921 }
 922
 923 sub print_ct_mountpoint {
 924     my ($info, $nomp) = @_;
 925     my $skip = $nomp ? ['mp'] : [];
 926     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 927 }
 928
 929 sub print_lxc_network {
 930     my $net = shift;
 931     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 932 }
 933
 934 sub parse_lxc_network {
 935     my ($data) = @_;
 936
 937     my $res = {};
 938
 939     return $res if !$data;
 940
 941     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 942
 943     $res->{type} = 'veth';
 944     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 945
 946     return $res;
 947 }
 948
 949 sub read_cgroup_value {
 950     my ($group, $vmid, $name, $full) = @_;
 951
 952     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 953
 954     return PVE::Tools::file_get_contents($path) if $full;
 955
 956     return PVE::Tools::file_read_firstline($path);
 957 }
 958
 959 sub write_cgroup_value {
 960    my ($group, $vmid, $name, $value) = @_;
 961
 962    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 963    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 964
 965 }
 966
 967 sub find_lxc_console_pids {
 968
 969     my $res = {};
 970
 971     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 972         my ($pid) = @_;
 973
 974         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 975         return if !$cmdline;
 976
 977         my @args = split(/\0/, $cmdline);
 978
 979         # serach for lxc-console -n <vmid>
 980         return if scalar(@args) != 3;
 981         return if $args[1] ne '-n';
 982         return if $args[2] !~ m/^\d+$/;
 983         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 984
 985         my $vmid = $args[2];
 986
 987         push @{$res->{$vmid}}, $pid;
 988     });
 989
 990     return $res;
 991 }
 992
 993 sub find_lxc_pid {
 994     my ($vmid) = @_;
 995
 996     my $pid = undef;
 997     my $parser = sub {
 998         my $line = shift;
 999         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1000     };
1001     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1002
1003     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1004
1005     return $pid;
1006 }
1007
1008 # Note: we cannot use Net:IP, because that only allows strict
1009 # CIDR networks
1010 sub parse_ipv4_cidr {
1011     my ($cidr, $noerr) = @_;
1012
1013     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1014         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1015     }
1016
1017     return undef if $noerr;
1018
1019     die "unable to parse ipv4 address/mask\n";
1020 }
1021
1022 sub check_lock {
1023     my ($conf) = @_;
1024
1025     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1026 }
1027
1028 sub check_protection {
1029     my ($vm_conf, $err_msg) = @_;
1030
1031     if ($vm_conf->{protection}) {
1032         die "$err_msg - protection mode enabled\n";
1033     }
1034 }
1035
1036 sub update_lxc_config {
1037     my ($storage_cfg, $vmid, $conf) = @_;
1038
1039     my $dir = "/var/lib/lxc/$vmid";
1040
1041     if ($conf->{template}) {
1042
1043         unlink "$dir/config";
1044
1045         return;
1046     }
1047
1048     my $raw = '';
1049
1050     die "missing 'arch' - internal error" if !$conf->{arch};
1051     $raw .= "lxc.arch = $conf->{arch}\n";
1052
1053     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1054     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1055         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1056     } else {
1057         die "implement me";
1058     }
1059
1060     if (!has_dev_console($conf)) {
1061         $raw .= "lxc.console = none\n";
1062         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1063     }
1064
1065     my $ttycount = get_tty_count($conf);
1066     $raw .= "lxc.tty = $ttycount\n";
1067
1068     # some init scripts expects a linux terminal (turnkey).
1069     $raw .= "lxc.environment = TERM=linux\n";
1070
1071     my $utsname = $conf->{hostname} || "CT$vmid";
1072     $raw .= "lxc.utsname = $utsname\n";
1073
1074     my $memory = $conf->{memory} || 512;
1075     my $swap = $conf->{swap} // 0;
1076
1077     my $lxcmem = int($memory*1024*1024);
1078     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1079
1080     my $lxcswap = int(($memory + $swap)*1024*1024);
1081     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1082
1083     if (my $cpulimit = $conf->{cpulimit}) {
1084         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1085         my $value = int(100000*$cpulimit);
1086         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1087     }
1088
1089     my $shares = $conf->{cpuunits} || 1024;
1090     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1091
1092     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1093     $mountpoint->{mp} = '/';
1094
1095     my ($path, $use_loopdev) = mountpoint_mount_path($mountpoint, $storage_cfg);
1096     $path = "loop:$path" if $use_loopdev;
1097
1098     $raw .= "lxc.rootfs = $path\n";
1099
1100     my $netcount = 0;
1101     foreach my $k (keys %$conf) {
1102         next if $k !~ m/^net(\d+)$/;
1103         my $ind = $1;
1104         my $d = parse_lxc_network($conf->{$k});
1105         $netcount++;
1106         $raw .= "lxc.network.type = veth\n";
1107         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1108         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1109         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1110         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1111     }
1112
1113     if (my $lxcconf = $conf->{lxc}) {
1114         foreach my $entry (@$lxcconf) {
1115             my ($k, $v) = @$entry;
1116             $netcount++ if $k eq 'lxc.network.type';
1117             $raw .= "$k = $v\n";
1118         }
1119     }
1120
1121     $raw .= "lxc.network.type = empty\n" if !$netcount;
1122
1123     File::Path::mkpath("$dir/rootfs");
1124
1125     PVE::Tools::file_set_contents("$dir/config", $raw);
1126 }
1127
1128 # verify and cleanup nameserver list (replace \0 with ' ')
1129 sub verify_nameserver_list {
1130     my ($nameserver_list) = @_;
1131
1132     my @list = ();
1133     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1134         PVE::JSONSchema::pve_verify_ip($server);
1135         push @list, $server;
1136     }
1137
1138     return join(' ', @list);
1139 }
1140
1141 sub verify_searchdomain_list {
1142     my ($searchdomain_list) = @_;
1143
1144     my @list = ();
1145     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1146         # todo: should we add checks for valid dns domains?
1147         push @list, $server;
1148     }
1149
1150     return join(' ', @list);
1151 }
1152
1153 sub add_unused_volume {
1154     my ($config, $volid) = @_;
1155
1156     # skip bind mounts and block devices
1157     return if $volid =~ m|^/|;
1158
1159     my $key;
1160     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1161         my $test = "unused$ind";
1162         if (my $vid = $config->{$test}) {
1163             return if $vid eq $volid; # do not add duplicates
1164         } else {
1165             $key = $test;
1166         }
1167     }
1168
1169     die "To many unused volume - please delete them first.\n" if !$key;
1170
1171     $config->{$key} = $volid;
1172
1173     return $key;
1174 }
1175
1176 sub update_pct_config {
1177     my ($vmid, $conf, $running, $param, $delete) = @_;
1178
1179     my @nohotplug;
1180
1181     my $new_disks = 0;
1182     my @deleted_volumes;
1183
1184     my $rootdir;
1185     if ($running) {
1186         my $pid = find_lxc_pid($vmid);
1187         $rootdir = "/proc/$pid/root";
1188     }
1189
1190     my $hotplug_error = sub {
1191         if ($running) {
1192             push @nohotplug, @_;
1193             return 1;
1194         } else {
1195             return 0;
1196         }
1197     };
1198
1199     if (defined($delete)) {
1200         foreach my $opt (@$delete) {
1201             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1202                 die "unable to delete required option '$opt'\n";
1203             } elsif ($opt eq 'swap') {
1204                 delete $conf->{$opt};
1205                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1206             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1207                 delete $conf->{$opt};
1208             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1209                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1210                 next if $hotplug_error->($opt);
1211                 delete $conf->{$opt};
1212             } elsif ($opt =~ m/^net(\d)$/) {
1213                 delete $conf->{$opt};
1214                 next if !$running;
1215                 my $netid = $1;
1216                 PVE::Network::veth_delete("veth${vmid}i$netid");
1217             } elsif ($opt eq 'protection') {
1218                 delete $conf->{$opt};
1219             } elsif ($opt =~ m/^unused(\d+)$/) {
1220                 next if $hotplug_error->($opt);
1221                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1222                 push @deleted_volumes, $conf->{$opt};
1223                 delete $conf->{$opt};
1224             } elsif ($opt =~ m/^mp(\d+)$/) {
1225                 next if $hotplug_error->($opt);
1226                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1227                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1228                 add_unused_volume($conf, $mountpoint->{volume});
1229                 delete $conf->{$opt};
1230             } else {
1231                 die "implement me"
1232             }
1233             write_config($vmid, $conf) if $running;
1234         }
1235     }
1236
1237     # There's no separate swap size to configure, there's memory and "total"
1238     # memory (iow. memory+swap). This means we have to change them together.
1239     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1240     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1241     if (defined($wanted_memory) || defined($wanted_swap)) {
1242
1243         $wanted_memory //= ($conf->{memory} || 512);
1244         $wanted_swap //=  ($conf->{swap} || 0);
1245
1246         my $total = $wanted_memory + $wanted_swap;
1247         if ($running) {
1248             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1249             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1250         }
1251         $conf->{memory} = $wanted_memory;
1252         $conf->{swap} = $wanted_swap;
1253
1254         write_config($vmid, $conf) if $running;
1255     }
1256
1257     foreach my $opt (keys %$param) {
1258         my $value = $param->{$opt};
1259         if ($opt eq 'hostname') {
1260             $conf->{$opt} = $value;
1261         } elsif ($opt eq 'onboot') {
1262             $conf->{$opt} = $value ? 1 : 0;
1263         } elsif ($opt eq 'startup') {
1264             $conf->{$opt} = $value;
1265         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1266             next if $hotplug_error->($opt);
1267             $conf->{$opt} = $value;
1268         } elsif ($opt eq 'nameserver') {
1269             next if $hotplug_error->($opt);
1270             my $list = verify_nameserver_list($value);
1271             $conf->{$opt} = $list;
1272         } elsif ($opt eq 'searchdomain') {
1273             next if $hotplug_error->($opt);
1274             my $list = verify_searchdomain_list($value);
1275             $conf->{$opt} = $list;
1276         } elsif ($opt eq 'cpulimit') {
1277             next if $hotplug_error->($opt); # FIXME: hotplug
1278             $conf->{$opt} = $value;
1279         } elsif ($opt eq 'cpuunits') {
1280             $conf->{$opt} = $value;
1281             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1282         } elsif ($opt eq 'description') {
1283             $conf->{$opt} = PVE::Tools::encode_text($value);
1284         } elsif ($opt =~ m/^net(\d+)$/) {
1285             my $netid = $1;
1286             my $net = parse_lxc_network($value);
1287             if (!$running) {
1288                 $conf->{$opt} = print_lxc_network($net);
1289             } else {
1290                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1291             }
1292         } elsif ($opt eq 'protection') {
1293             $conf->{$opt} = $value ? 1 : 0;
1294         } elsif ($opt =~ m/^mp(\d+)$/) {
1295             next if $hotplug_error->($opt);
1296             check_protection($conf, "can't update CT $vmid drive '$opt'");
1297             $conf->{$opt} = $value;
1298             $new_disks = 1;
1299         } elsif ($opt eq 'rootfs') {
1300             check_protection($conf, "can't update CT $vmid drive '$opt'");
1301             die "implement me: $opt";
1302         } else {
1303             die "implement me: $opt";
1304         }
1305         write_config($vmid, $conf) if $running;
1306     }
1307
1308     if (@deleted_volumes) {
1309         my $storage_cfg = PVE::Storage::config();
1310         foreach my $volume (@deleted_volumes) {
1311             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1312         }
1313     }
1314
1315     if ($new_disks) {
1316         my $storage_cfg = PVE::Storage::config();
1317         create_disks($storage_cfg, $vmid, $conf, $conf);
1318     }
1319
1320     # This should be the last thing we do here
1321     if ($running && scalar(@nohotplug)) {
1322         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1323     }
1324 }
1325
1326 sub has_dev_console {
1327     my ($conf) = @_;
1328
1329     return !(defined($conf->{console}) && !$conf->{console});
1330 }
1331
1332 sub get_tty_count {
1333     my ($conf) = @_;
1334
1335     return $conf->{tty} // $confdesc->{tty}->{default};
1336 }
1337
1338 sub get_cmode {
1339     my ($conf) = @_;
1340
1341     return $conf->{cmode} // $confdesc->{cmode}->{default};
1342 }
1343
1344 sub get_console_command {
1345     my ($vmid, $conf) = @_;
1346
1347     my $cmode = get_cmode($conf);
1348
1349     if ($cmode eq 'console') {
1350         return ['lxc-console', '-n',  $vmid, '-t', 0];
1351     } elsif ($cmode eq 'tty') {
1352         return ['lxc-console', '-n',  $vmid];
1353     } elsif ($cmode eq 'shell') {
1354         return ['lxc-attach', '--clear-env', '-n', $vmid];
1355     } else {
1356         die "internal error";
1357     }
1358 }
1359
1360 sub get_primary_ips {
1361     my ($conf) = @_;
1362
1363     # return data from net0
1364
1365     return undef if !defined($conf->{net0});
1366     my $net = parse_lxc_network($conf->{net0});
1367
1368     my $ipv4 = $net->{ip};
1369     if ($ipv4) {
1370         if ($ipv4 =~ /^(dhcp|manual)$/) {
1371             $ipv4 = undef
1372         } else {
1373             $ipv4 =~ s!/\d+$!!;
1374         }
1375     }
1376     my $ipv6 = $net->{ip6};
1377     if ($ipv6) {
1378         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1379             $ipv6 = undef;
1380         } else {
1381             $ipv6 =~ s!/\d+$!!;
1382         }
1383     }
1384
1385     return ($ipv4, $ipv6);
1386 }
1387
1388 sub delete_mountpoint_volume {
1389     my ($storage_cfg, $vmid, $volume) = @_;
1390
1391     # skip bind mounts and block devices
1392     if ($volume =~ m|^/|) {
1393             return;
1394     }
1395
1396     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1397     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1398 }
1399
1400 sub destroy_lxc_container {
1401     my ($storage_cfg, $vmid, $conf) = @_;
1402
1403     foreach_mountpoint($conf, sub {
1404         my ($ms, $mountpoint) = @_;
1405         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1406     });
1407
1408     rmdir "/var/lib/lxc/$vmid/rootfs";
1409     unlink "/var/lib/lxc/$vmid/config";
1410     rmdir "/var/lib/lxc/$vmid";
1411     destroy_config($vmid);
1412
1413     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1414     #PVE::Tools::run_command($cmd);
1415 }
1416
1417 sub vm_stop_cleanup {
1418     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1419
1420     eval {
1421         if (!$keepActive) {
1422
1423             my $vollist = get_vm_volumes($conf);
1424             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1425         }
1426     };
1427     warn $@ if $@; # avoid errors - just warn
1428 }
1429
1430 my $safe_num_ne = sub {
1431     my ($a, $b) = @_;
1432
1433     return 0 if !defined($a) && !defined($b);
1434     return 1 if !defined($a);
1435     return 1 if !defined($b);
1436
1437     return $a != $b;
1438 };
1439
1440 my $safe_string_ne = sub {
1441     my ($a, $b) = @_;
1442
1443     return 0 if !defined($a) && !defined($b);
1444     return 1 if !defined($a);
1445     return 1 if !defined($b);
1446
1447     return $a ne $b;
1448 };
1449
1450 sub update_net {
1451     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1452
1453     if ($newnet->{type} ne 'veth') {
1454         # for when there are physical interfaces
1455         die "cannot update interface of type $newnet->{type}";
1456     }
1457
1458     my $veth = "veth${vmid}i${netid}";
1459     my $eth = $newnet->{name};
1460
1461     if (my $oldnetcfg = $conf->{$opt}) {
1462         my $oldnet = parse_lxc_network($oldnetcfg);
1463
1464         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1465             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1466
1467             PVE::Network::veth_delete($veth);
1468             delete $conf->{$opt};
1469             write_config($vmid, $conf);
1470
1471             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1472
1473         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1474                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1475                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1476
1477                 if ($oldnet->{bridge}) {
1478                     PVE::Network::tap_unplug($veth);
1479                     foreach (qw(bridge tag firewall)) {
1480                         delete $oldnet->{$_};
1481                     }
1482                     $conf->{$opt} = print_lxc_network($oldnet);
1483                     write_config($vmid, $conf);
1484                 }
1485
1486                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1487                 foreach (qw(bridge tag firewall)) {
1488                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1489                 }
1490                 $conf->{$opt} = print_lxc_network($oldnet);
1491                 write_config($vmid, $conf);
1492         }
1493     } else {
1494         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1495     }
1496
1497     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1498 }
1499
1500 sub hotplug_net {
1501     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1502
1503     my $veth = "veth${vmid}i${netid}";
1504     my $vethpeer = $veth . "p";
1505     my $eth = $newnet->{name};
1506
1507     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1508     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1509
1510     # attach peer in container
1511     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1512     PVE::Tools::run_command($cmd);
1513
1514     # link up peer in container
1515     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1516     PVE::Tools::run_command($cmd);
1517
1518     my $done = { type => 'veth' };
1519     foreach (qw(bridge tag firewall hwaddr name)) {
1520         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1521     }
1522     $conf->{$opt} = print_lxc_network($done);
1523
1524     write_config($vmid, $conf);
1525 }
1526
1527 sub update_ipconfig {
1528     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1529
1530     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1531
1532     my $optdata = parse_lxc_network($conf->{$opt});
1533     my $deleted = [];
1534     my $added = [];
1535     my $nscmd = sub {
1536         my $cmdargs = shift;
1537         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1538     };
1539     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1540
1541     my $change_ip_config = sub {
1542         my ($ipversion) = @_;
1543
1544         my $family_opt = "-$ipversion";
1545         my $suffix = $ipversion == 4 ? '' : $ipversion;
1546         my $gw= "gw$suffix";
1547         my $ip= "ip$suffix";
1548
1549         my $newip = $newnet->{$ip};
1550         my $newgw = $newnet->{$gw};
1551         my $oldip = $optdata->{$ip};
1552
1553         my $change_ip = &$safe_string_ne($oldip, $newip);
1554         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1555
1556         return if !$change_ip && !$change_gw;
1557
1558         # step 1: add new IP, if this fails we cancel
1559         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1560         if ($change_ip && $is_real_ip) {
1561             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1562             if (my $err = $@) {
1563                 warn $err;
1564                 return;
1565             }
1566         }
1567
1568         # step 2: replace gateway
1569         #   If this fails we delete the added IP and cancel.
1570         #   If it succeeds we save the config and delete the old IP, ignoring
1571         #   errors. The config is then saved.
1572         # Note: 'ip route replace' can add
1573         if ($change_gw) {
1574             if ($newgw) {
1575                 eval {
1576                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1577                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1578                     }
1579                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1580                 };
1581                 if (my $err = $@) {
1582                     warn $err;
1583                     # the route was not replaced, the old IP is still available
1584                     # rollback (delete new IP) and cancel
1585                     if ($change_ip) {
1586                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1587                         warn $@ if $@; # no need to die here
1588                     }
1589                     return;
1590                 }
1591             } else {
1592                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1593                 # if the route was not deleted, the guest might have deleted it manually
1594                 # warn and continue
1595                 warn $@ if $@;
1596             }
1597         }
1598
1599         # from this point on we save the configuration
1600         # step 3: delete old IP ignoring errors
1601         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1602             # We need to enable promote_secondaries, otherwise our newly added
1603             # address will be removed along with the old one.
1604             my $promote = 0;
1605             eval {
1606                 if ($ipversion == 4) {
1607                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1608                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1609                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1610                 }
1611                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1612             };
1613             warn $@ if $@; # no need to die here
1614
1615             if ($ipversion == 4) {
1616                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1617             }
1618         }
1619
1620         foreach my $property ($ip, $gw) {
1621             if ($newnet->{$property}) {
1622                 $optdata->{$property} = $newnet->{$property};
1623             } else {
1624                 delete $optdata->{$property};
1625             }
1626         }
1627         $conf->{$opt} = print_lxc_network($optdata);
1628         write_config($vmid, $conf);
1629         $lxc_setup->setup_network($conf);
1630     };
1631
1632     &$change_ip_config(4);
1633     &$change_ip_config(6);
1634
1635 }
1636
1637 # Internal snapshots
1638
1639 # NOTE: Snapshot create/delete involves several non-atomic
1640 # action, and can take a long time.
1641 # So we try to avoid locking the file and use 'lock' variable
1642 # inside the config file instead.
1643
1644 my $snapshot_copy_config = sub {
1645     my ($source, $dest) = @_;
1646
1647     foreach my $k (keys %$source) {
1648         next if $k eq 'snapshots';
1649         next if $k eq 'snapstate';
1650         next if $k eq 'snaptime';
1651         next if $k eq 'vmstate';
1652         next if $k eq 'lock';
1653         next if $k eq 'digest';
1654         next if $k eq 'description';
1655
1656         $dest->{$k} = $source->{$k};
1657     }
1658 };
1659
1660 my $snapshot_prepare = sub {
1661     my ($vmid, $snapname, $comment) = @_;
1662
1663     my $snap;
1664
1665     my $updatefn =  sub {
1666
1667         my $conf = load_config($vmid);
1668
1669         die "you can't take a snapshot if it's a template\n"
1670             if is_template($conf);
1671
1672         check_lock($conf);
1673
1674         $conf->{lock} = 'snapshot';
1675
1676         die "snapshot name '$snapname' already used\n"
1677             if defined($conf->{snapshots}->{$snapname});
1678
1679         my $storecfg = PVE::Storage::config();
1680         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1681
1682         $snap = $conf->{snapshots}->{$snapname} = {};
1683
1684         &$snapshot_copy_config($conf, $snap);
1685
1686         $snap->{'snapstate'} = "prepare";
1687         $snap->{'snaptime'} = time();
1688         $snap->{'description'} = $comment if $comment;
1689         $conf->{snapshots}->{$snapname} = $snap;
1690
1691         write_config($vmid, $conf);
1692     };
1693
1694     lock_container($vmid, 10, $updatefn);
1695
1696     return $snap;
1697 };
1698
1699 my $snapshot_commit = sub {
1700     my ($vmid, $snapname) = @_;
1701
1702     my $updatefn = sub {
1703
1704         my $conf = load_config($vmid);
1705
1706         die "missing snapshot lock\n"
1707             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1708
1709         die "snapshot '$snapname' does not exist\n"
1710             if !defined($conf->{snapshots}->{$snapname});
1711
1712         die "wrong snapshot state\n"
1713             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1714                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1715
1716         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1717         delete $conf->{lock};
1718         $conf->{parent} = $snapname;
1719
1720         write_config($vmid, $conf);
1721     };
1722
1723     lock_container($vmid, 10 ,$updatefn);
1724 };
1725
1726 sub has_feature {
1727     my ($feature, $conf, $storecfg, $snapname) = @_;
1728
1729     my $err;
1730
1731     foreach_mountpoint($conf, sub {
1732         my ($ms, $mountpoint) = @_;
1733
1734         return if $err; # skip further test
1735
1736         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1737
1738         # TODO: implement support for mountpoints
1739         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1740             if $ms ne 'rootfs';
1741     });
1742
1743     return $err ? 0 : 1;
1744 }
1745
1746 sub snapshot_create {
1747     my ($vmid, $snapname, $comment) = @_;
1748
1749     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1750
1751     my $conf = load_config($vmid);
1752
1753     my $running = check_running($vmid);
1754     eval {
1755         if ($running) {
1756             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1757             PVE::Tools::run_command(['/bin/sync']);
1758         };
1759
1760         my $storecfg = PVE::Storage::config();
1761         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1762         my $volid = $rootinfo->{volume};
1763
1764         if ($running) {
1765             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1766         };
1767
1768         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1769         &$snapshot_commit($vmid, $snapname);
1770     };
1771     if(my $err = $@) {
1772         snapshot_delete($vmid, $snapname, 1);
1773         die "$err\n";
1774     }
1775 }
1776
1777 sub snapshot_delete {
1778     my ($vmid, $snapname, $force) = @_;
1779
1780     my $snap;
1781
1782     my $conf;
1783
1784     my $updatefn =  sub {
1785
1786         $conf = load_config($vmid);
1787
1788         die "you can't delete a snapshot if vm is a template\n"
1789             if is_template($conf);
1790
1791         $snap = $conf->{snapshots}->{$snapname};
1792
1793         check_lock($conf);
1794
1795         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1796
1797         $snap->{snapstate} = 'delete';
1798
1799         write_config($vmid, $conf);
1800     };
1801
1802     lock_container($vmid, 10, $updatefn);
1803
1804     my $storecfg = PVE::Storage::config();
1805
1806     my $del_snap =  sub {
1807
1808         check_lock($conf);
1809
1810         if ($conf->{parent} eq $snapname) {
1811             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1812                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1813             } else {
1814                 delete $conf->{parent};
1815             }
1816         }
1817
1818         delete $conf->{snapshots}->{$snapname};
1819
1820         write_config($vmid, $conf);
1821     };
1822
1823     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1824     my $rootinfo = parse_ct_mountpoint($rootfs);
1825     my $volid = $rootinfo->{volume};
1826
1827     eval {
1828         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1829     };
1830     my $err = $@;
1831
1832     if(!$err || ($err && $force)) {
1833         lock_container($vmid, 10, $del_snap);
1834         if ($err) {
1835             die "Can't delete snapshot: $vmid $snapname $err\n";
1836         }
1837     }
1838 }
1839
1840 sub snapshot_rollback {
1841     my ($vmid, $snapname) = @_;
1842
1843     my $storecfg = PVE::Storage::config();
1844
1845     my $conf = load_config($vmid);
1846
1847     die "you can't rollback if vm is a template\n" if is_template($conf);
1848
1849     my $snap = $conf->{snapshots}->{$snapname};
1850
1851     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1852
1853     my $rootfs = $snap->{rootfs};
1854     my $rootinfo = parse_ct_mountpoint($rootfs);
1855     my $volid = $rootinfo->{volume};
1856
1857     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1858
1859     my $updatefn = sub {
1860
1861         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1862             if $snap->{snapstate};
1863
1864         check_lock($conf);
1865
1866         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1867
1868         die "unable to rollback vm $vmid: vm is running\n"
1869             if check_running($vmid);
1870
1871         $conf->{lock} = 'rollback';
1872
1873         my $forcemachine;
1874
1875         # copy snapshot config to current config
1876
1877         my $tmp_conf = $conf;
1878         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1879         $conf->{snapshots} = $tmp_conf->{snapshots};
1880         delete $conf->{snaptime};
1881         delete $conf->{snapname};
1882         $conf->{parent} = $snapname;
1883
1884         write_config($vmid, $conf);
1885     };
1886
1887     my $unlockfn = sub {
1888         delete $conf->{lock};
1889         write_config($vmid, $conf);
1890     };
1891
1892     lock_container($vmid, 10, $updatefn);
1893
1894     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1895
1896     lock_container($vmid, 5, $unlockfn);
1897 }
1898
1899 sub template_create {
1900     my ($vmid, $conf) = @_;
1901
1902     my $storecfg = PVE::Storage::config();
1903
1904     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1905     my $volid = $rootinfo->{volume};
1906
1907     die "Template feature is not available for '$volid'\n"
1908         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1909
1910     PVE::Storage::activate_volumes($storecfg, [$volid]);
1911
1912     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1913     $rootinfo->{volume} = $template_volid;
1914     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1915
1916     write_config($vmid, $conf);
1917 }
1918
1919 sub is_template {
1920     my ($conf) = @_;
1921
1922     return 1 if defined $conf->{template} && $conf->{template} == 1;
1923 }
1924
1925 sub mountpoint_names {
1926     my ($reverse) = @_;
1927
1928     my @names = ('rootfs');
1929
1930     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1931         push @names, "mp$i";
1932     }
1933
1934     return $reverse ? reverse @names : @names;
1935 }
1936
1937 # The container might have *different* symlinks than the host. realpath/abs_path
1938 # use the actual filesystem to resolve links.
1939 sub sanitize_mountpoint {
1940     my ($mp) = @_;
1941     $mp = '/' . $mp; # we always start with a slash
1942     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1943     $mp =~ s@/\./@@g; # collapse /./
1944     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1945     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1946     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1947     return $mp;
1948 }
1949
1950 sub foreach_mountpoint_full {
1951     my ($conf, $reverse, $func) = @_;
1952
1953     foreach my $key (mountpoint_names($reverse)) {
1954         my $value = $conf->{$key};
1955         next if !defined($value);
1956         my $mountpoint = parse_ct_mountpoint($value, 1);
1957         next if !defined($mountpoint);
1958
1959         # just to be sure: rootfs is /
1960         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1961         $mountpoint->{mp} = sanitize_mountpoint($path);
1962
1963         $path = $mountpoint->{volume};
1964         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1965
1966         &$func($key, $mountpoint);
1967     }
1968 }
1969
1970 sub foreach_mountpoint {
1971     my ($conf, $func) = @_;
1972
1973     foreach_mountpoint_full($conf, 0, $func);
1974 }
1975
1976 sub foreach_mountpoint_reverse {
1977     my ($conf, $func) = @_;
1978
1979     foreach_mountpoint_full($conf, 1, $func);
1980 }
1981
1982 sub check_ct_modify_config_perm {
1983     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1984
1985     return 1 if $authuser ne 'root@pam';
1986
1987     foreach my $opt (@$key_list) {
1988
1989         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1990             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1991         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1992             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1993         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1994             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1995         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1996                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1997             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1998         } else {
1999             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2000         }
2001     }
2002
2003     return 1;
2004 }
2005
2006 sub umount_all {
2007     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2008
2009     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2010     my $volid_list = get_vm_volumes($conf);
2011
2012     foreach_mountpoint_reverse($conf, sub {
2013         my ($ms, $mountpoint) = @_;
2014
2015         my $volid = $mountpoint->{volume};
2016         my $mount = $mountpoint->{mp};
2017
2018         return if !$volid || !$mount;
2019
2020         my $mount_path = "$rootdir/$mount";
2021         $mount_path =~ s!/+!/!g;
2022
2023         return if !PVE::ProcFSTools::is_mounted($mount_path);
2024
2025         eval {
2026             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2027         };
2028         if (my $err = $@) {
2029             if ($noerr) {
2030                 warn $err;
2031             } else {
2032                 die $err;
2033             }
2034         }
2035     });
2036 }
2037
2038 sub mount_all {
2039     my ($vmid, $storage_cfg, $conf) = @_;
2040
2041     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2042     File::Path::make_path($rootdir);
2043
2044     my $volid_list = get_vm_volumes($conf);
2045     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2046
2047     eval {
2048         foreach_mountpoint($conf, sub {
2049             my ($ms, $mountpoint) = @_;
2050
2051             my $volid = $mountpoint->{volume};
2052             my $mount = $mountpoint->{mp};
2053
2054             return if !$volid || !$mount;
2055
2056             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2057             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2058                 PVE::Storage::parse_volname($storage_cfg, $volid);
2059
2060             die "unable to mount base volume - internal error" if $isBase;
2061
2062             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2063         });
2064     };
2065     if (my $err = $@) {
2066         warn "mounting container failed - $err";
2067         umount_all($vmid, $storage_cfg, $conf, 1);
2068     }
2069
2070     return $rootdir;
2071 }
2072
2073
2074 sub mountpoint_mount_path {
2075     my ($mountpoint, $storage_cfg, $snapname) = @_;
2076
2077     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2078 }
2079
2080 my $check_mount_path = sub {
2081     my ($path) = @_;
2082     $path = File::Spec->canonpath($path);
2083     my $real = Cwd::realpath($path);
2084     if ($real ne $path) {
2085         die "mount path modified by symlink: $path != $real";
2086     }
2087 };
2088
2089 # use $rootdir = undef to just return the corresponding mount path
2090 sub mountpoint_mount {
2091     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2092
2093     my $volid = $mountpoint->{volume};
2094     my $mount = $mountpoint->{mp};
2095
2096     return if !$volid || !$mount;
2097
2098     my $mount_path;
2099
2100     if (defined($rootdir)) {
2101         $rootdir =~ s!/+$!!;
2102         $mount_path = "$rootdir/$mount";
2103         $mount_path =~ s!/+!/!g;
2104         &$check_mount_path($mount_path);
2105         File::Path::mkpath($mount_path);
2106     }
2107
2108     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2109
2110     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2111
2112     if ($storage) {
2113
2114         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2115         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2116
2117         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2118             PVE::Storage::parse_volname($storage_cfg, $volid);
2119
2120         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2121
2122         if ($format eq 'subvol') {
2123             if ($mount_path) {
2124                 if ($snapname) {
2125                     if ($scfg->{type} eq 'zfspool') {
2126                         my $path_arg = $path;
2127                         $path_arg =~ s!^/+!!;
2128                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2129                     } else {
2130                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2131                     }
2132                 } else {
2133                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2134                 }
2135             }
2136             return wantarray ? ($path, 0) : $path;
2137         } elsif ($format eq 'raw' || $format eq 'iso') {
2138             my $use_loopdev = 0;
2139             my @extra_opts;
2140             if ($scfg->{path}) {
2141                 push @extra_opts, '-o', 'loop';
2142                 $use_loopdev = 1;
2143             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2144                 # do nothing
2145             } else {
2146                 die "unsupported storage type '$scfg->{type}'\n";
2147             }
2148             if ($mount_path) {
2149                 if ($format eq 'iso') {
2150                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2151                 } elsif ($isBase || defined($snapname)) {
2152                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2153                 } else {
2154                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2155                 }
2156             }
2157             return wantarray ? ($path, $use_loopdev) : $path;
2158         } else {
2159             die "unsupported image format '$format'\n";
2160         }
2161     } elsif ($volid =~ m|^/dev/.+|) {
2162         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2163         return wantarray ? ($volid, 0) : $volid;
2164     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2165         &$check_mount_path($volid);
2166         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2167         return wantarray ? ($volid, 0) : $volid;
2168     }
2169
2170     die "unsupported storage";
2171 }
2172
2173 sub get_vm_volumes {
2174     my ($conf, $excludes) = @_;
2175
2176     my $vollist = [];
2177
2178     foreach_mountpoint($conf, sub {
2179         my ($ms, $mountpoint) = @_;
2180
2181         return if $excludes && $ms eq $excludes;
2182
2183         my $volid = $mountpoint->{volume};
2184
2185         return if !$volid || $volid =~ m|^/|;
2186
2187         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2188         return if !$sid;
2189
2190         push @$vollist, $volid;
2191     });
2192
2193     return $vollist;
2194 }
2195
2196 sub mkfs {
2197     my ($dev) = @_;
2198
2199     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2200 }
2201
2202 sub format_disk {
2203     my ($storage_cfg, $volid) = @_;
2204
2205     if ($volid =~ m!^/dev/.+!) {
2206         mkfs($volid);
2207         return;
2208     }
2209
2210     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2211
2212     die "cannot format volume '$volid' with no storage\n" if !$storage;
2213
2214     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2215
2216     my $path = PVE::Storage::path($storage_cfg, $volid);
2217
2218     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2219         PVE::Storage::parse_volname($storage_cfg, $volid);
2220
2221     die "cannot format volume '$volid' (format == $format)\n"
2222         if $format ne 'raw';
2223
2224     mkfs($path);
2225 }
2226
2227 sub destroy_disks {
2228     my ($storecfg, $vollist) = @_;
2229
2230     foreach my $volid (@$vollist) {
2231         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2232         warn $@ if $@;
2233     }
2234 }
2235
2236 sub create_disks {
2237     my ($storecfg, $vmid, $settings, $conf) = @_;
2238
2239     my $vollist = [];
2240
2241     eval {
2242         foreach_mountpoint($settings, sub {
2243             my ($ms, $mountpoint) = @_;
2244
2245             my $volid = $mountpoint->{volume};
2246             my $mp = $mountpoint->{mp};
2247
2248             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2249
2250             return if !$storage;
2251
2252             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2253                 my ($storeid, $size_gb) = ($1, $2);
2254
2255                 my $size_kb = int(${size_gb}*1024) * 1024;
2256
2257                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2258                 # fixme: use better naming ct-$vmid-disk-X.raw?
2259
2260                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2261                     if ($size_kb > 0) {
2262                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2263                                                            undef, $size_kb);
2264                         format_disk($storecfg, $volid);
2265                     } else {
2266                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2267                                                            undef, 0);
2268                     }
2269                 } elsif ($scfg->{type} eq 'zfspool') {
2270
2271                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2272                                                        undef, $size_kb);
2273                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2274
2275                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2276                     format_disk($storecfg, $volid);
2277
2278                 } elsif ($scfg->{type} eq 'rbd') {
2279
2280                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2281                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2282                     format_disk($storecfg, $volid);
2283                 } else {
2284                     die "unable to create containers on storage type '$scfg->{type}'\n";
2285                 }
2286                 push @$vollist, $volid;
2287                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2288                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2289             } else {
2290                 # use specified/existing volid
2291             }
2292         });
2293     };
2294     # free allocated images on error
2295     if (my $err = $@) {
2296         destroy_disks($storecfg, $vollist);
2297         die $err;
2298     }
2299     return $vollist;
2300 }
2301
2302 # bash completion helper
2303
2304 sub complete_os_templates {
2305     my ($cmdname, $pname, $cvalue) = @_;
2306
2307     my $cfg = PVE::Storage::config();
2308
2309     my $storeid;
2310
2311     if ($cvalue =~ m/^([^:]+):/) {
2312         $storeid = $1;
2313     }
2314
2315     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2316     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2317
2318     my $res = [];
2319     foreach my $id (keys %$data) {
2320         foreach my $item (@{$data->{$id}}) {
2321             push @$res, $item->{volid} if defined($item->{volid});
2322         }
2323     }
2324
2325     return $res;
2326 }
2327
2328 my $complete_ctid_full = sub {
2329     my ($running) = @_;
2330
2331     my $idlist = vmstatus();
2332
2333     my $active_hash = list_active_containers();
2334
2335     my $res = [];
2336
2337     foreach my $id (keys %$idlist) {
2338         my $d = $idlist->{$id};
2339         if (defined($running)) {
2340             next if $d->{template};
2341             next if $running && !$active_hash->{$id};
2342             next if !$running && $active_hash->{$id};
2343         }
2344         push @$res, $id;
2345
2346     }
2347     return $res;
2348 };
2349
2350 sub complete_ctid {
2351     return &$complete_ctid_full();
2352 }
2353
2354 sub complete_ctid_stopped {
2355     return &$complete_ctid_full(0);
2356 }
2357
2358 sub complete_ctid_running {
2359     return &$complete_ctid_full(1);
2360 }
2361
2362 1;