src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21
  22 use Data::Dumper;
  23
  24 my $nodename = PVE::INotify::nodename();
  25
  26 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  27
  28 my $rootfs_desc = {
  29     volume => {
  30         type => 'string',
  31         default_key => 1,
  32         format_description => 'volume',
  33         description => 'Volume, device or directory to mount into the container.',
  34     },
  35     backup => {
  36         type => 'boolean',
  37         format_description => '[1|0]',
  38         description => 'Whether to include the mountpoint in backups.',
  39         optional => 1,
  40     },
  41     size => {
  42         type => 'string',
  43         format_description => 'DiskSize',
  44         pattern => '\d+[TGMK]?',
  45         description => 'Volume size (read only value).',
  46         optional => 1,
  47     },
  48 };
  49
  50 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  51     type => 'string', format => $rootfs_desc,
  52     description => "Use volume as container root.",
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  57     description => "The name of the snapshot.",
  58     type => 'string', format => 'pve-configid',
  59     maxLength => 40,
  60 });
  61
  62 my $confdesc = {
  63     lock => {
  64         optional => 1,
  65         type => 'string',
  66         description => "Lock/unlock the VM.",
  67         enum => [qw(migrate backup snapshot rollback)],
  68     },
  69     onboot => {
  70         optional => 1,
  71         type => 'boolean',
  72         description => "Specifies whether a VM will be started during system bootup.",
  73         default => 0,
  74     },
  75     startup => get_standard_option('pve-startup-order'),
  76     template => {
  77         optional => 1,
  78         type => 'boolean',
  79         description => "Enable/disable Template.",
  80         default => 0,
  81     },
  82     arch => {
  83         optional => 1,
  84         type => 'string',
  85         enum => ['amd64', 'i386'],
  86         description => "OS architecture type.",
  87         default => 'amd64',
  88     },
  89     ostype => {
  90         optional => 1,
  91         type => 'string',
  92         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  93         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  94     },
  95     console => {
  96         optional => 1,
  97         type => 'boolean',
  98         description => "Attach a console device (/dev/console) to the container.",
  99         default => 1,
 100     },
 101     tty => {
 102         optional => 1,
 103         type => 'integer',
 104         description => "Specify the number of tty available to the container",
 105         minimum => 0,
 106         maximum => 6,
 107         default => 2,
 108     },
 109     cpulimit => {
 110         optional => 1,
 111         type => 'number',
 112         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 113         minimum => 0,
 114         maximum => 128,
 115         default => 0,
 116     },
 117     cpuunits => {
 118         optional => 1,
 119         type => 'integer',
 120         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 121         minimum => 0,
 122         maximum => 500000,
 123         default => 1024,
 124     },
 125     memory => {
 126         optional => 1,
 127         type => 'integer',
 128         description => "Amount of RAM for the VM in MB.",
 129         minimum => 16,
 130         default => 512,
 131     },
 132     swap => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "Amount of SWAP for the VM in MB.",
 136         minimum => 0,
 137         default => 512,
 138     },
 139     hostname => {
 140         optional => 1,
 141         description => "Set a host name for the container.",
 142         type => 'string', format => 'dns-name',
 143         maxLength => 255,
 144     },
 145     description => {
 146         optional => 1,
 147         type => 'string',
 148         description => "Container description. Only used on the configuration web interface.",
 149     },
 150     searchdomain => {
 151         optional => 1,
 152         type => 'string', format => 'dns-name-list',
 153         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 154     },
 155     nameserver => {
 156         optional => 1,
 157         type => 'string', format => 'address-list',
 158         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 159     },
 160     rootfs => get_standard_option('pve-ct-rootfs'),
 161     parent => {
 162         optional => 1,
 163         type => 'string', format => 'pve-configid',
 164         maxLength => 40,
 165         description => "Parent snapshot name. This is used internally, and should not be modified.",
 166     },
 167     snaptime => {
 168         optional => 1,
 169         description => "Timestamp for snapshots.",
 170         type => 'integer',
 171         minimum => 0,
 172     },
 173     cmode => {
 174         optional => 1,
 175         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 176         type => 'string',
 177         enum => ['shell', 'console', 'tty'],
 178         default => 'tty',
 179     },
 180     protection => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 184         default => 0,
 185     },
 186 };
 187
 188 my $valid_lxc_conf_keys = {
 189     'lxc.include' => 1,
 190     'lxc.arch' => 1,
 191     'lxc.utsname' => 1,
 192     'lxc.haltsignal' => 1,
 193     'lxc.rebootsignal' => 1,
 194     'lxc.stopsignal' => 1,
 195     'lxc.init_cmd' => 1,
 196     'lxc.network.type' => 1,
 197     'lxc.network.flags' => 1,
 198     'lxc.network.link' => 1,
 199     'lxc.network.mtu' => 1,
 200     'lxc.network.name' => 1,
 201     'lxc.network.hwaddr' => 1,
 202     'lxc.network.ipv4' => 1,
 203     'lxc.network.ipv4.gateway' => 1,
 204     'lxc.network.ipv6' => 1,
 205     'lxc.network.ipv6.gateway' => 1,
 206     'lxc.network.script.up' => 1,
 207     'lxc.network.script.down' => 1,
 208     'lxc.pts' => 1,
 209     'lxc.console.logfile' => 1,
 210     'lxc.console' => 1,
 211     'lxc.tty' => 1,
 212     'lxc.devttydir' => 1,
 213     'lxc.hook.autodev' => 1,
 214     'lxc.autodev' => 1,
 215     'lxc.kmsg' => 1,
 216     'lxc.mount' => 1,
 217     'lxc.mount.entry' => 1,
 218     'lxc.mount.auto' => 1,
 219     'lxc.rootfs' => 1,
 220     'lxc.rootfs.mount' => 1,
 221     'lxc.rootfs.options' => 1,
 222     # lxc.cgroup.*
 223     'lxc.cap.drop' => 1,
 224     'lxc.cap.keep' => 1,
 225     'lxc.aa_profile' => 1,
 226     'lxc.aa_allow_incomplete' => 1,
 227     'lxc.se_context' => 1,
 228     'lxc.seccomp' => 1,
 229     'lxc.id_map' => 1,
 230     'lxc.hook.pre-start' => 1,
 231     'lxc.hook.pre-mount' => 1,
 232     'lxc.hook.mount' => 1,
 233     'lxc.hook.start' => 1,
 234     'lxc.hook.post-stop' => 1,
 235     'lxc.hook.clone' => 1,
 236     'lxc.hook.destroy' => 1,
 237     'lxc.loglevel' => 1,
 238     'lxc.logfile' => 1,
 239     'lxc.start.auto' => 1,
 240     'lxc.start.delay' => 1,
 241     'lxc.start.order' => 1,
 242     'lxc.group' => 1,
 243     'lxc.environment' => 1,
 244     'lxc.' => 1,
 245     'lxc.' => 1,
 246     'lxc.' => 1,
 247     'lxc.' => 1,
 248 };
 249
 250 my $netconf_desc = {
 251     type => {
 252         type => 'string',
 253         optional => 1,
 254         description => "Network interface type.",
 255         enum => [qw(veth)],
 256     },
 257     name => {
 258         type => 'string',
 259         format_description => 'String',
 260         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 261         pattern => '[-_.\w\d]+',
 262     },
 263     bridge => {
 264         type => 'string',
 265         format_description => 'vmbr<Number>',
 266         description => 'Bridge to attach the network device to.',
 267         pattern => '[-_.\w\d]+',
 268     },
 269     hwaddr => {
 270         type => 'string',
 271         format_description => 'MAC',
 272         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 273         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 274         optional => 1,
 275     },
 276     mtu => {
 277         type => 'integer',
 278         format_description => 'Number',
 279         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 280         optional => 1,
 281     },
 282     ip => {
 283         type => 'string',
 284         format => 'pve-ipv4-config',
 285         format_description => 'IPv4Format/CIDR',
 286         description => 'IPv4 address in CIDR format.',
 287         optional => 1,
 288     },
 289     gw => {
 290         type => 'string',
 291         format => 'ipv4',
 292         format_description => 'GatewayIPv4',
 293         description => 'Default gateway for IPv4 traffic.',
 294         optional => 1,
 295     },
 296     ip6 => {
 297         type => 'string',
 298         format => 'pve-ipv6-config',
 299         format_description => 'IPv6Format/CIDR',
 300         description => 'IPv6 address in CIDR format.',
 301         optional => 1,
 302     },
 303     gw6 => {
 304         type => 'string',
 305         format => 'ipv6',
 306         format_description => 'GatewayIPv6',
 307         description => 'Default gateway for IPv6 traffic.',
 308         optional => 1,
 309     },
 310     firewall => {
 311         type => 'boolean',
 312         format_description => '[1|0]',
 313         description => "Controls whether this interface's firewall rules should be used.",
 314         optional => 1,
 315     },
 316     tag => {
 317         type => 'integer',
 318         format_description => 'VlanNo',
 319         minimum => '2',
 320         maximum => '4094',
 321         description => "VLAN tag foro this interface.",
 322         optional => 1,
 323     },
 324 };
 325 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 326
 327 my $MAX_LXC_NETWORKS = 10;
 328 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 329     $confdesc->{"net$i"} = {
 330         optional => 1,
 331         type => 'string', format => $netconf_desc,
 332         description => "Specifies network interfaces for the container.",
 333     };
 334 }
 335
 336 my $mp_desc = {
 337     %$rootfs_desc,
 338     mp => {
 339         type => 'string',
 340         format_description => 'Path',
 341         description => 'Path to the mountpoint as seen from inside the container.',
 342         optional => 1,
 343     },
 344 };
 345 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 346
 347 my $MAX_MOUNT_POINTS = 10;
 348 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 349     $confdesc->{"mp$i"} = {
 350         optional => 1,
 351         type => 'string', format => $mp_desc,
 352         description => "Use volume as container mount point (experimental feature).",
 353         optional => 1,
 354     };
 355 }
 356
 357 sub write_pct_config {
 358     my ($filename, $conf) = @_;
 359
 360     delete $conf->{snapstate}; # just to be sure
 361
 362     my $generate_raw_config = sub {
 363         my ($conf) = @_;
 364
 365         my $raw = '';
 366
 367         # add description as comment to top of file
 368         my $descr = $conf->{description} || '';
 369         foreach my $cl (split(/\n/, $descr)) {
 370             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 371         }
 372
 373         foreach my $key (sort keys %$conf) {
 374             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 375                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 376             my $value = $conf->{$key};
 377             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 378             $raw .= "$key: $value\n";
 379         }
 380
 381         if (my $lxcconf = $conf->{lxc}) {
 382             foreach my $entry (@$lxcconf) {
 383                 my ($k, $v) = @$entry;
 384                 $raw .= "$k: $v\n";
 385             }
 386         }
 387
 388         return $raw;
 389     };
 390
 391     my $raw = &$generate_raw_config($conf);
 392
 393     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 394         $raw .= "\n[$snapname]\n";
 395         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 396     }
 397
 398     return $raw;
 399 }
 400
 401 sub check_type {
 402     my ($key, $value) = @_;
 403
 404     die "unknown setting '$key'\n" if !$confdesc->{$key};
 405
 406     my $type = $confdesc->{$key}->{type};
 407
 408     if (!defined($value)) {
 409         die "got undefined value\n";
 410     }
 411
 412     if ($value =~ m/[\n\r]/) {
 413         die "property contains a line feed\n";
 414     }
 415
 416     if ($type eq 'boolean') {
 417         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 418         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 419         die "type check ('boolean') failed - got '$value'\n";
 420     } elsif ($type eq 'integer') {
 421         return int($1) if $value =~ m/^(\d+)$/;
 422         die "type check ('integer') failed - got '$value'\n";
 423     } elsif ($type eq 'number') {
 424         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 425         die "type check ('number') failed - got '$value'\n";
 426     } elsif ($type eq 'string') {
 427         if (my $fmt = $confdesc->{$key}->{format}) {
 428             PVE::JSONSchema::check_format($fmt, $value);
 429             return $value;
 430         }
 431         return $value;
 432     } else {
 433         die "internal error"
 434     }
 435 }
 436
 437 sub parse_pct_config {
 438     my ($filename, $raw) = @_;
 439
 440     return undef if !defined($raw);
 441
 442     my $res = {
 443         digest => Digest::SHA::sha1_hex($raw),
 444         snapshots => {},
 445     };
 446
 447     $filename =~ m|/lxc/(\d+).conf$|
 448         || die "got strange filename '$filename'";
 449
 450     my $vmid = $1;
 451
 452     my $conf = $res;
 453     my $descr = '';
 454     my $section = '';
 455
 456     my @lines = split(/\n/, $raw);
 457     foreach my $line (@lines) {
 458         next if $line =~ m/^\s*$/;
 459
 460         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 461             $section = $1;
 462             $conf->{description} = $descr if $descr;
 463             $descr = '';
 464             $conf = $res->{snapshots}->{$section} = {};
 465             next;
 466         }
 467
 468         if ($line =~ m/^\#(.*)\s*$/) {
 469             $descr .= PVE::Tools::decode_text($1) . "\n";
 470             next;
 471         }
 472
 473         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 474             my $key = $1;
 475             my $value = $3;
 476             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 477                 push @{$conf->{lxc}}, [$key, $value];
 478             } else {
 479                 warn "vm $vmid - unable to parse config: $line\n";
 480             }
 481         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 482             $descr .= PVE::Tools::decode_text($2);
 483         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 484             $conf->{snapstate} = $1;
 485         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 486             my $key = $1;
 487             my $value = $2;
 488             eval { $value = check_type($key, $value); };
 489             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 490             $conf->{$key} = $value;
 491         } else {
 492             warn "vm $vmid - unable to parse config: $line\n";
 493         }
 494     }
 495
 496     $conf->{description} = $descr if $descr;
 497
 498     delete $res->{snapstate}; # just to be sure
 499
 500     return $res;
 501 }
 502
 503 sub config_list {
 504     my $vmlist = PVE::Cluster::get_vmlist();
 505     my $res = {};
 506     return $res if !$vmlist || !$vmlist->{ids};
 507     my $ids = $vmlist->{ids};
 508
 509     foreach my $vmid (keys %$ids) {
 510         next if !$vmid; # skip CT0
 511         my $d = $ids->{$vmid};
 512         next if !$d->{node} || $d->{node} ne $nodename;
 513         next if !$d->{type} || $d->{type} ne 'lxc';
 514         $res->{$vmid}->{type} = 'lxc';
 515     }
 516     return $res;
 517 }
 518
 519 sub cfs_config_path {
 520     my ($vmid, $node) = @_;
 521
 522     $node = $nodename if !$node;
 523     return "nodes/$node/lxc/$vmid.conf";
 524 }
 525
 526 sub config_file {
 527     my ($vmid, $node) = @_;
 528
 529     my $cfspath = cfs_config_path($vmid, $node);
 530     return "/etc/pve/$cfspath";
 531 }
 532
 533 sub load_config {
 534     my ($vmid, $node) = @_;
 535
 536     $node = $nodename if !$node;
 537     my $cfspath = cfs_config_path($vmid, $node);
 538
 539     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 540     die "container $vmid does not exists\n" if !defined($conf);
 541
 542     return $conf;
 543 }
 544
 545 sub create_config {
 546     my ($vmid, $conf) = @_;
 547
 548     my $dir = "/etc/pve/nodes/$nodename/lxc";
 549     mkdir $dir;
 550
 551     write_config($vmid, $conf);
 552 }
 553
 554 sub destroy_config {
 555     my ($vmid) = @_;
 556
 557     unlink config_file($vmid, $nodename);
 558 }
 559
 560 sub write_config {
 561     my ($vmid, $conf) = @_;
 562
 563     my $cfspath = cfs_config_path($vmid);
 564
 565     PVE::Cluster::cfs_write_file($cfspath, $conf);
 566 }
 567
 568 # flock: we use one file handle per process, so lock file
 569 # can be called multiple times and succeeds for the same process.
 570
 571 my $lock_handles =  {};
 572 my $lockdir = "/run/lock/lxc";
 573
 574 sub lock_filename {
 575     my ($vmid) = @_;
 576
 577     return "$lockdir/pve-config-${vmid}.lock";
 578 }
 579
 580 sub lock_aquire {
 581     my ($vmid, $timeout) = @_;
 582
 583     $timeout = 10 if !$timeout;
 584     my $mode = LOCK_EX;
 585
 586     my $filename = lock_filename($vmid);
 587
 588     mkdir $lockdir if !-d $lockdir;
 589
 590     my $lock_func = sub {
 591         if (!$lock_handles->{$$}->{$filename}) {
 592             my $fh = new IO::File(">>$filename") ||
 593                 die "can't open file - $!\n";
 594             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 595         }
 596
 597         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 598             print STDERR "trying to aquire lock...";
 599             my $success;
 600             while(1) {
 601                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 602                 # try again on EINTR (see bug #273)
 603                 if ($success || ($! != EINTR)) {
 604                     last;
 605                 }
 606             }
 607             if (!$success) {
 608                 print STDERR " failed\n";
 609                 die "can't aquire lock - $!\n";
 610             }
 611
 612             print STDERR " OK\n";
 613         }
 614
 615         $lock_handles->{$$}->{$filename}->{refcount}++;
 616     };
 617
 618     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 619     my $err = $@;
 620     if ($err) {
 621         die "can't lock file '$filename' - $err";
 622     }
 623 }
 624
 625 sub lock_release {
 626     my ($vmid) = @_;
 627
 628     my $filename = lock_filename($vmid);
 629
 630     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 631         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 632         if ($refcount <= 0) {
 633             $lock_handles->{$$}->{$filename} = undef;
 634             close ($fh);
 635         }
 636     }
 637 }
 638
 639 sub lock_container {
 640     my ($vmid, $timeout, $code, @param) = @_;
 641
 642     my $res;
 643
 644     lock_aquire($vmid, $timeout);
 645     eval { $res = &$code(@param) };
 646     my $err = $@;
 647     lock_release($vmid);
 648
 649     die $err if $err;
 650
 651     return $res;
 652 }
 653
 654 sub option_exists {
 655     my ($name) = @_;
 656
 657     return defined($confdesc->{$name});
 658 }
 659
 660 # add JSON properties for create and set function
 661 sub json_config_properties {
 662     my $prop = shift;
 663
 664     foreach my $opt (keys %$confdesc) {
 665         next if $opt eq 'parent' || $opt eq 'snaptime';
 666         next if $prop->{$opt};
 667         $prop->{$opt} = $confdesc->{$opt};
 668     }
 669
 670     return $prop;
 671 }
 672
 673 sub json_config_properties_no_rootfs {
 674     my $prop = shift;
 675
 676     foreach my $opt (keys %$confdesc) {
 677         next if $prop->{$opt};
 678         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 679         $prop->{$opt} = $confdesc->{$opt};
 680     }
 681
 682     return $prop;
 683 }
 684
 685 # container status helpers
 686
 687 sub list_active_containers {
 688
 689     my $filename = "/proc/net/unix";
 690
 691     # similar test is used by lcxcontainers.c: list_active_containers
 692     my $res = {};
 693
 694     my $fh = IO::File->new ($filename, "r");
 695     return $res if !$fh;
 696
 697     while (defined(my $line = <$fh>)) {
 698         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 699             my $path = $1;
 700             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 701                 $res->{$1} = 1;
 702             }
 703         }
 704     }
 705
 706     close($fh);
 707
 708     return $res;
 709 }
 710
 711 # warning: this is slow
 712 sub check_running {
 713     my ($vmid) = @_;
 714
 715     my $active_hash = list_active_containers();
 716
 717     return 1 if defined($active_hash->{$vmid});
 718
 719     return undef;
 720 }
 721
 722 sub get_container_disk_usage {
 723     my ($vmid) = @_;
 724
 725     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 726
 727     my $res = {
 728         total => 0,
 729         used => 0,
 730         avail => 0,
 731     };
 732
 733     my $parser = sub {
 734         my $line = shift;
 735         if (my ($fsid, $total, $used, $avail) = $line =~
 736             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 737             $res = {
 738                 total => $total,
 739                 used => $used,
 740                 avail => $avail,
 741             };
 742         }
 743     };
 744     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 745     warn $@ if $@;
 746
 747     return $res;
 748 }
 749
 750 sub vmstatus {
 751     my ($opt_vmid) = @_;
 752
 753     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 754
 755     my $active_hash = list_active_containers();
 756
 757     foreach my $vmid (keys %$list) {
 758         my $d = $list->{$vmid};
 759
 760         my $running = defined($active_hash->{$vmid});
 761
 762         $d->{status} = $running ? 'running' : 'stopped';
 763
 764         my $cfspath = cfs_config_path($vmid);
 765         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 766
 767         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 768         $d->{name} =~ s/[\s]//g;
 769
 770         $d->{cpus} = $conf->{cpulimit} // 0;
 771
 772         if ($running) {
 773             my $res = get_container_disk_usage($vmid);
 774             $d->{disk} = $res->{used};
 775             $d->{maxdisk} = $res->{total};
 776         } else {
 777             $d->{disk} = 0;
 778             # use 4GB by default ??
 779             if (my $rootfs = $conf->{rootfs}) {
 780                 my $rootinfo = parse_ct_mountpoint($rootfs);
 781                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 782             } else {
 783                 $d->{maxdisk} = 4*1024*1024*1024;
 784             }
 785         }
 786
 787         $d->{mem} = 0;
 788         $d->{swap} = 0;
 789         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 790         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 791
 792         $d->{uptime} = 0;
 793         $d->{cpu} = 0;
 794
 795         $d->{netout} = 0;
 796         $d->{netin} = 0;
 797
 798         $d->{diskread} = 0;
 799         $d->{diskwrite} = 0;
 800
 801         $d->{template} = is_template($conf);
 802     }
 803
 804     foreach my $vmid (keys %$list) {
 805         my $d = $list->{$vmid};
 806         next if $d->{status} ne 'running';
 807
 808         $d->{uptime} = 100; # fixme:
 809
 810         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 811         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 812
 813         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 814         my @bytes = split(/\n/, $blkio_bytes);
 815         foreach my $byte (@bytes) {
 816             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 817                 $d->{diskread} = $2 if $key eq 'Read';
 818                 $d->{diskwrite} = $2 if $key eq 'Write';
 819             }
 820         }
 821     }
 822
 823     return $list;
 824 }
 825
 826 my $parse_size = sub {
 827     my ($value) = @_;
 828
 829     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 830     my ($size, $unit) = ($1, $3);
 831     if ($unit) {
 832         if ($unit eq 'K') {
 833             $size = $size * 1024;
 834         } elsif ($unit eq 'M') {
 835             $size = $size * 1024 * 1024;
 836         } elsif ($unit eq 'G') {
 837             $size = $size * 1024 * 1024 * 1024;
 838         }
 839     }
 840     return int($size);
 841 };
 842
 843 my $format_size = sub {
 844     my ($size) = @_;
 845
 846     $size = int($size);
 847
 848     my $kb = int($size/1024);
 849     return $size if $kb*1024 != $size;
 850
 851     my $mb = int($kb/1024);
 852     return "${kb}K" if $mb*1024 != $kb;
 853
 854     my $gb = int($mb/1024);
 855     return "${mb}M" if $gb*1024 != $mb;
 856
 857     return "${gb}G";
 858 };
 859
 860 sub parse_ct_mountpoint {
 861     my ($data) = @_;
 862
 863     $data //= '';
 864
 865     my $res;
 866     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 867     if ($@) {
 868         warn $@;
 869         return undef;
 870     }
 871
 872     return undef if !defined($res->{volume});
 873
 874     if ($res->{size}) {
 875         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 876     }
 877
 878     return $res;
 879 }
 880
 881 sub print_ct_mountpoint {
 882     my ($info, $nomp) = @_;
 883
 884     my $opts = '';
 885
 886     die "missing volume\n" if !$info->{volume};
 887
 888     foreach my $o (qw(backup)) {
 889         $opts .= ",$o=$info->{$o}" if defined($info->{$o});
 890     }
 891
 892     if ($info->{size}) {
 893         $opts .= ",size=" . &$format_size($info->{size});
 894     }
 895
 896     $opts .= ",mp=$info->{mp}" if !$nomp;
 897
 898     return "$info->{volume}$opts";
 899 }
 900
 901 sub print_lxc_network {
 902     my $net = shift;
 903
 904     die "no network name defined\n" if !$net->{name};
 905
 906     my $res = "name=$net->{name}";
 907
 908     foreach my $k (qw(hwaddr mtu bridge ip gw ip6 gw6 firewall tag)) {
 909         next if !defined($net->{$k});
 910         $res .= ",$k=$net->{$k}";
 911     }
 912
 913     return $res;
 914 }
 915
 916 sub parse_lxc_network {
 917     my ($data) = @_;
 918
 919     my $res = {};
 920
 921     return $res if !$data;
 922
 923     eval { $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data) };
 924     if ($@) {
 925         warn $@;
 926         return undef;
 927     }
 928
 929     $res->{type} = 'veth';
 930     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 931
 932     return $res;
 933 }
 934
 935 sub read_cgroup_value {
 936     my ($group, $vmid, $name, $full) = @_;
 937
 938     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 939
 940     return PVE::Tools::file_get_contents($path) if $full;
 941
 942     return PVE::Tools::file_read_firstline($path);
 943 }
 944
 945 sub write_cgroup_value {
 946    my ($group, $vmid, $name, $value) = @_;
 947
 948    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 949    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 950
 951 }
 952
 953 sub find_lxc_console_pids {
 954
 955     my $res = {};
 956
 957     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 958         my ($pid) = @_;
 959
 960         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 961         return if !$cmdline;
 962
 963         my @args = split(/\0/, $cmdline);
 964
 965         # serach for lxc-console -n <vmid>
 966         return if scalar(@args) != 3;
 967         return if $args[1] ne '-n';
 968         return if $args[2] !~ m/^\d+$/;
 969         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 970
 971         my $vmid = $args[2];
 972
 973         push @{$res->{$vmid}}, $pid;
 974     });
 975
 976     return $res;
 977 }
 978
 979 sub find_lxc_pid {
 980     my ($vmid) = @_;
 981
 982     my $pid = undef;
 983     my $parser = sub {
 984         my $line = shift;
 985         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 986     };
 987     PVE::Tools::run_command(['lxc-info', '-n', $vmid], outfunc => $parser);
 988
 989     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 990
 991     return $pid;
 992 }
 993
 994 my $ipv4_reverse_mask = [
 995     '0.0.0.0',
 996     '128.0.0.0',
 997     '192.0.0.0',
 998     '224.0.0.0',
 999     '240.0.0.0',
1000     '248.0.0.0',
1001     '252.0.0.0',
1002     '254.0.0.0',
1003     '255.0.0.0',
1004     '255.128.0.0',
1005     '255.192.0.0',
1006     '255.224.0.0',
1007     '255.240.0.0',
1008     '255.248.0.0',
1009     '255.252.0.0',
1010     '255.254.0.0',
1011     '255.255.0.0',
1012     '255.255.128.0',
1013     '255.255.192.0',
1014     '255.255.224.0',
1015     '255.255.240.0',
1016     '255.255.248.0',
1017     '255.255.252.0',
1018     '255.255.254.0',
1019     '255.255.255.0',
1020     '255.255.255.128',
1021     '255.255.255.192',
1022     '255.255.255.224',
1023     '255.255.255.240',
1024     '255.255.255.248',
1025     '255.255.255.252',
1026     '255.255.255.254',
1027     '255.255.255.255',
1028 ];
1029
1030 # Note: we cannot use Net:IP, because that only allows strict
1031 # CIDR networks
1032 sub parse_ipv4_cidr {
1033     my ($cidr, $noerr) = @_;
1034
1035     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
1036         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
1037     }
1038
1039     return undef if $noerr;
1040
1041     die "unable to parse ipv4 address/mask\n";
1042 }
1043
1044 sub check_lock {
1045     my ($conf) = @_;
1046
1047     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1048 }
1049
1050 sub check_protection {
1051     my ($vm_conf, $err_msg) = @_;
1052
1053     if ($vm_conf->{protection}) {
1054         die "$err_msg - protection mode enabled\n";
1055     }
1056 }
1057
1058 sub update_lxc_config {
1059     my ($storage_cfg, $vmid, $conf) = @_;
1060
1061     my $dir = "/var/lib/lxc/$vmid";
1062
1063     if ($conf->{template}) {
1064
1065         unlink "$dir/config";
1066
1067         return;
1068     }
1069
1070     my $raw = '';
1071
1072     die "missing 'arch' - internal error" if !$conf->{arch};
1073     $raw .= "lxc.arch = $conf->{arch}\n";
1074
1075     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1076     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1077         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1078     } else {
1079         die "implement me";
1080     }
1081
1082     if (!has_dev_console($conf)) {
1083         $raw .= "lxc.console = none\n";
1084         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1085     }
1086
1087     my $ttycount = get_tty_count($conf);
1088     $raw .= "lxc.tty = $ttycount\n";
1089
1090     # some init scripts expects a linux terminal (turnkey).
1091     $raw .= "lxc.environment = TERM=linux\n";
1092
1093     my $utsname = $conf->{hostname} || "CT$vmid";
1094     $raw .= "lxc.utsname = $utsname\n";
1095
1096     my $memory = $conf->{memory} || 512;
1097     my $swap = $conf->{swap} // 0;
1098
1099     my $lxcmem = int($memory*1024*1024);
1100     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1101
1102     my $lxcswap = int(($memory + $swap)*1024*1024);
1103     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1104
1105     if (my $cpulimit = $conf->{cpulimit}) {
1106         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1107         my $value = int(100000*$cpulimit);
1108         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1109     }
1110
1111     my $shares = $conf->{cpuunits} || 1024;
1112     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1113
1114     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1115     $mountpoint->{mp} = '/';
1116
1117     my ($path, $use_loopdev) = mountpoint_mount_path($mountpoint, $storage_cfg);
1118     $path = "loop:$path" if $use_loopdev;
1119
1120     $raw .= "lxc.rootfs = $path\n";
1121
1122     my $netcount = 0;
1123     foreach my $k (keys %$conf) {
1124         next if $k !~ m/^net(\d+)$/;
1125         my $ind = $1;
1126         my $d = parse_lxc_network($conf->{$k});
1127         $netcount++;
1128         $raw .= "lxc.network.type = veth\n";
1129         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1130         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1131         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1132         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1133     }
1134
1135     if (my $lxcconf = $conf->{lxc}) {
1136         foreach my $entry (@$lxcconf) {
1137             my ($k, $v) = @$entry;
1138             $netcount++ if $k eq 'lxc.network.type';
1139             $raw .= "$k = $v\n";
1140         }
1141     }
1142
1143     $raw .= "lxc.network.type = empty\n" if !$netcount;
1144
1145     File::Path::mkpath("$dir/rootfs");
1146
1147     PVE::Tools::file_set_contents("$dir/config", $raw);
1148 }
1149
1150 # verify and cleanup nameserver list (replace \0 with ' ')
1151 sub verify_nameserver_list {
1152     my ($nameserver_list) = @_;
1153
1154     my @list = ();
1155     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1156         PVE::JSONSchema::pve_verify_ip($server);
1157         push @list, $server;
1158     }
1159
1160     return join(' ', @list);
1161 }
1162
1163 sub verify_searchdomain_list {
1164     my ($searchdomain_list) = @_;
1165
1166     my @list = ();
1167     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1168         # todo: should we add checks for valid dns domains?
1169         push @list, $server;
1170     }
1171
1172     return join(' ', @list);
1173 }
1174
1175 sub update_pct_config {
1176     my ($vmid, $conf, $running, $param, $delete) = @_;
1177
1178     my @nohotplug;
1179
1180     my $new_disks = 0;
1181
1182     my $rootdir;
1183     if ($running) {
1184         my $pid = find_lxc_pid($vmid);
1185         $rootdir = "/proc/$pid/root";
1186     }
1187
1188     if (defined($delete)) {
1189         foreach my $opt (@$delete) {
1190             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1191                 die "unable to delete required option '$opt'\n";
1192             } elsif ($opt eq 'swap') {
1193                 delete $conf->{$opt};
1194                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1195             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1196                 delete $conf->{$opt};
1197             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1198                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1199                 delete $conf->{$opt};
1200                 push @nohotplug, $opt;
1201                 next if $running;
1202             } elsif ($opt =~ m/^net(\d)$/) {
1203                 delete $conf->{$opt};
1204                 next if !$running;
1205                 my $netid = $1;
1206                 PVE::Network::veth_delete("veth${vmid}i$netid");
1207             } elsif ($opt eq 'protection') {
1208                 delete $conf->{$opt};
1209             } elsif ($opt =~ m/^mp(\d+)$/) {
1210                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1211                 delete $conf->{$opt};
1212                 push @nohotplug, $opt;
1213                 next if $running;
1214             } else {
1215                 die "implement me"
1216             }
1217             write_config($vmid, $conf) if $running;
1218         }
1219     }
1220
1221     # There's no separate swap size to configure, there's memory and "total"
1222     # memory (iow. memory+swap). This means we have to change them together.
1223     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1224     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1225     if (defined($wanted_memory) || defined($wanted_swap)) {
1226
1227         $wanted_memory //= ($conf->{memory} || 512);
1228         $wanted_swap //=  ($conf->{swap} || 0);
1229
1230         my $total = $wanted_memory + $wanted_swap;
1231         if ($running) {
1232             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1233             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1234         }
1235         $conf->{memory} = $wanted_memory;
1236         $conf->{swap} = $wanted_swap;
1237
1238         write_config($vmid, $conf) if $running;
1239     }
1240
1241     foreach my $opt (keys %$param) {
1242         my $value = $param->{$opt};
1243         if ($opt eq 'hostname') {
1244             $conf->{$opt} = $value;
1245         } elsif ($opt eq 'onboot') {
1246             $conf->{$opt} = $value ? 1 : 0;
1247         } elsif ($opt eq 'startup') {
1248             $conf->{$opt} = $value;
1249         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1250             $conf->{$opt} = $value;
1251             push @nohotplug, $opt;
1252             next if $running;
1253         } elsif ($opt eq 'nameserver') {
1254             my $list = verify_nameserver_list($value);
1255             $conf->{$opt} = $list;
1256             push @nohotplug, $opt;
1257             next if $running;
1258         } elsif ($opt eq 'searchdomain') {
1259             my $list = verify_searchdomain_list($value);
1260             $conf->{$opt} = $list;
1261             push @nohotplug, $opt;
1262             next if $running;
1263         } elsif ($opt eq 'cpulimit') {
1264             $conf->{$opt} = $value;
1265             push @nohotplug, $opt; # fixme: hotplug
1266             next;
1267         } elsif ($opt eq 'cpuunits') {
1268             $conf->{$opt} = $value;
1269             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1270         } elsif ($opt eq 'description') {
1271             $conf->{$opt} = PVE::Tools::encode_text($value);
1272         } elsif ($opt =~ m/^net(\d+)$/) {
1273             my $netid = $1;
1274             my $net = parse_lxc_network($value);
1275             if (!$running) {
1276                 $conf->{$opt} = print_lxc_network($net);
1277             } else {
1278                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1279             }
1280         } elsif ($opt eq 'protection') {
1281             $conf->{$opt} = $value ? 1 : 0;
1282         } elsif ($opt =~ m/^mp(\d+)$/) {
1283             check_protection($conf, "can't update CT $vmid drive '$opt'");
1284             $conf->{$opt} = $value;
1285             $new_disks = 1;
1286             push @nohotplug, $opt;
1287             next;
1288         } elsif ($opt eq 'rootfs') {
1289             check_protection($conf, "can't update CT $vmid drive '$opt'");
1290             die "implement me: $opt";
1291         } else {
1292             die "implement me: $opt";
1293         }
1294         write_config($vmid, $conf) if $running;
1295     }
1296
1297     if ($running && scalar(@nohotplug)) {
1298         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1299     }
1300
1301     if ($new_disks) {
1302         my $storage_cfg = PVE::Storage::config();
1303         create_disks($storage_cfg, $vmid, $conf, $conf);
1304     }
1305 }
1306
1307 sub has_dev_console {
1308     my ($conf) = @_;
1309
1310     return !(defined($conf->{console}) && !$conf->{console});
1311 }
1312
1313 sub get_tty_count {
1314     my ($conf) = @_;
1315
1316     return $conf->{tty} // $confdesc->{tty}->{default};
1317 }
1318
1319 sub get_cmode {
1320     my ($conf) = @_;
1321
1322     return $conf->{cmode} // $confdesc->{cmode}->{default};
1323 }
1324
1325 sub get_console_command {
1326     my ($vmid, $conf) = @_;
1327
1328     my $cmode = get_cmode($conf);
1329
1330     if ($cmode eq 'console') {
1331         return ['lxc-console', '-n',  $vmid, '-t', 0];
1332     } elsif ($cmode eq 'tty') {
1333         return ['lxc-console', '-n',  $vmid];
1334     } elsif ($cmode eq 'shell') {
1335         return ['lxc-attach', '--clear-env', '-n', $vmid];
1336     } else {
1337         die "internal error";
1338     }
1339 }
1340
1341 sub get_primary_ips {
1342     my ($conf) = @_;
1343
1344     # return data from net0
1345
1346     return undef if !defined($conf->{net0});
1347     my $net = parse_lxc_network($conf->{net0});
1348
1349     my $ipv4 = $net->{ip};
1350     if ($ipv4) {
1351         if ($ipv4 =~ /^(dhcp|manual)$/) {
1352             $ipv4 = undef
1353         } else {
1354             $ipv4 =~ s!/\d+$!!;
1355         }
1356     }
1357     my $ipv6 = $net->{ip6};
1358     if ($ipv6) {
1359         if ($ipv6 =~ /^(dhcp|manual)$/) {
1360             $ipv6 = undef;
1361         } else {
1362             $ipv6 =~ s!/\d+$!!;
1363         }
1364     }
1365
1366     return ($ipv4, $ipv6);
1367 }
1368
1369
1370 sub destroy_lxc_container {
1371     my ($storage_cfg, $vmid, $conf) = @_;
1372
1373     foreach_mountpoint($conf, sub {
1374         my ($ms, $mountpoint) = @_;
1375
1376         # skip bind mounts and block devices
1377         if ($mountpoint->{volume} =~ m|^/|) {
1378                 return;
1379         }
1380
1381         my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $mountpoint->{volume});
1382         PVE::Storage::vdisk_free($storage_cfg, $mountpoint->{volume}) if $vmid == $owner;
1383     });
1384
1385     rmdir "/var/lib/lxc/$vmid/rootfs";
1386     unlink "/var/lib/lxc/$vmid/config";
1387     rmdir "/var/lib/lxc/$vmid";
1388     destroy_config($vmid);
1389
1390     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1391     #PVE::Tools::run_command($cmd);
1392 }
1393
1394 sub vm_stop_cleanup {
1395     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1396
1397     eval {
1398         if (!$keepActive) {
1399
1400             my $vollist = get_vm_volumes($conf);
1401             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1402         }
1403     };
1404     warn $@ if $@; # avoid errors - just warn
1405 }
1406
1407 my $safe_num_ne = sub {
1408     my ($a, $b) = @_;
1409
1410     return 0 if !defined($a) && !defined($b);
1411     return 1 if !defined($a);
1412     return 1 if !defined($b);
1413
1414     return $a != $b;
1415 };
1416
1417 my $safe_string_ne = sub {
1418     my ($a, $b) = @_;
1419
1420     return 0 if !defined($a) && !defined($b);
1421     return 1 if !defined($a);
1422     return 1 if !defined($b);
1423
1424     return $a ne $b;
1425 };
1426
1427 sub update_net {
1428     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1429
1430     if ($newnet->{type} ne 'veth') {
1431         # for when there are physical interfaces
1432         die "cannot update interface of type $newnet->{type}";
1433     }
1434
1435     my $veth = "veth${vmid}i${netid}";
1436     my $eth = $newnet->{name};
1437
1438     if (my $oldnetcfg = $conf->{$opt}) {
1439         my $oldnet = parse_lxc_network($oldnetcfg);
1440
1441         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1442             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1443
1444             PVE::Network::veth_delete($veth);
1445             delete $conf->{$opt};
1446             write_config($vmid, $conf);
1447
1448             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1449
1450         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1451                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1452                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1453
1454                 if ($oldnet->{bridge}) {
1455                     PVE::Network::tap_unplug($veth);
1456                     foreach (qw(bridge tag firewall)) {
1457                         delete $oldnet->{$_};
1458                     }
1459                     $conf->{$opt} = print_lxc_network($oldnet);
1460                     write_config($vmid, $conf);
1461                 }
1462
1463                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1464                 foreach (qw(bridge tag firewall)) {
1465                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1466                 }
1467                 $conf->{$opt} = print_lxc_network($oldnet);
1468                 write_config($vmid, $conf);
1469         }
1470     } else {
1471         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1472     }
1473
1474     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1475 }
1476
1477 sub hotplug_net {
1478     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1479
1480     my $veth = "veth${vmid}i${netid}";
1481     my $vethpeer = $veth . "p";
1482     my $eth = $newnet->{name};
1483
1484     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1485     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1486
1487     # attach peer in container
1488     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1489     PVE::Tools::run_command($cmd);
1490
1491     # link up peer in container
1492     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1493     PVE::Tools::run_command($cmd);
1494
1495     my $done = { type => 'veth' };
1496     foreach (qw(bridge tag firewall hwaddr name)) {
1497         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1498     }
1499     $conf->{$opt} = print_lxc_network($done);
1500
1501     write_config($vmid, $conf);
1502 }
1503
1504 sub update_ipconfig {
1505     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1506
1507     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1508
1509     my $optdata = parse_lxc_network($conf->{$opt});
1510     my $deleted = [];
1511     my $added = [];
1512     my $nscmd = sub {
1513         my $cmdargs = shift;
1514         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1515     };
1516     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1517
1518     my $change_ip_config = sub {
1519         my ($ipversion) = @_;
1520
1521         my $family_opt = "-$ipversion";
1522         my $suffix = $ipversion == 4 ? '' : $ipversion;
1523         my $gw= "gw$suffix";
1524         my $ip= "ip$suffix";
1525
1526         my $newip = $newnet->{$ip};
1527         my $newgw = $newnet->{$gw};
1528         my $oldip = $optdata->{$ip};
1529
1530         my $change_ip = &$safe_string_ne($oldip, $newip);
1531         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1532
1533         return if !$change_ip && !$change_gw;
1534
1535         # step 1: add new IP, if this fails we cancel
1536         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1537             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1538             if (my $err = $@) {
1539                 warn $err;
1540                 return;
1541             }
1542         }
1543
1544         # step 2: replace gateway
1545         #   If this fails we delete the added IP and cancel.
1546         #   If it succeeds we save the config and delete the old IP, ignoring
1547         #   errors. The config is then saved.
1548         # Note: 'ip route replace' can add
1549         if ($change_gw) {
1550             if ($newgw) {
1551                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1552                 if (my $err = $@) {
1553                     warn $err;
1554                     # the route was not replaced, the old IP is still available
1555                     # rollback (delete new IP) and cancel
1556                     if ($change_ip) {
1557                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1558                         warn $@ if $@; # no need to die here
1559                     }
1560                     return;
1561                 }
1562             } else {
1563                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1564                 # if the route was not deleted, the guest might have deleted it manually
1565                 # warn and continue
1566                 warn $@ if $@;
1567             }
1568         }
1569
1570         # from this point on we save the configuration
1571         # step 3: delete old IP ignoring errors
1572         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1573             # We need to enable promote_secondaries, otherwise our newly added
1574             # address will be removed along with the old one.
1575             my $promote = 0;
1576             eval {
1577                 if ($ipversion == 4) {
1578                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1579                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1580                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1581                 }
1582                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1583             };
1584             warn $@ if $@; # no need to die here
1585
1586             if ($ipversion == 4) {
1587                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1588             }
1589         }
1590
1591         foreach my $property ($ip, $gw) {
1592             if ($newnet->{$property}) {
1593                 $optdata->{$property} = $newnet->{$property};
1594             } else {
1595                 delete $optdata->{$property};
1596             }
1597         }
1598         $conf->{$opt} = print_lxc_network($optdata);
1599         write_config($vmid, $conf);
1600         $lxc_setup->setup_network($conf);
1601     };
1602
1603     &$change_ip_config(4);
1604     &$change_ip_config(6);
1605
1606 }
1607
1608 # Internal snapshots
1609
1610 # NOTE: Snapshot create/delete involves several non-atomic
1611 # action, and can take a long time.
1612 # So we try to avoid locking the file and use 'lock' variable
1613 # inside the config file instead.
1614
1615 my $snapshot_copy_config = sub {
1616     my ($source, $dest) = @_;
1617
1618     foreach my $k (keys %$source) {
1619         next if $k eq 'snapshots';
1620         next if $k eq 'snapstate';
1621         next if $k eq 'snaptime';
1622         next if $k eq 'vmstate';
1623         next if $k eq 'lock';
1624         next if $k eq 'digest';
1625         next if $k eq 'description';
1626
1627         $dest->{$k} = $source->{$k};
1628     }
1629 };
1630
1631 my $snapshot_prepare = sub {
1632     my ($vmid, $snapname, $comment) = @_;
1633
1634     my $snap;
1635
1636     my $updatefn =  sub {
1637
1638         my $conf = load_config($vmid);
1639
1640         die "you can't take a snapshot if it's a template\n"
1641             if is_template($conf);
1642
1643         check_lock($conf);
1644
1645         $conf->{lock} = 'snapshot';
1646
1647         die "snapshot name '$snapname' already used\n"
1648             if defined($conf->{snapshots}->{$snapname});
1649
1650         my $storecfg = PVE::Storage::config();
1651         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1652
1653         $snap = $conf->{snapshots}->{$snapname} = {};
1654
1655         &$snapshot_copy_config($conf, $snap);
1656
1657         $snap->{'snapstate'} = "prepare";
1658         $snap->{'snaptime'} = time();
1659         $snap->{'description'} = $comment if $comment;
1660         $conf->{snapshots}->{$snapname} = $snap;
1661
1662         write_config($vmid, $conf);
1663     };
1664
1665     lock_container($vmid, 10, $updatefn);
1666
1667     return $snap;
1668 };
1669
1670 my $snapshot_commit = sub {
1671     my ($vmid, $snapname) = @_;
1672
1673     my $updatefn = sub {
1674
1675         my $conf = load_config($vmid);
1676
1677         die "missing snapshot lock\n"
1678             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1679
1680         die "snapshot '$snapname' does not exist\n"
1681             if !defined($conf->{snapshots}->{$snapname});
1682
1683         die "wrong snapshot state\n"
1684             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1685                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1686
1687         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1688         delete $conf->{lock};
1689         $conf->{parent} = $snapname;
1690
1691         write_config($vmid, $conf);
1692     };
1693
1694     lock_container($vmid, 10 ,$updatefn);
1695 };
1696
1697 sub has_feature {
1698     my ($feature, $conf, $storecfg, $snapname) = @_;
1699
1700     my $err;
1701
1702     foreach_mountpoint($conf, sub {
1703         my ($ms, $mountpoint) = @_;
1704
1705         return if $err; # skip further test
1706
1707         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1708
1709         # TODO: implement support for mountpoints
1710         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1711             if $ms ne 'rootfs';
1712     });
1713
1714     return $err ? 0 : 1;
1715 }
1716
1717 sub snapshot_create {
1718     my ($vmid, $snapname, $comment) = @_;
1719
1720     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1721
1722     my $conf = load_config($vmid);
1723
1724     my $running = check_running($vmid);
1725     eval {
1726         if ($running) {
1727             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1728             PVE::Tools::run_command(['/bin/sync']);
1729         };
1730
1731         my $storecfg = PVE::Storage::config();
1732         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1733         my $volid = $rootinfo->{volume};
1734
1735         if ($running) {
1736             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1737         };
1738
1739         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1740         &$snapshot_commit($vmid, $snapname);
1741     };
1742     if(my $err = $@) {
1743         snapshot_delete($vmid, $snapname, 1);
1744         die "$err\n";
1745     }
1746 }
1747
1748 sub snapshot_delete {
1749     my ($vmid, $snapname, $force) = @_;
1750
1751     my $snap;
1752
1753     my $conf;
1754
1755     my $updatefn =  sub {
1756
1757         $conf = load_config($vmid);
1758
1759         die "you can't delete a snapshot if vm is a template\n"
1760             if is_template($conf);
1761
1762         $snap = $conf->{snapshots}->{$snapname};
1763
1764         check_lock($conf);
1765
1766         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1767
1768         $snap->{snapstate} = 'delete';
1769
1770         write_config($vmid, $conf);
1771     };
1772
1773     lock_container($vmid, 10, $updatefn);
1774
1775     my $storecfg = PVE::Storage::config();
1776
1777     my $del_snap =  sub {
1778
1779         check_lock($conf);
1780
1781         if ($conf->{parent} eq $snapname) {
1782             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1783                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1784             } else {
1785                 delete $conf->{parent};
1786             }
1787         }
1788
1789         delete $conf->{snapshots}->{$snapname};
1790
1791         write_config($vmid, $conf);
1792     };
1793
1794     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1795     my $rootinfo = parse_ct_mountpoint($rootfs);
1796     my $volid = $rootinfo->{volume};
1797
1798     eval {
1799         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1800     };
1801     my $err = $@;
1802
1803     if(!$err || ($err && $force)) {
1804         lock_container($vmid, 10, $del_snap);
1805         if ($err) {
1806             die "Can't delete snapshot: $vmid $snapname $err\n";
1807         }
1808     }
1809 }
1810
1811 sub snapshot_rollback {
1812     my ($vmid, $snapname) = @_;
1813
1814     my $storecfg = PVE::Storage::config();
1815
1816     my $conf = load_config($vmid);
1817
1818     die "you can't rollback if vm is a template\n" if is_template($conf);
1819
1820     my $snap = $conf->{snapshots}->{$snapname};
1821
1822     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1823
1824     my $rootfs = $snap->{rootfs};
1825     my $rootinfo = parse_ct_mountpoint($rootfs);
1826     my $volid = $rootinfo->{volume};
1827
1828     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1829
1830     my $updatefn = sub {
1831
1832         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1833             if $snap->{snapstate};
1834
1835         check_lock($conf);
1836
1837         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1838
1839         die "unable to rollback vm $vmid: vm is running\n"
1840             if check_running($vmid);
1841
1842         $conf->{lock} = 'rollback';
1843
1844         my $forcemachine;
1845
1846         # copy snapshot config to current config
1847
1848         my $tmp_conf = $conf;
1849         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1850         $conf->{snapshots} = $tmp_conf->{snapshots};
1851         delete $conf->{snaptime};
1852         delete $conf->{snapname};
1853         $conf->{parent} = $snapname;
1854
1855         write_config($vmid, $conf);
1856     };
1857
1858     my $unlockfn = sub {
1859         delete $conf->{lock};
1860         write_config($vmid, $conf);
1861     };
1862
1863     lock_container($vmid, 10, $updatefn);
1864
1865     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1866
1867     lock_container($vmid, 5, $unlockfn);
1868 }
1869
1870 sub template_create {
1871     my ($vmid, $conf) = @_;
1872
1873     my $storecfg = PVE::Storage::config();
1874
1875     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1876     my $volid = $rootinfo->{volume};
1877
1878     die "Template feature is not available for '$volid'\n"
1879         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1880
1881     PVE::Storage::activate_volumes($storecfg, [$volid]);
1882
1883     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1884     $rootinfo->{volume} = $template_volid;
1885     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1886
1887     write_config($vmid, $conf);
1888 }
1889
1890 sub is_template {
1891     my ($conf) = @_;
1892
1893     return 1 if defined $conf->{template} && $conf->{template} == 1;
1894 }
1895
1896 sub mountpoint_names {
1897     my ($reverse) = @_;
1898
1899     my @names = ('rootfs');
1900
1901     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1902         push @names, "mp$i";
1903     }
1904
1905     return $reverse ? reverse @names : @names;
1906 }
1907
1908 # The container might have *different* symlinks than the host. realpath/abs_path
1909 # use the actual filesystem to resolve links.
1910 sub sanitize_mountpoint {
1911     my ($mp) = @_;
1912     $mp = '/' . $mp; # we always start with a slash
1913     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1914     $mp =~ s@/\./@@g; # collapse /./
1915     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1916     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1917     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1918     return $mp;
1919 }
1920
1921 sub foreach_mountpoint_full {
1922     my ($conf, $reverse, $func) = @_;
1923
1924     foreach my $key (mountpoint_names($reverse)) {
1925         my $value = $conf->{$key};
1926         next if !defined($value);
1927         my $mountpoint = parse_ct_mountpoint($value);
1928
1929         # just to be sure: rootfs is /
1930         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1931         $mountpoint->{mp} = sanitize_mountpoint($path);
1932
1933         $path = $mountpoint->{volume};
1934         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1935
1936         &$func($key, $mountpoint);
1937     }
1938 }
1939
1940 sub foreach_mountpoint {
1941     my ($conf, $func) = @_;
1942
1943     foreach_mountpoint_full($conf, 0, $func);
1944 }
1945
1946 sub foreach_mountpoint_reverse {
1947     my ($conf, $func) = @_;
1948
1949     foreach_mountpoint_full($conf, 1, $func);
1950 }
1951
1952 sub check_ct_modify_config_perm {
1953     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1954
1955     return 1 if $authuser ne 'root@pam';
1956
1957     foreach my $opt (@$key_list) {
1958
1959         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1960             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1961         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1962             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1963         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1964             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1965         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1966                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1967             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1968         } else {
1969             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1970         }
1971     }
1972
1973     return 1;
1974 }
1975
1976 sub umount_all {
1977     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
1978
1979     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1980     my $volid_list = get_vm_volumes($conf);
1981
1982     foreach_mountpoint_reverse($conf, sub {
1983         my ($ms, $mountpoint) = @_;
1984
1985         my $volid = $mountpoint->{volume};
1986         my $mount = $mountpoint->{mp};
1987
1988         return if !$volid || !$mount;
1989
1990         my $mount_path = "$rootdir/$mount";
1991         $mount_path =~ s!/+!/!g;
1992
1993         return if !PVE::ProcFSTools::is_mounted($mount_path);
1994
1995         eval {
1996             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1997         };
1998         if (my $err = $@) {
1999             if ($noerr) {
2000                 warn $err;
2001             } else {
2002                 die $err;
2003             }
2004         }
2005     });
2006 }
2007
2008 sub mount_all {
2009     my ($vmid, $storage_cfg, $conf) = @_;
2010
2011     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2012     File::Path::make_path($rootdir);
2013
2014     my $volid_list = get_vm_volumes($conf);
2015     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2016
2017     eval {
2018         foreach_mountpoint($conf, sub {
2019             my ($ms, $mountpoint) = @_;
2020
2021             my $volid = $mountpoint->{volume};
2022             my $mount = $mountpoint->{mp};
2023
2024             return if !$volid || !$mount;
2025
2026             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2027             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2028                 PVE::Storage::parse_volname($storage_cfg, $volid);
2029
2030             die "unable to mount base volume - internal error" if $isBase;
2031
2032             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2033         });
2034     };
2035     if (my $err = $@) {
2036         warn "mounting container failed - $err";
2037         umount_all($vmid, $storage_cfg, $conf, 1);
2038     }
2039
2040     return $rootdir;
2041 }
2042
2043
2044 sub mountpoint_mount_path {
2045     my ($mountpoint, $storage_cfg, $snapname) = @_;
2046
2047     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2048 }
2049
2050 my $check_mount_path = sub {
2051     my ($path) = @_;
2052     $path = File::Spec->canonpath($path);
2053     my $real = Cwd::realpath($path);
2054     if ($real ne $path) {
2055         die "mount path modified by symlink: $path != $real";
2056     }
2057 };
2058
2059 # use $rootdir = undef to just return the corresponding mount path
2060 sub mountpoint_mount {
2061     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2062
2063     my $volid = $mountpoint->{volume};
2064     my $mount = $mountpoint->{mp};
2065
2066     return if !$volid || !$mount;
2067
2068     my $mount_path;
2069
2070     if (defined($rootdir)) {
2071         $rootdir =~ s!/+$!!;
2072         $mount_path = "$rootdir/$mount";
2073         $mount_path =~ s!/+!/!g;
2074         &$check_mount_path($mount_path);
2075         File::Path::mkpath($mount_path);
2076     }
2077
2078     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2079
2080     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2081
2082     if ($storage) {
2083
2084         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2085         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2086
2087         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2088             PVE::Storage::parse_volname($storage_cfg, $volid);
2089
2090         if ($format eq 'subvol') {
2091             if ($mount_path) {
2092                 if ($snapname) {
2093                     if ($scfg->{type} eq 'zfspool') {
2094                         my $path_arg = $path;
2095                         $path_arg =~ s!^/+!!;
2096                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', '-t', 'zfs', $path_arg, $mount_path]);
2097                     } else {
2098                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2099                     }
2100                 } else {
2101                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2102                 }
2103             }
2104             return wantarray ? ($path, 0) : $path;
2105         } elsif ($format eq 'raw') {
2106             my $use_loopdev = 0;
2107             my @extra_opts;
2108             if ($scfg->{path}) {
2109                 push @extra_opts, '-o', 'loop';
2110                 $use_loopdev = 1;
2111             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2112                 # do nothing
2113             } else {
2114                 die "unsupported storage type '$scfg->{type}'\n";
2115             }
2116             if ($mount_path) {
2117                 if ($isBase || defined($snapname)) {
2118                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2119                 } else {
2120                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2121                 }
2122             }
2123             return wantarray ? ($path, $use_loopdev) : $path;
2124         } else {
2125             die "unsupported image format '$format'\n";
2126         }
2127     } elsif ($volid =~ m|^/dev/.+|) {
2128         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2129         return wantarray ? ($volid, 0) : $volid;
2130     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2131         &$check_mount_path($volid);
2132         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2133         return wantarray ? ($volid, 0) : $volid;
2134     }
2135
2136     die "unsupported storage";
2137 }
2138
2139 sub get_vm_volumes {
2140     my ($conf, $excludes) = @_;
2141
2142     my $vollist = [];
2143
2144     foreach_mountpoint($conf, sub {
2145         my ($ms, $mountpoint) = @_;
2146
2147         return if $excludes && $ms eq $excludes;
2148
2149         my $volid = $mountpoint->{volume};
2150
2151         return if !$volid || $volid =~ m|^/|;
2152
2153         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2154         return if !$sid;
2155
2156         push @$vollist, $volid;
2157     });
2158
2159     return $vollist;
2160 }
2161
2162 sub mkfs {
2163     my ($dev) = @_;
2164
2165     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2166 }
2167
2168 sub format_disk {
2169     my ($storage_cfg, $volid) = @_;
2170
2171     if ($volid =~ m!^/dev/.+!) {
2172         mkfs($volid);
2173         return;
2174     }
2175
2176     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2177
2178     die "cannot format volume '$volid' with no storage\n" if !$storage;
2179
2180     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2181
2182     my $path = PVE::Storage::path($storage_cfg, $volid);
2183
2184     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2185         PVE::Storage::parse_volname($storage_cfg, $volid);
2186
2187     die "cannot format volume '$volid' (format == $format)\n"
2188         if $format ne 'raw';
2189
2190     mkfs($path);
2191 }
2192
2193 sub destroy_disks {
2194     my ($storecfg, $vollist) = @_;
2195
2196     foreach my $volid (@$vollist) {
2197         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2198         warn $@ if $@;
2199     }
2200 }
2201
2202 sub create_disks {
2203     my ($storecfg, $vmid, $settings, $conf) = @_;
2204
2205     my $vollist = [];
2206
2207     eval {
2208         foreach_mountpoint($settings, sub {
2209             my ($ms, $mountpoint) = @_;
2210
2211             my $volid = $mountpoint->{volume};
2212             my $mp = $mountpoint->{mp};
2213
2214             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2215
2216             return if !$storage;
2217
2218             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2219                 my ($storeid, $size_gb) = ($1, $2);
2220
2221                 my $size_kb = int(${size_gb}*1024) * 1024;
2222
2223                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2224                 # fixme: use better naming ct-$vmid-disk-X.raw?
2225
2226                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2227                     if ($size_kb > 0) {
2228                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2229                                                            undef, $size_kb);
2230                         format_disk($storecfg, $volid);
2231                     } else {
2232                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2233                                                            undef, 0);
2234                     }
2235                 } elsif ($scfg->{type} eq 'zfspool') {
2236
2237                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2238                                                        undef, $size_kb);
2239                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2240
2241                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2242                     format_disk($storecfg, $volid);
2243
2244                 } elsif ($scfg->{type} eq 'rbd') {
2245
2246                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2247                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2248                     format_disk($storecfg, $volid);
2249                 } else {
2250                     die "unable to create containers on storage type '$scfg->{type}'\n";
2251                 }
2252                 push @$vollist, $volid;
2253                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2254                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2255             } else {
2256                 # use specified/existing volid
2257             }
2258         });
2259     };
2260     # free allocated images on error
2261     if (my $err = $@) {
2262         destroy_disks($storecfg, $vollist);
2263         die $err;
2264     }
2265     return $vollist;
2266 }
2267
2268 # bash completion helper
2269
2270 sub complete_os_templates {
2271     my ($cmdname, $pname, $cvalue) = @_;
2272
2273     my $cfg = PVE::Storage::config();
2274
2275     my $storeid;
2276
2277     if ($cvalue =~ m/^([^:]+):/) {
2278         $storeid = $1;
2279     }
2280
2281     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2282     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2283
2284     my $res = [];
2285     foreach my $id (keys %$data) {
2286         foreach my $item (@{$data->{$id}}) {
2287             push @$res, $item->{volid} if defined($item->{volid});
2288         }
2289     }
2290
2291     return $res;
2292 }
2293
2294 my $complete_ctid_full = sub {
2295     my ($running) = @_;
2296
2297     my $idlist = vmstatus();
2298
2299     my $active_hash = list_active_containers();
2300
2301     my $res = [];
2302
2303     foreach my $id (keys %$idlist) {
2304         my $d = $idlist->{$id};
2305         if (defined($running)) {
2306             next if $d->{template};
2307             next if $running && !$active_hash->{$id};
2308             next if !$running && $active_hash->{$id};
2309         }
2310         push @$res, $id;
2311
2312     }
2313     return $res;
2314 };
2315
2316 sub complete_ctid {
2317     return &$complete_ctid_full();
2318 }
2319
2320 sub complete_ctid_stopped {
2321     return &$complete_ctid_full(0);
2322 }
2323
2324 sub complete_ctid_running {
2325     return &$complete_ctid_full(1);
2326 }
2327
2328 1;