src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use Fcntl ':flock';
   9
  10 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  11 use PVE::Storage;
  12 use PVE::SafeSyslog;
  13 use PVE::INotify;
  14 use PVE::JSONSchema qw(get_standard_option);
  15 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  16 use PVE::Network;
  17 use PVE::AccessControl;
  18 use PVE::ProcFSTools;
  19
  20 use Data::Dumper;
  21
  22 my $nodename = PVE::INotify::nodename();
  23
  24 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  25
  26 PVE::JSONSchema::register_format('pve-lxc-network', \&verify_lxc_network);
  27 sub verify_lxc_network {
  28     my ($value, $noerr) = @_;
  29
  30     return $value if parse_lxc_network($value);
  31
  32     return undef if $noerr;
  33
  34     die "unable to parse network setting\n";
  35 }
  36
  37 PVE::JSONSchema::register_format('pve-ct-mountpoint', \&verify_ct_mountpoint);
  38 sub verify_ct_mountpoint {
  39     my ($value, $noerr) = @_;
  40
  41     return $value if parse_ct_mountpoint($value);
  42
  43     return undef if $noerr;
  44
  45     die "unable to parse CT mountpoint options\n";
  46 }
  47
  48 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  49     type => 'string', format => 'pve-ct-mountpoint',
  50     typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+]',
  51     description => "Use volume as container root.",
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  56     description => "The name of the snapshot.",
  57     type => 'string', format => 'pve-configid',
  58     maxLength => 40,
  59 });
  60
  61 my $confdesc = {
  62     lock => {
  63         optional => 1,
  64         type => 'string',
  65         description => "Lock/unlock the VM.",
  66         enum => [qw(migrate backup snapshot rollback)],
  67     },
  68     onboot => {
  69         optional => 1,
  70         type => 'boolean',
  71         description => "Specifies whether a VM will be started during system bootup.",
  72         default => 0,
  73     },
  74     startup => get_standard_option('pve-startup-order'),
  75     template => {
  76         optional => 1,
  77         type => 'boolean',
  78         description => "Enable/disable Template.",
  79         default => 0,
  80     },
  81     arch => {
  82         optional => 1,
  83         type => 'string',
  84         enum => ['amd64', 'i386'],
  85         description => "OS architecture type.",
  86         default => 'amd64',
  87     },
  88     ostype => {
  89         optional => 1,
  90         type => 'string',
  91         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  92         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  93     },
  94     console => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Attach a console device (/dev/console) to the container.",
  98         default => 1,
  99     },
 100     tty => {
 101         optional => 1,
 102         type => 'integer',
 103         description => "Specify the number of tty available to the container",
 104         minimum => 0,
 105         maximum => 6,
 106         default => 2,
 107     },
 108     cpulimit => {
 109         optional => 1,
 110         type => 'number',
 111         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 112         minimum => 0,
 113         maximum => 128,
 114         default => 0,
 115     },
 116     cpuunits => {
 117         optional => 1,
 118         type => 'integer',
 119         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 120         minimum => 0,
 121         maximum => 500000,
 122         default => 1024,
 123     },
 124     memory => {
 125         optional => 1,
 126         type => 'integer',
 127         description => "Amount of RAM for the VM in MB.",
 128         minimum => 16,
 129         default => 512,
 130     },
 131     swap => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "Amount of SWAP for the VM in MB.",
 135         minimum => 0,
 136         default => 512,
 137     },
 138     hostname => {
 139         optional => 1,
 140         description => "Set a host name for the container.",
 141         type => 'string',
 142         maxLength => 255,
 143     },
 144     description => {
 145         optional => 1,
 146         type => 'string',
 147         description => "Container description. Only used on the configuration web interface.",
 148     },
 149     searchdomain => {
 150         optional => 1,
 151         type => 'string',
 152         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 153     },
 154     nameserver => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 158     },
 159     rootfs => get_standard_option('pve-ct-rootfs'),
 160     parent => {
 161         optional => 1,
 162         type => 'string', format => 'pve-configid',
 163         maxLength => 40,
 164         description => "Parent snapshot name. This is used internally, and should not be modified.",
 165     },
 166     snaptime => {
 167         optional => 1,
 168         description => "Timestamp for snapshots.",
 169         type => 'integer',
 170         minimum => 0,
 171     },
 172     cmode => {
 173         optional => 1,
 174         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 175         type => 'string',
 176         enum => ['shell', 'console', 'tty'],
 177         default => 'tty',
 178     },
 179     protection => {
 180         optional => 1,
 181         type => 'boolean',
 182         description => "Sets the protection flag of the container. This will prevent the remove operation.",
 183         default => 0,
 184     },
 185 };
 186
 187 my $valid_lxc_conf_keys = {
 188     'lxc.include' => 1,
 189     'lxc.arch' => 1,
 190     'lxc.utsname' => 1,
 191     'lxc.haltsignal' => 1,
 192     'lxc.rebootsignal' => 1,
 193     'lxc.stopsignal' => 1,
 194     'lxc.init_cmd' => 1,
 195     'lxc.network.type' => 1,
 196     'lxc.network.flags' => 1,
 197     'lxc.network.link' => 1,
 198     'lxc.network.mtu' => 1,
 199     'lxc.network.name' => 1,
 200     'lxc.network.hwaddr' => 1,
 201     'lxc.network.ipv4' => 1,
 202     'lxc.network.ipv4.gateway' => 1,
 203     'lxc.network.ipv6' => 1,
 204     'lxc.network.ipv6.gateway' => 1,
 205     'lxc.network.script.up' => 1,
 206     'lxc.network.script.down' => 1,
 207     'lxc.pts' => 1,
 208     'lxc.console.logfile' => 1,
 209     'lxc.console' => 1,
 210     'lxc.tty' => 1,
 211     'lxc.devttydir' => 1,
 212     'lxc.hook.autodev' => 1,
 213     'lxc.autodev' => 1,
 214     'lxc.kmsg' => 1,
 215     'lxc.mount' => 1,
 216     'lxc.mount.entry' => 1,
 217     'lxc.mount.auto' => 1,
 218     'lxc.rootfs' => 1,
 219     'lxc.rootfs.mount' => 1,
 220     'lxc.rootfs.options' => 1,
 221     # lxc.cgroup.*
 222     'lxc.cap.drop' => 1,
 223     'lxc.cap.keep' => 1,
 224     'lxc.aa_profile' => 1,
 225     'lxc.aa_allow_incomplete' => 1,
 226     'lxc.se_context' => 1,
 227     'lxc.seccomp' => 1,
 228     'lxc.id_map' => 1,
 229     'lxc.hook.pre-start' => 1,
 230     'lxc.hook.pre-mount' => 1,
 231     'lxc.hook.mount' => 1,
 232     'lxc.hook.start' => 1,
 233     'lxc.hook.post-stop' => 1,
 234     'lxc.hook.clone' => 1,
 235     'lxc.hook.destroy' => 1,
 236     'lxc.loglevel' => 1,
 237     'lxc.logfile' => 1,
 238     'lxc.start.auto' => 1,
 239     'lxc.start.delay' => 1,
 240     'lxc.start.order' => 1,
 241     'lxc.group' => 1,
 242     'lxc.environment' => 1,
 243     'lxc.' => 1,
 244     'lxc.' => 1,
 245     'lxc.' => 1,
 246     'lxc.' => 1,
 247 };
 248
 249 my $MAX_LXC_NETWORKS = 10;
 250 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 251     $confdesc->{"net$i"} = {
 252         optional => 1,
 253         type => 'string', format => 'pve-lxc-network',
 254         description => "Specifies network interfaces for the container.\n\n".
 255             "The string should have the follow format:\n\n".
 256             "-net<[0-9]> bridge=<vmbr<Nummber>>[,hwaddr=<MAC>]\n".
 257             "[,mtu=<Number>][,name=<String>][,ip=<IPv4Format/CIDR>]\n".
 258             ",ip6=<IPv6Format/CIDR>][,gw=<GatwayIPv4>]\n".
 259             ",gw6=<GatwayIPv6>][,firewall=<[1|0]>][,tag=<VlanNo>]",
 260     };
 261 }
 262
 263 my $MAX_MOUNT_POINTS = 10;
 264 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 265     $confdesc->{"mp$i"} = {
 266         optional => 1,
 267         type => 'string', format => 'pve-ct-mountpoint',
 268         typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+] [,mp=mountpoint]',
 269         description => "Use volume as container mount point (experimental feature).",
 270         optional => 1,
 271     };
 272 }
 273
 274 sub write_pct_config {
 275     my ($filename, $conf) = @_;
 276
 277     delete $conf->{snapstate}; # just to be sure
 278
 279     my $generate_raw_config = sub {
 280         my ($conf) = @_;
 281
 282         my $raw = '';
 283
 284         # add description as comment to top of file
 285         my $descr = $conf->{description} || '';
 286         foreach my $cl (split(/\n/, $descr)) {
 287             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 288         }
 289
 290         foreach my $key (sort keys %$conf) {
 291             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 292                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 293             $raw .= "$key: $conf->{$key}\n";
 294         }
 295
 296         if (my $lxcconf = $conf->{lxc}) {
 297             foreach my $entry (@$lxcconf) {
 298                 my ($k, $v) = @$entry;
 299                 $raw .= "$k: $v\n";
 300             }
 301         }
 302
 303         return $raw;
 304     };
 305
 306     my $raw = &$generate_raw_config($conf);
 307
 308     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 309         $raw .= "\n[$snapname]\n";
 310         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 311     }
 312
 313     return $raw;
 314 }
 315
 316 sub check_type {
 317     my ($key, $value) = @_;
 318
 319     die "unknown setting '$key'\n" if !$confdesc->{$key};
 320
 321     my $type = $confdesc->{$key}->{type};
 322
 323     if (!defined($value)) {
 324         die "got undefined value\n";
 325     }
 326
 327     if ($value =~ m/[\n\r]/) {
 328         die "property contains a line feed\n";
 329     }
 330
 331     if ($type eq 'boolean') {
 332         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 333         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 334         die "type check ('boolean') failed - got '$value'\n";
 335     } elsif ($type eq 'integer') {
 336         return int($1) if $value =~ m/^(\d+)$/;
 337         die "type check ('integer') failed - got '$value'\n";
 338     } elsif ($type eq 'number') {
 339         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 340         die "type check ('number') failed - got '$value'\n";
 341     } elsif ($type eq 'string') {
 342         if (my $fmt = $confdesc->{$key}->{format}) {
 343             PVE::JSONSchema::check_format($fmt, $value);
 344             return $value;
 345         }
 346         return $value;
 347     } else {
 348         die "internal error"
 349     }
 350 }
 351
 352 sub parse_pct_config {
 353     my ($filename, $raw) = @_;
 354
 355     return undef if !defined($raw);
 356
 357     my $res = {
 358         digest => Digest::SHA::sha1_hex($raw),
 359         snapshots => {},
 360     };
 361
 362     $filename =~ m|/lxc/(\d+).conf$|
 363         || die "got strange filename '$filename'";
 364
 365     my $vmid = $1;
 366
 367     my $conf = $res;
 368     my $descr = '';
 369     my $section = '';
 370
 371     my @lines = split(/\n/, $raw);
 372     foreach my $line (@lines) {
 373         next if $line =~ m/^\s*$/;
 374
 375         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 376             $section = $1;
 377             $conf->{description} = $descr if $descr;
 378             $descr = '';
 379             $conf = $res->{snapshots}->{$section} = {};
 380             next;
 381         }
 382
 383         if ($line =~ m/^\#(.*)\s*$/) {
 384             $descr .= PVE::Tools::decode_text($1) . "\n";
 385             next;
 386         }
 387
 388         if ($line =~ m/^(lxc\.[a-z0-9_\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 389             my $key = $1;
 390             my $value = $3;
 391             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 392                 push @{$conf->{lxc}}, [$key, $value];
 393             } else {
 394                 warn "vm $vmid - unable to parse config: $line\n";
 395             }
 396         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 397             $descr .= PVE::Tools::decode_text($2);
 398         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 399             $conf->{snapstate} = $1;
 400         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
 401             my $key = $1;
 402             my $value = $2;
 403             eval { $value = check_type($key, $value); };
 404             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 405             $conf->{$key} = $value;
 406         } else {
 407             warn "vm $vmid - unable to parse config: $line\n";
 408         }
 409     }
 410
 411     $conf->{description} = $descr if $descr;
 412
 413     delete $res->{snapstate}; # just to be sure
 414
 415     return $res;
 416 }
 417
 418 sub config_list {
 419     my $vmlist = PVE::Cluster::get_vmlist();
 420     my $res = {};
 421     return $res if !$vmlist || !$vmlist->{ids};
 422     my $ids = $vmlist->{ids};
 423
 424     foreach my $vmid (keys %$ids) {
 425         next if !$vmid; # skip CT0
 426         my $d = $ids->{$vmid};
 427         next if !$d->{node} || $d->{node} ne $nodename;
 428         next if !$d->{type} || $d->{type} ne 'lxc';
 429         $res->{$vmid}->{type} = 'lxc';
 430     }
 431     return $res;
 432 }
 433
 434 sub cfs_config_path {
 435     my ($vmid, $node) = @_;
 436
 437     $node = $nodename if !$node;
 438     return "nodes/$node/lxc/$vmid.conf";
 439 }
 440
 441 sub config_file {
 442     my ($vmid, $node) = @_;
 443
 444     my $cfspath = cfs_config_path($vmid, $node);
 445     return "/etc/pve/$cfspath";
 446 }
 447
 448 sub load_config {
 449     my ($vmid, $node) = @_;
 450
 451     $node = $nodename if !$node;
 452     my $cfspath = cfs_config_path($vmid, $node);
 453
 454     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 455     die "container $vmid does not exists\n" if !defined($conf);
 456
 457     return $conf;
 458 }
 459
 460 sub create_config {
 461     my ($vmid, $conf) = @_;
 462
 463     my $dir = "/etc/pve/nodes/$nodename/lxc";
 464     mkdir $dir;
 465
 466     write_config($vmid, $conf);
 467 }
 468
 469 sub destroy_config {
 470     my ($vmid) = @_;
 471
 472     unlink config_file($vmid, $nodename);
 473 }
 474
 475 sub write_config {
 476     my ($vmid, $conf) = @_;
 477
 478     my $cfspath = cfs_config_path($vmid);
 479
 480     PVE::Cluster::cfs_write_file($cfspath, $conf);
 481 }
 482
 483 # flock: we use one file handle per process, so lock file
 484 # can be called multiple times and succeeds for the same process.
 485
 486 my $lock_handles =  {};
 487 my $lockdir = "/run/lock/lxc";
 488
 489 sub lock_filename {
 490     my ($vmid) = @_;
 491
 492     return "$lockdir/pve-config-${vmid}.lock";
 493 }
 494
 495 sub lock_aquire {
 496     my ($vmid, $timeout) = @_;
 497
 498     $timeout = 10 if !$timeout;
 499     my $mode = LOCK_EX;
 500
 501     my $filename = lock_filename($vmid);
 502
 503     mkdir $lockdir if !-d $lockdir;
 504
 505     my $lock_func = sub {
 506         if (!$lock_handles->{$$}->{$filename}) {
 507             my $fh = new IO::File(">>$filename") ||
 508                 die "can't open file - $!\n";
 509             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 510         }
 511
 512         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 513             print STDERR "trying to aquire lock...";
 514             my $success;
 515             while(1) {
 516                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 517                 # try again on EINTR (see bug #273)
 518                 if ($success || ($! != EINTR)) {
 519                     last;
 520                 }
 521             }
 522             if (!$success) {
 523                 print STDERR " failed\n";
 524                 die "can't aquire lock - $!\n";
 525             }
 526
 527             print STDERR " OK\n";
 528         }
 529
 530         $lock_handles->{$$}->{$filename}->{refcount}++;
 531     };
 532
 533     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 534     my $err = $@;
 535     if ($err) {
 536         die "can't lock file '$filename' - $err";
 537     }
 538 }
 539
 540 sub lock_release {
 541     my ($vmid) = @_;
 542
 543     my $filename = lock_filename($vmid);
 544
 545     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 546         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 547         if ($refcount <= 0) {
 548             $lock_handles->{$$}->{$filename} = undef;
 549             close ($fh);
 550         }
 551     }
 552 }
 553
 554 sub lock_container {
 555     my ($vmid, $timeout, $code, @param) = @_;
 556
 557     my $res;
 558
 559     lock_aquire($vmid, $timeout);
 560     eval { $res = &$code(@param) };
 561     my $err = $@;
 562     lock_release($vmid);
 563
 564     die $err if $err;
 565
 566     return $res;
 567 }
 568
 569 sub option_exists {
 570     my ($name) = @_;
 571
 572     return defined($confdesc->{$name});
 573 }
 574
 575 # add JSON properties for create and set function
 576 sub json_config_properties {
 577     my $prop = shift;
 578
 579     foreach my $opt (keys %$confdesc) {
 580         next if $opt eq 'parent' || $opt eq 'snaptime';
 581         next if $prop->{$opt};
 582         $prop->{$opt} = $confdesc->{$opt};
 583     }
 584
 585     return $prop;
 586 }
 587
 588 sub json_config_properties_no_rootfs {
 589     my $prop = shift;
 590
 591     foreach my $opt (keys %$confdesc) {
 592         next if $prop->{$opt};
 593         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 594         $prop->{$opt} = $confdesc->{$opt};
 595     }
 596
 597     return $prop;
 598 }
 599
 600 # container status helpers
 601
 602 sub list_active_containers {
 603
 604     my $filename = "/proc/net/unix";
 605
 606     # similar test is used by lcxcontainers.c: list_active_containers
 607     my $res = {};
 608
 609     my $fh = IO::File->new ($filename, "r");
 610     return $res if !$fh;
 611
 612     while (defined(my $line = <$fh>)) {
 613         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 614             my $path = $1;
 615             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 616                 $res->{$1} = 1;
 617             }
 618         }
 619     }
 620
 621     close($fh);
 622
 623     return $res;
 624 }
 625
 626 # warning: this is slow
 627 sub check_running {
 628     my ($vmid) = @_;
 629
 630     my $active_hash = list_active_containers();
 631
 632     return 1 if defined($active_hash->{$vmid});
 633
 634     return undef;
 635 }
 636
 637 sub get_container_disk_usage {
 638     my ($vmid) = @_;
 639
 640     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 641
 642     my $res = {
 643         total => 0,
 644         used => 0,
 645         avail => 0,
 646     };
 647
 648     my $parser = sub {
 649         my $line = shift;
 650         if (my ($fsid, $total, $used, $avail) = $line =~
 651             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 652             $res = {
 653                 total => $total,
 654                 used => $used,
 655                 avail => $avail,
 656             };
 657         }
 658     };
 659     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 660     warn $@ if $@;
 661
 662     return $res;
 663 }
 664
 665 sub vmstatus {
 666     my ($opt_vmid) = @_;
 667
 668     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 669
 670     my $active_hash = list_active_containers();
 671
 672     foreach my $vmid (keys %$list) {
 673         my $d = $list->{$vmid};
 674
 675         my $running = defined($active_hash->{$vmid});
 676
 677         $d->{status} = $running ? 'running' : 'stopped';
 678
 679         my $cfspath = cfs_config_path($vmid);
 680         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 681
 682         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 683         $d->{name} =~ s/[\s]//g;
 684
 685         $d->{cpus} = $conf->{cpulimit} // 0;
 686
 687         if ($running) {
 688             my $res = get_container_disk_usage($vmid);
 689             $d->{disk} = $res->{used};
 690             $d->{maxdisk} = $res->{total};
 691         } else {
 692             $d->{disk} = 0;
 693             # use 4GB by default ??
 694             if (my $rootfs = $conf->{rootfs}) {
 695                 my $rootinfo = parse_ct_mountpoint($rootfs);
 696                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 697             } else {
 698                 $d->{maxdisk} = 4*1024*1024*1024;
 699             }
 700         }
 701
 702         $d->{mem} = 0;
 703         $d->{swap} = 0;
 704         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 705         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 706
 707         $d->{uptime} = 0;
 708         $d->{cpu} = 0;
 709
 710         $d->{netout} = 0;
 711         $d->{netin} = 0;
 712
 713         $d->{diskread} = 0;
 714         $d->{diskwrite} = 0;
 715
 716         $d->{template} = is_template($conf);
 717     }
 718
 719     foreach my $vmid (keys %$list) {
 720         my $d = $list->{$vmid};
 721         next if $d->{status} ne 'running';
 722
 723         $d->{uptime} = 100; # fixme:
 724
 725         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 726         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 727
 728         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 729         my @bytes = split(/\n/, $blkio_bytes);
 730         foreach my $byte (@bytes) {
 731             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 732                 $d->{diskread} = $2 if $key eq 'Read';
 733                 $d->{diskwrite} = $2 if $key eq 'Write';
 734             }
 735         }
 736     }
 737
 738     return $list;
 739 }
 740
 741 my $parse_size = sub {
 742     my ($value) = @_;
 743
 744     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 745     my ($size, $unit) = ($1, $3);
 746     if ($unit) {
 747         if ($unit eq 'K') {
 748             $size = $size * 1024;
 749         } elsif ($unit eq 'M') {
 750             $size = $size * 1024 * 1024;
 751         } elsif ($unit eq 'G') {
 752             $size = $size * 1024 * 1024 * 1024;
 753         }
 754     }
 755     return int($size);
 756 };
 757
 758 my $format_size = sub {
 759     my ($size) = @_;
 760
 761     $size = int($size);
 762
 763     my $kb = int($size/1024);
 764     return $size if $kb*1024 != $size;
 765
 766     my $mb = int($kb/1024);
 767     return "${kb}K" if $mb*1024 != $kb;
 768
 769     my $gb = int($mb/1024);
 770     return "${mb}M" if $gb*1024 != $mb;
 771
 772     return "${gb}G";
 773 };
 774
 775 sub parse_ct_mountpoint {
 776     my ($data) = @_;
 777
 778     $data //= '';
 779
 780     my $res = {};
 781
 782     foreach my $p (split (/,/, $data)) {
 783         next if $p =~ m/^\s*$/;
 784
 785         if ($p =~ m/^(volume|backup|size|mp)=(.+)$/) {
 786             my ($k, $v) = ($1, $2);
 787             return undef if defined($res->{$k});
 788             $res->{$k} = $v;
 789         } else {
 790             if (!$res->{volume} && $p !~ m/=/) {
 791                 $res->{volume} = $p;
 792             } else {
 793                 return undef;
 794             }
 795         }
 796     }
 797
 798     return undef if !defined($res->{volume});
 799
 800     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 801
 802     if ($res->{size}) {
 803         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 804     }
 805
 806     return $res;
 807 }
 808
 809 sub print_ct_mountpoint {
 810     my ($info, $nomp) = @_;
 811
 812     my $opts = '';
 813
 814     die "missing volume\n" if !$info->{volume};
 815
 816     foreach my $o (qw(backup)) {
 817         $opts .= ",$o=$info->{$o}" if defined($info->{$o});
 818     }
 819
 820     if ($info->{size}) {
 821         $opts .= ",size=" . &$format_size($info->{size});
 822     }
 823
 824     $opts .= ",mp=$info->{mp}" if !$nomp;
 825
 826     return "$info->{volume}$opts";
 827 }
 828
 829 sub print_lxc_network {
 830     my $net = shift;
 831
 832     die "no network name defined\n" if !$net->{name};
 833
 834     my $res = "name=$net->{name}";
 835
 836     foreach my $k (qw(hwaddr mtu bridge ip gw ip6 gw6 firewall tag)) {
 837         next if !defined($net->{$k});
 838         $res .= ",$k=$net->{$k}";
 839     }
 840
 841     return $res;
 842 }
 843
 844 sub parse_lxc_network {
 845     my ($data) = @_;
 846
 847     my $res = {};
 848
 849     return $res if !$data;
 850
 851     foreach my $pv (split (/,/, $data)) {
 852         if ($pv =~ m/^(bridge|hwaddr|mtu|name|ip|ip6|gw|gw6|firewall|tag)=(\S+)$/) {
 853             $res->{$1} = $2;
 854         } else {
 855             return undef;
 856         }
 857     }
 858
 859     $res->{type} = 'veth';
 860     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 861
 862     return $res;
 863 }
 864
 865 sub read_cgroup_value {
 866     my ($group, $vmid, $name, $full) = @_;
 867
 868     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 869
 870     return PVE::Tools::file_get_contents($path) if $full;
 871
 872     return PVE::Tools::file_read_firstline($path);
 873 }
 874
 875 sub write_cgroup_value {
 876    my ($group, $vmid, $name, $value) = @_;
 877
 878    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 879    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 880
 881 }
 882
 883 sub find_lxc_console_pids {
 884
 885     my $res = {};
 886
 887     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 888         my ($pid) = @_;
 889
 890         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 891         return if !$cmdline;
 892
 893         my @args = split(/\0/, $cmdline);
 894
 895         # serach for lxc-console -n <vmid>
 896         return if scalar(@args) != 3;
 897         return if $args[1] ne '-n';
 898         return if $args[2] !~ m/^\d+$/;
 899         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 900
 901         my $vmid = $args[2];
 902
 903         push @{$res->{$vmid}}, $pid;
 904     });
 905
 906     return $res;
 907 }
 908
 909 sub find_lxc_pid {
 910     my ($vmid) = @_;
 911
 912     my $pid = undef;
 913     my $parser = sub {
 914         my $line = shift;
 915         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 916     };
 917     PVE::Tools::run_command(['lxc-info', '-n', $vmid], outfunc => $parser);
 918
 919     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 920
 921     return $pid;
 922 }
 923
 924 my $ipv4_reverse_mask = [
 925     '0.0.0.0',
 926     '128.0.0.0',
 927     '192.0.0.0',
 928     '224.0.0.0',
 929     '240.0.0.0',
 930     '248.0.0.0',
 931     '252.0.0.0',
 932     '254.0.0.0',
 933     '255.0.0.0',
 934     '255.128.0.0',
 935     '255.192.0.0',
 936     '255.224.0.0',
 937     '255.240.0.0',
 938     '255.248.0.0',
 939     '255.252.0.0',
 940     '255.254.0.0',
 941     '255.255.0.0',
 942     '255.255.128.0',
 943     '255.255.192.0',
 944     '255.255.224.0',
 945     '255.255.240.0',
 946     '255.255.248.0',
 947     '255.255.252.0',
 948     '255.255.254.0',
 949     '255.255.255.0',
 950     '255.255.255.128',
 951     '255.255.255.192',
 952     '255.255.255.224',
 953     '255.255.255.240',
 954     '255.255.255.248',
 955     '255.255.255.252',
 956     '255.255.255.254',
 957     '255.255.255.255',
 958 ];
 959
 960 # Note: we cannot use Net:IP, because that only allows strict
 961 # CIDR networks
 962 sub parse_ipv4_cidr {
 963     my ($cidr, $noerr) = @_;
 964
 965     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
 966         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
 967     }
 968
 969     return undef if $noerr;
 970
 971     die "unable to parse ipv4 address/mask\n";
 972 }
 973
 974 sub check_lock {
 975     my ($conf) = @_;
 976
 977     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
 978 }
 979
 980 sub update_lxc_config {
 981     my ($storage_cfg, $vmid, $conf) = @_;
 982
 983     my $dir = "/var/lib/lxc/$vmid";
 984
 985     if ($conf->{template}) {
 986
 987         unlink "$dir/config";
 988
 989         return;
 990     }
 991
 992     my $raw = '';
 993
 994     die "missing 'arch' - internal error" if !$conf->{arch};
 995     $raw .= "lxc.arch = $conf->{arch}\n";
 996
 997     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
 998     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
 999         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1000     } else {
1001         die "implement me";
1002     }
1003
1004     if (!has_dev_console($conf)) {
1005         $raw .= "lxc.console = none\n";
1006         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1007     }
1008
1009     my $ttycount = get_tty_count($conf);
1010     $raw .= "lxc.tty = $ttycount\n";
1011
1012     my $utsname = $conf->{hostname} || "CT$vmid";
1013     $raw .= "lxc.utsname = $utsname\n";
1014
1015     my $memory = $conf->{memory} || 512;
1016     my $swap = $conf->{swap} // 0;
1017
1018     my $lxcmem = int($memory*1024*1024);
1019     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1020
1021     my $lxcswap = int(($memory + $swap)*1024*1024);
1022     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1023
1024     if (my $cpulimit = $conf->{cpulimit}) {
1025         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1026         my $value = int(100000*$cpulimit);
1027         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1028     }
1029
1030     my $shares = $conf->{cpuunits} || 1024;
1031     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1032
1033     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1034     $mountpoint->{mp} = '/';
1035     my $volid = $mountpoint->{volume};
1036     my $path = mountpoint_mount_path($mountpoint, $storage_cfg);
1037
1038     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1039
1040     if ($storage) {
1041         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1042         $path = "loop:$path" if $scfg->{path};
1043     }
1044
1045     $raw .= "lxc.rootfs = $path\n";
1046
1047     my $netcount = 0;
1048     foreach my $k (keys %$conf) {
1049         next if $k !~ m/^net(\d+)$/;
1050         my $ind = $1;
1051         my $d = parse_lxc_network($conf->{$k});
1052         $netcount++;
1053         $raw .= "lxc.network.type = veth\n";
1054         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1055         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1056         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1057         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1058     }
1059
1060     if (my $lxcconf = $conf->{lxc}) {
1061         foreach my $entry (@$lxcconf) {
1062             my ($k, $v) = @$entry;
1063             $netcount++ if $k eq 'lxc.network.type';
1064             $raw .= "$k = $v\n";
1065         }
1066     }
1067
1068     $raw .= "lxc.network.type = empty\n" if !$netcount;
1069
1070     File::Path::mkpath("$dir/rootfs");
1071
1072     PVE::Tools::file_set_contents("$dir/config", $raw);
1073 }
1074
1075 # verify and cleanup nameserver list (replace \0 with ' ')
1076 sub verify_nameserver_list {
1077     my ($nameserver_list) = @_;
1078
1079     my @list = ();
1080     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1081         PVE::JSONSchema::pve_verify_ip($server);
1082         push @list, $server;
1083     }
1084
1085     return join(' ', @list);
1086 }
1087
1088 sub verify_searchdomain_list {
1089     my ($searchdomain_list) = @_;
1090
1091     my @list = ();
1092     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1093         # todo: should we add checks for valid dns domains?
1094         push @list, $server;
1095     }
1096
1097     return join(' ', @list);
1098 }
1099
1100 sub update_pct_config {
1101     my ($vmid, $conf, $running, $param, $delete) = @_;
1102
1103     my @nohotplug;
1104
1105     my $new_disks = [];
1106
1107     my $rootdir;
1108     if ($running) {
1109         my $pid = find_lxc_pid($vmid);
1110         $rootdir = "/proc/$pid/root";
1111     }
1112
1113     if (defined($delete)) {
1114         foreach my $opt (@$delete) {
1115             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1116                 die "unable to delete required option '$opt'\n";
1117             } elsif ($opt eq 'swap') {
1118                 delete $conf->{$opt};
1119                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1120             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1121                 delete $conf->{$opt};
1122             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1123                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1124                 delete $conf->{$opt};
1125                 push @nohotplug, $opt;
1126                 next if $running;
1127             } elsif ($opt =~ m/^net(\d)$/) {
1128                 delete $conf->{$opt};
1129                 next if !$running;
1130                 my $netid = $1;
1131                 PVE::Network::veth_delete("veth${vmid}i$netid");
1132             } elsif ($opt eq 'protection') {
1133                 delete $conf->{$opt};
1134             } elsif ($opt =~ m/^mp(\d+)$/) {
1135                 delete $conf->{$opt};
1136                 push @nohotplug, $opt;
1137                 next if $running;
1138             } elsif ($opt eq 'rootfs') {
1139                 die "implement me"
1140             } else {
1141                 die "implement me"
1142             }
1143             write_config($vmid, $conf) if $running;
1144         }
1145     }
1146
1147     # There's no separate swap size to configure, there's memory and "total"
1148     # memory (iow. memory+swap). This means we have to change them together.
1149     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1150     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1151     if (defined($wanted_memory) || defined($wanted_swap)) {
1152
1153         $wanted_memory //= ($conf->{memory} || 512);
1154         $wanted_swap //=  ($conf->{swap} || 0);
1155
1156         my $total = $wanted_memory + $wanted_swap;
1157         if ($running) {
1158             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1159             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1160         }
1161         $conf->{memory} = $wanted_memory;
1162         $conf->{swap} = $wanted_swap;
1163
1164         write_config($vmid, $conf) if $running;
1165     }
1166
1167     foreach my $opt (keys %$param) {
1168         my $value = $param->{$opt};
1169         if ($opt eq 'hostname') {
1170             $conf->{$opt} = $value;
1171         } elsif ($opt eq 'onboot') {
1172             $conf->{$opt} = $value ? 1 : 0;
1173         } elsif ($opt eq 'startup') {
1174             $conf->{$opt} = $value;
1175         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1176             $conf->{$opt} = $value;
1177             push @nohotplug, $opt;
1178             next if $running;
1179         } elsif ($opt eq 'nameserver') {
1180             my $list = verify_nameserver_list($value);
1181             $conf->{$opt} = $list;
1182             push @nohotplug, $opt;
1183             next if $running;
1184         } elsif ($opt eq 'searchdomain') {
1185             my $list = verify_searchdomain_list($value);
1186             $conf->{$opt} = $list;
1187             push @nohotplug, $opt;
1188             next if $running;
1189         } elsif ($opt eq 'cpulimit') {
1190             $conf->{$opt} = $value;
1191             push @nohotplug, $opt; # fixme: hotplug
1192             next;
1193         } elsif ($opt eq 'cpuunits') {
1194             $conf->{$opt} = $value;
1195             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1196         } elsif ($opt eq 'description') {
1197             $conf->{$opt} = PVE::Tools::encode_text($value);
1198         } elsif ($opt =~ m/^net(\d+)$/) {
1199             my $netid = $1;
1200             my $net = parse_lxc_network($value);
1201             if (!$running) {
1202                 $conf->{$opt} = print_lxc_network($net);
1203             } else {
1204                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1205             }
1206         } elsif ($opt eq 'protection') {
1207             $conf->{$opt} = $value ? 1 : 0;
1208         } elsif ($opt =~ m/^mp(\d+)$/) {
1209             $conf->{$opt} = $value;
1210             push @$new_disks, $opt;
1211             push @nohotplug, $opt;
1212             next;
1213         } elsif ($opt eq 'rootfs') {
1214             die "implement me: $opt";
1215         } else {
1216             die "implement me: $opt";
1217         }
1218         write_config($vmid, $conf) if $running;
1219     }
1220
1221     if ($running && scalar(@nohotplug)) {
1222         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1223     }
1224
1225     if (@$new_disks) {
1226         my $storage_cfg = PVE::Storage::config();
1227         create_disks($storage_cfg, $vmid, $conf, $conf);
1228         mount_all($vmid, $storage_cfg, $conf, $new_disks, 1);
1229         umount_all($vmid, $storage_cfg, $conf, 0);
1230     }
1231 }
1232
1233 sub has_dev_console {
1234     my ($conf) = @_;
1235
1236     return !(defined($conf->{console}) && !$conf->{console});
1237 }
1238
1239 sub get_tty_count {
1240     my ($conf) = @_;
1241
1242     return $conf->{tty} // $confdesc->{tty}->{default};
1243 }
1244
1245 sub get_cmode {
1246     my ($conf) = @_;
1247
1248     return $conf->{cmode} // $confdesc->{cmode}->{default};
1249 }
1250
1251 sub get_console_command {
1252     my ($vmid, $conf) = @_;
1253
1254     my $cmode = get_cmode($conf);
1255
1256     if ($cmode eq 'console') {
1257         return ['lxc-console', '-n',  $vmid, '-t', 0];
1258     } elsif ($cmode eq 'tty') {
1259         return ['lxc-console', '-n',  $vmid];
1260     } elsif ($cmode eq 'shell') {
1261         return ['lxc-attach', '--clear-env', '-n', $vmid];
1262     } else {
1263         die "internal error";
1264     }
1265 }
1266
1267 sub get_primary_ips {
1268     my ($conf) = @_;
1269
1270     # return data from net0
1271
1272     return undef if !defined($conf->{net0});
1273     my $net = parse_lxc_network($conf->{net0});
1274
1275     my $ipv4 = $net->{ip};
1276     if ($ipv4) {
1277         if ($ipv4 =~ /^(dhcp|manual)$/) {
1278             $ipv4 = undef
1279         } else {
1280             $ipv4 =~ s!/\d+$!!;
1281         }
1282     }
1283     my $ipv6 = $net->{ip6};
1284     if ($ipv6) {
1285         if ($ipv6 =~ /^(dhcp|manual)$/) {
1286             $ipv6 = undef;
1287         } else {
1288             $ipv6 =~ s!/\d+$!!;
1289         }
1290     }
1291
1292     return ($ipv4, $ipv6);
1293 }
1294
1295
1296 sub destroy_lxc_container {
1297     my ($storage_cfg, $vmid, $conf) = @_;
1298
1299     foreach_mountpoint($conf, sub {
1300         my ($ms, $mountpoint) = @_;
1301         my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $mountpoint->{volume});
1302         PVE::Storage::vdisk_free($storage_cfg, $mountpoint->{volume}) if $vmid == $owner;
1303     });
1304
1305     rmdir "/var/lib/lxc/$vmid/rootfs";
1306     unlink "/var/lib/lxc/$vmid/config";
1307     rmdir "/var/lib/lxc/$vmid";
1308     destroy_config($vmid);
1309
1310     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1311     #PVE::Tools::run_command($cmd);
1312 }
1313
1314 sub vm_stop_cleanup {
1315     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1316
1317     eval {
1318         if (!$keepActive) {
1319
1320             my $vollist = get_vm_volumes($conf);
1321             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1322         }
1323     };
1324     warn $@ if $@; # avoid errors - just warn
1325 }
1326
1327 my $safe_num_ne = sub {
1328     my ($a, $b) = @_;
1329
1330     return 0 if !defined($a) && !defined($b);
1331     return 1 if !defined($a);
1332     return 1 if !defined($b);
1333
1334     return $a != $b;
1335 };
1336
1337 my $safe_string_ne = sub {
1338     my ($a, $b) = @_;
1339
1340     return 0 if !defined($a) && !defined($b);
1341     return 1 if !defined($a);
1342     return 1 if !defined($b);
1343
1344     return $a ne $b;
1345 };
1346
1347 sub update_net {
1348     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1349
1350     if ($newnet->{type} ne 'veth') {
1351         # for when there are physical interfaces
1352         die "cannot update interface of type $newnet->{type}";
1353     }
1354
1355     my $veth = "veth${vmid}i${netid}";
1356     my $eth = $newnet->{name};
1357
1358     if (my $oldnetcfg = $conf->{$opt}) {
1359         my $oldnet = parse_lxc_network($oldnetcfg);
1360
1361         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1362             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1363
1364             PVE::Network::veth_delete($veth);
1365             delete $conf->{$opt};
1366             write_config($vmid, $conf);
1367
1368             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1369
1370         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1371                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1372                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1373
1374                 if ($oldnet->{bridge}) {
1375                     PVE::Network::tap_unplug($veth);
1376                     foreach (qw(bridge tag firewall)) {
1377                         delete $oldnet->{$_};
1378                     }
1379                     $conf->{$opt} = print_lxc_network($oldnet);
1380                     write_config($vmid, $conf);
1381                 }
1382
1383                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1384                 foreach (qw(bridge tag firewall)) {
1385                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1386                 }
1387                 $conf->{$opt} = print_lxc_network($oldnet);
1388                 write_config($vmid, $conf);
1389         }
1390     } else {
1391         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1392     }
1393
1394     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1395 }
1396
1397 sub hotplug_net {
1398     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1399
1400     my $veth = "veth${vmid}i${netid}";
1401     my $vethpeer = $veth . "p";
1402     my $eth = $newnet->{name};
1403
1404     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1405     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1406
1407     # attach peer in container
1408     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1409     PVE::Tools::run_command($cmd);
1410
1411     # link up peer in container
1412     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1413     PVE::Tools::run_command($cmd);
1414
1415     my $done = { type => 'veth' };
1416     foreach (qw(bridge tag firewall hwaddr name)) {
1417         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1418     }
1419     $conf->{$opt} = print_lxc_network($done);
1420
1421     write_config($vmid, $conf);
1422 }
1423
1424 sub update_ipconfig {
1425     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1426
1427     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1428
1429     my $optdata = parse_lxc_network($conf->{$opt});
1430     my $deleted = [];
1431     my $added = [];
1432     my $nscmd = sub {
1433         my $cmdargs = shift;
1434         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1435     };
1436     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1437
1438     my $change_ip_config = sub {
1439         my ($ipversion) = @_;
1440
1441         my $family_opt = "-$ipversion";
1442         my $suffix = $ipversion == 4 ? '' : $ipversion;
1443         my $gw= "gw$suffix";
1444         my $ip= "ip$suffix";
1445
1446         my $newip = $newnet->{$ip};
1447         my $newgw = $newnet->{$gw};
1448         my $oldip = $optdata->{$ip};
1449
1450         my $change_ip = &$safe_string_ne($oldip, $newip);
1451         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1452
1453         return if !$change_ip && !$change_gw;
1454
1455         # step 1: add new IP, if this fails we cancel
1456         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1457             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1458             if (my $err = $@) {
1459                 warn $err;
1460                 return;
1461             }
1462         }
1463
1464         # step 2: replace gateway
1465         #   If this fails we delete the added IP and cancel.
1466         #   If it succeeds we save the config and delete the old IP, ignoring
1467         #   errors. The config is then saved.
1468         # Note: 'ip route replace' can add
1469         if ($change_gw) {
1470             if ($newgw) {
1471                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1472                 if (my $err = $@) {
1473                     warn $err;
1474                     # the route was not replaced, the old IP is still available
1475                     # rollback (delete new IP) and cancel
1476                     if ($change_ip) {
1477                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1478                         warn $@ if $@; # no need to die here
1479                     }
1480                     return;
1481                 }
1482             } else {
1483                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1484                 # if the route was not deleted, the guest might have deleted it manually
1485                 # warn and continue
1486                 warn $@ if $@;
1487             }
1488         }
1489
1490         # from this point on we save the configuration
1491         # step 3: delete old IP ignoring errors
1492         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1493             # We need to enable promote_secondaries, otherwise our newly added
1494             # address will be removed along with the old one.
1495             my $promote = 0;
1496             eval {
1497                 if ($ipversion == 4) {
1498                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1499                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1500                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1501                 }
1502                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1503             };
1504             warn $@ if $@; # no need to die here
1505
1506             if ($ipversion == 4) {
1507                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1508             }
1509         }
1510
1511         foreach my $property ($ip, $gw) {
1512             if ($newnet->{$property}) {
1513                 $optdata->{$property} = $newnet->{$property};
1514             } else {
1515                 delete $optdata->{$property};
1516             }
1517         }
1518         $conf->{$opt} = print_lxc_network($optdata);
1519         write_config($vmid, $conf);
1520         $lxc_setup->setup_network($conf);
1521     };
1522
1523     &$change_ip_config(4);
1524     &$change_ip_config(6);
1525
1526 }
1527
1528 # Internal snapshots
1529
1530 # NOTE: Snapshot create/delete involves several non-atomic
1531 # action, and can take a long time.
1532 # So we try to avoid locking the file and use 'lock' variable
1533 # inside the config file instead.
1534
1535 my $snapshot_copy_config = sub {
1536     my ($source, $dest) = @_;
1537
1538     foreach my $k (keys %$source) {
1539         next if $k eq 'snapshots';
1540         next if $k eq 'snapstate';
1541         next if $k eq 'snaptime';
1542         next if $k eq 'vmstate';
1543         next if $k eq 'lock';
1544         next if $k eq 'digest';
1545         next if $k eq 'description';
1546
1547         $dest->{$k} = $source->{$k};
1548     }
1549 };
1550
1551 my $snapshot_prepare = sub {
1552     my ($vmid, $snapname, $comment) = @_;
1553
1554     my $snap;
1555
1556     my $updatefn =  sub {
1557
1558         my $conf = load_config($vmid);
1559
1560         die "you can't take a snapshot if it's a template\n"
1561             if is_template($conf);
1562
1563         check_lock($conf);
1564
1565         $conf->{lock} = 'snapshot';
1566
1567         die "snapshot name '$snapname' already used\n"
1568             if defined($conf->{snapshots}->{$snapname});
1569
1570         my $storecfg = PVE::Storage::config();
1571         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1572
1573         $snap = $conf->{snapshots}->{$snapname} = {};
1574
1575         &$snapshot_copy_config($conf, $snap);
1576
1577         $snap->{'snapstate'} = "prepare";
1578         $snap->{'snaptime'} = time();
1579         $snap->{'description'} = $comment if $comment;
1580         $conf->{snapshots}->{$snapname} = $snap;
1581
1582         write_config($vmid, $conf);
1583     };
1584
1585     lock_container($vmid, 10, $updatefn);
1586
1587     return $snap;
1588 };
1589
1590 my $snapshot_commit = sub {
1591     my ($vmid, $snapname) = @_;
1592
1593     my $updatefn = sub {
1594
1595         my $conf = load_config($vmid);
1596
1597         die "missing snapshot lock\n"
1598             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1599
1600         die "snapshot '$snapname' does not exist\n"
1601             if !defined($conf->{snapshots}->{$snapname});
1602
1603         die "wrong snapshot state\n"
1604             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1605                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1606
1607         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1608         delete $conf->{lock};
1609         $conf->{parent} = $snapname;
1610
1611         write_config($vmid, $conf);
1612     };
1613
1614     lock_container($vmid, 10 ,$updatefn);
1615 };
1616
1617 sub has_feature {
1618     my ($feature, $conf, $storecfg, $snapname) = @_;
1619
1620     my $err;
1621
1622     foreach_mountpoint($conf, sub {
1623         my ($ms, $mountpoint) = @_;
1624
1625         return if $err; # skip further test
1626
1627         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1628
1629         # TODO: implement support for mountpoints
1630         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1631             if $ms ne 'rootfs';
1632     });
1633
1634     return $err ? 0 : 1;
1635 }
1636
1637 sub snapshot_create {
1638     my ($vmid, $snapname, $comment) = @_;
1639
1640     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1641
1642     my $conf = load_config($vmid);
1643
1644     my $cmd = "/usr/bin/lxc-freeze -n $vmid";
1645     my $running = check_running($vmid);
1646     eval {
1647         if ($running) {
1648             PVE::Tools::run_command($cmd);
1649         };
1650
1651         my $storecfg = PVE::Storage::config();
1652         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1653         my $volid = $rootinfo->{volume};
1654
1655         $cmd = "/usr/bin/lxc-unfreeze -n $vmid";
1656         if ($running) {
1657             PVE::Tools::run_command($cmd);
1658         };
1659
1660         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1661         &$snapshot_commit($vmid, $snapname);
1662     };
1663     if(my $err = $@) {
1664         snapshot_delete($vmid, $snapname, 1);
1665         die "$err\n";
1666     }
1667 }
1668
1669 sub snapshot_delete {
1670     my ($vmid, $snapname, $force) = @_;
1671
1672     my $snap;
1673
1674     my $conf;
1675
1676     my $updatefn =  sub {
1677
1678         $conf = load_config($vmid);
1679
1680         die "you can't delete a snapshot if vm is a template\n"
1681             if is_template($conf);
1682
1683         $snap = $conf->{snapshots}->{$snapname};
1684
1685         check_lock($conf);
1686
1687         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1688
1689         $snap->{snapstate} = 'delete';
1690
1691         write_config($vmid, $conf);
1692     };
1693
1694     lock_container($vmid, 10, $updatefn);
1695
1696     my $storecfg = PVE::Storage::config();
1697
1698     my $del_snap =  sub {
1699
1700         check_lock($conf);
1701
1702         if ($conf->{parent} eq $snapname) {
1703             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1704                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1705             } else {
1706                 delete $conf->{parent};
1707             }
1708         }
1709
1710         delete $conf->{snapshots}->{$snapname};
1711
1712         write_config($vmid, $conf);
1713     };
1714
1715     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1716     my $rootinfo = parse_ct_mountpoint($rootfs);
1717     my $volid = $rootinfo->{volume};
1718
1719     eval {
1720         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1721     };
1722     my $err = $@;
1723
1724     if(!$err || ($err && $force)) {
1725         lock_container($vmid, 10, $del_snap);
1726         if ($err) {
1727             die "Can't delete snapshot: $vmid $snapname $err\n";
1728         }
1729     }
1730 }
1731
1732 sub snapshot_rollback {
1733     my ($vmid, $snapname) = @_;
1734
1735     my $storecfg = PVE::Storage::config();
1736
1737     my $conf = load_config($vmid);
1738
1739     die "you can't rollback if vm is a template\n" if is_template($conf);
1740
1741     my $snap = $conf->{snapshots}->{$snapname};
1742
1743     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1744
1745     my $rootfs = $snap->{rootfs};
1746     my $rootinfo = parse_ct_mountpoint($rootfs);
1747     my $volid = $rootinfo->{volume};
1748
1749     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1750
1751     my $updatefn = sub {
1752
1753         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1754             if $snap->{snapstate};
1755
1756         check_lock($conf);
1757
1758         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1759
1760         die "unable to rollback vm $vmid: vm is running\n"
1761             if check_running($vmid);
1762
1763         $conf->{lock} = 'rollback';
1764
1765         my $forcemachine;
1766
1767         # copy snapshot config to current config
1768
1769         my $tmp_conf = $conf;
1770         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1771         $conf->{snapshots} = $tmp_conf->{snapshots};
1772         delete $conf->{snaptime};
1773         delete $conf->{snapname};
1774         $conf->{parent} = $snapname;
1775
1776         write_config($vmid, $conf);
1777     };
1778
1779     my $unlockfn = sub {
1780         delete $conf->{lock};
1781         write_config($vmid, $conf);
1782     };
1783
1784     lock_container($vmid, 10, $updatefn);
1785
1786     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1787
1788     lock_container($vmid, 5, $unlockfn);
1789 }
1790
1791 sub template_create {
1792     my ($vmid, $conf) = @_;
1793
1794     my $storecfg = PVE::Storage::config();
1795
1796     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1797     my $volid = $rootinfo->{volume};
1798
1799     die "Template feature is not available for '$volid'\n"
1800         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1801
1802     PVE::Storage::activate_volumes($storecfg, [$volid]);
1803
1804     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1805     $rootinfo->{volume} = $template_volid;
1806     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1807
1808     write_config($vmid, $conf);
1809 }
1810
1811 sub is_template {
1812     my ($conf) = @_;
1813
1814     return 1 if defined $conf->{template} && $conf->{template} == 1;
1815 }
1816
1817 sub mountpoint_names {
1818     my ($reverse) = @_;
1819
1820     my @names = ('rootfs');
1821
1822     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1823         push @names, "mp$i";
1824     }
1825
1826     return $reverse ? reverse @names : @names;
1827 }
1828
1829 sub foreach_mountpoint_full {
1830     my ($conf, $reverse, $func) = @_;
1831
1832     foreach my $key (mountpoint_names($reverse)) {
1833         my $value = $conf->{$key};
1834         next if !defined($value);
1835         my $mountpoint = parse_ct_mountpoint($value);
1836         $mountpoint->{mp} = '/' if $key eq 'rootfs'; # just to be sure
1837         &$func($key, $mountpoint);
1838     }
1839 }
1840
1841 sub foreach_mountpoint {
1842     my ($conf, $func) = @_;
1843
1844     foreach_mountpoint_full($conf, 0, $func);
1845 }
1846
1847 sub foreach_mountpoint_reverse {
1848     my ($conf, $func) = @_;
1849
1850     foreach_mountpoint_full($conf, 1, $func);
1851 }
1852
1853 sub check_ct_modify_config_perm {
1854     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1855
1856     return 1 if $authuser ne 'root@pam';
1857
1858     foreach my $opt (@$key_list) {
1859
1860         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1861             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1862         } elsif ($opt eq 'disk') {
1863             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1864         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1865             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1866         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1867                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1868             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1869         } else {
1870             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1871         }
1872     }
1873
1874     return 1;
1875 }
1876
1877 sub umount_all {
1878     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
1879
1880     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1881     my $volid_list = get_vm_volumes($conf);
1882
1883     foreach_mountpoint_reverse($conf, sub {
1884         my ($ms, $mountpoint) = @_;
1885
1886         my $volid = $mountpoint->{volume};
1887         my $mount = $mountpoint->{mp};
1888
1889         return if !$volid || !$mount;
1890
1891         my $mount_path = "$rootdir/$mount";
1892         $mount_path =~ s!/+!/!g;
1893
1894         return if !PVE::ProcFSTools::is_mounted($mount_path);
1895
1896         eval {
1897             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1898         };
1899         if (my $err = $@) {
1900             if ($noerr) {
1901                 warn $err;
1902             } else {
1903                 die $err;
1904             }
1905         }
1906     });
1907 }
1908
1909 sub mount_all {
1910     my ($vmid, $storage_cfg, $conf, $mkdirs) = @_;
1911
1912     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1913     File::Path::make_path($rootdir);
1914
1915     my $volid_list = get_vm_volumes($conf);
1916     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
1917
1918     eval {
1919         foreach_mountpoint($conf, sub {
1920             my ($ms, $mountpoint) = @_;
1921
1922             my $volid = $mountpoint->{volume};
1923             my $mount = $mountpoint->{mp};
1924
1925             return if !$volid || !$mount;
1926
1927             my $image_path = PVE::Storage::path($storage_cfg, $volid);
1928             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1929                 PVE::Storage::parse_volname($storage_cfg, $volid);
1930
1931             die "unable to mount base volume - internal error" if $isBase;
1932
1933             File::Path::make_path "$rootdir/$mount" if $mkdirs;
1934             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
1935         });
1936     };
1937     if (my $err = $@) {
1938         warn "mounting container failed - $err";
1939         umount_all($vmid, $storage_cfg, $conf, 1);
1940     }
1941
1942     return $rootdir;
1943 }
1944
1945
1946 sub mountpoint_mount_path {
1947     my ($mountpoint, $storage_cfg, $snapname) = @_;
1948
1949     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
1950 }
1951
1952 # use $rootdir = undef to just return the corresponding mount path
1953 sub mountpoint_mount {
1954     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
1955
1956     my $volid = $mountpoint->{volume};
1957     my $mount = $mountpoint->{mp};
1958
1959     return if !$volid || !$mount;
1960
1961     my $mount_path;
1962
1963     if (defined($rootdir)) {
1964         $rootdir =~ s!/+$!!;
1965         $mount_path = "$rootdir/$mount";
1966         $mount_path =~ s!/+!/!g;
1967         File::Path::mkpath($mount_path);
1968     }
1969
1970     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1971
1972     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
1973
1974     if ($storage) {
1975
1976         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1977         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
1978         return $path if !$mount_path;
1979
1980         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1981             PVE::Storage::parse_volname($storage_cfg, $volid);
1982
1983         if ($format eq 'subvol') {
1984             if ($snapname) {
1985                 if ($scfg->{type} eq 'zfspool') {
1986                     my $path_arg = $path;
1987                     $path_arg =~ s!^/+!!;
1988                     PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
1989                 } else {
1990                     die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
1991                 }
1992             } else {
1993                 PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
1994             }
1995             return $path;
1996         } elsif ($format eq 'raw') {
1997             my @extra_opts;
1998             if ($scfg->{path}) {
1999                 push @extra_opts, '-o', 'loop';
2000             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'rbd') {
2001                 # do nothing
2002             } else {
2003                 die "unsupported storage type '$scfg->{type}'\n";
2004             }
2005             if ($isBase || defined($snapname)) {
2006                 PVE::Tools::run_command(['mount', '-o', "ro", @extra_opts, $path, $mount_path]);
2007             } else {
2008                 PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2009             }
2010             return $path;
2011         } else {
2012             die "unsupported image format '$format'\n";
2013         }
2014     } elsif ($volid =~ m|^/dev/.+|) {
2015         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2016         return $volid;
2017     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2018         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2019         return $volid;
2020     }
2021
2022     die "unsupported storage";
2023 }
2024
2025 sub get_vm_volumes {
2026     my ($conf, $excludes) = @_;
2027
2028     my $vollist = [];
2029
2030     foreach_mountpoint($conf, sub {
2031         my ($ms, $mountpoint) = @_;
2032
2033         return if $excludes && $ms eq $excludes;
2034
2035         my $volid = $mountpoint->{volume};
2036
2037         return if !$volid || $volid =~ m|^/|;
2038
2039         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2040         return if !$sid;
2041
2042         push @$vollist, $volid;
2043     });
2044
2045     return $vollist;
2046 }
2047
2048 sub mkfs {
2049     my ($dev) = @_;
2050
2051     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2052 }
2053
2054 sub format_disk {
2055     my ($storage_cfg, $volid) = @_;
2056
2057     if ($volid =~ m!^/dev/.+!) {
2058         mkfs($volid);
2059         return;
2060     }
2061
2062     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2063
2064     die "cannot format volume '$volid' with no storage\n" if !$storage;
2065
2066     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2067
2068     my $path = PVE::Storage::path($storage_cfg, $volid);
2069
2070     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2071         PVE::Storage::parse_volname($storage_cfg, $volid);
2072
2073     die "cannot format volume '$volid' (format == $format)\n"
2074         if $format ne 'raw';
2075
2076     mkfs($path);
2077 }
2078
2079 sub destroy_disks {
2080     my ($storecfg, $vollist) = @_;
2081
2082     foreach my $volid (@$vollist) {
2083         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2084         warn $@ if $@;
2085     }
2086 }
2087
2088 sub create_disks {
2089     my ($storecfg, $vmid, $settings, $conf) = @_;
2090
2091     my $vollist = [];
2092
2093     eval {
2094         foreach_mountpoint($settings, sub {
2095             my ($ms, $mountpoint) = @_;
2096
2097             my $volid = $mountpoint->{volume};
2098             my $mp = $mountpoint->{mp};
2099
2100             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2101
2102             return if !$storage;
2103
2104             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2105                 my ($storeid, $size_gb) = ($1, $2);
2106
2107                 my $size_kb = int(${size_gb}*1024) * 1024;
2108
2109                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2110                 # fixme: use better naming ct-$vmid-disk-X.raw?
2111
2112                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2113                     if ($size_kb > 0) {
2114                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2115                                                            undef, $size_kb);
2116                         format_disk($storecfg, $volid);
2117                     } else {
2118                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2119                                                            undef, 0);
2120                     }
2121                 } elsif ($scfg->{type} eq 'zfspool') {
2122
2123                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2124                                                        undef, $size_kb);
2125                 } elsif ($scfg->{type} eq 'drbd') {
2126
2127                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2128                     format_disk($storecfg, $volid);
2129
2130                 } elsif ($scfg->{type} eq 'rbd') {
2131
2132                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2133                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2134                     format_disk($storecfg, $volid);
2135                 } else {
2136                     die "unable to create containers on storage type '$scfg->{type}'\n";
2137                 }
2138                 push @$vollist, $volid;
2139                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2140                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2141             } else {
2142                 # use specified/existing volid
2143             }
2144         });
2145     };
2146     # free allocated images on error
2147     if (my $err = $@) {
2148         destroy_disks($storecfg, $vollist);
2149         die $err;
2150     }
2151     return $vollist;
2152 }
2153
2154 # bash completion helper
2155
2156 sub complete_os_templates {
2157     my ($cmdname, $pname, $cvalue) = @_;
2158
2159     my $cfg = PVE::Storage::config();
2160
2161     my $storeid;
2162
2163     if ($cvalue =~ m/^([^:]+):/) {
2164         $storeid = $1;
2165     }
2166
2167     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2168     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2169
2170     my $res = [];
2171     foreach my $id (keys %$data) {
2172         foreach my $item (@{$data->{$id}}) {
2173             push @$res, $item->{volid} if defined($item->{volid});
2174         }
2175     }
2176
2177     return $res;
2178 }
2179
2180 my $complete_ctid_full = sub {
2181     my ($running) = @_;
2182
2183     my $idlist = vmstatus();
2184
2185     my $active_hash = list_active_containers();
2186
2187     my $res = [];
2188
2189     foreach my $id (keys %$idlist) {
2190         my $d = $idlist->{$id};
2191         if (defined($running)) {
2192             next if $d->{template};
2193             next if $running && !$active_hash->{$id};
2194             next if !$running && $active_hash->{$id};
2195         }
2196         push @$res, $id;
2197
2198     }
2199     return $res;
2200 };
2201
2202 sub complete_ctid {
2203     return &$complete_ctid_full();
2204 }
2205
2206 sub complete_ctid_stopped {
2207     return &$complete_ctid_full(0);
2208 }
2209
2210 sub complete_ctid_running {
2211     return &$complete_ctid_full(1);
2212 }
2213
2214 1;