src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use Fcntl ':flock';
   9
  10 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  11 use PVE::Storage;
  12 use PVE::SafeSyslog;
  13 use PVE::INotify;
  14 use PVE::JSONSchema qw(get_standard_option);
  15 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  16 use PVE::Network;
  17 use PVE::AccessControl;
  18 use PVE::ProcFSTools;
  19
  20 use Data::Dumper;
  21
  22 my $nodename = PVE::INotify::nodename();
  23
  24 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  25
  26 PVE::JSONSchema::register_format('pve-lxc-network', \&verify_lxc_network);
  27 sub verify_lxc_network {
  28     my ($value, $noerr) = @_;
  29
  30     return $value if parse_lxc_network($value);
  31
  32     return undef if $noerr;
  33
  34     die "unable to parse network setting\n";
  35 }
  36
  37 PVE::JSONSchema::register_format('pve-ct-mountpoint', \&verify_ct_mountpoint);
  38 sub verify_ct_mountpoint {
  39     my ($value, $noerr) = @_;
  40
  41     return $value if parse_ct_mountpoint($value);
  42
  43     return undef if $noerr;
  44
  45     die "unable to parse CT mountpoint options\n";
  46 }
  47
  48 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  49     type => 'string', format => 'pve-ct-mountpoint',
  50     typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+]',
  51     description => "Use volume as container root.",
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  56     description => "The name of the snapshot.",
  57     type => 'string', format => 'pve-configid',
  58     maxLength => 40,
  59 });
  60
  61 my $confdesc = {
  62     lock => {
  63         optional => 1,
  64         type => 'string',
  65         description => "Lock/unlock the VM.",
  66         enum => [qw(migrate backup snapshot rollback)],
  67     },
  68     onboot => {
  69         optional => 1,
  70         type => 'boolean',
  71         description => "Specifies whether a VM will be started during system bootup.",
  72         default => 0,
  73     },
  74     startup => get_standard_option('pve-startup-order'),
  75     template => {
  76         optional => 1,
  77         type => 'boolean',
  78         description => "Enable/disable Template.",
  79         default => 0,
  80     },
  81     arch => {
  82         optional => 1,
  83         type => 'string',
  84         enum => ['amd64', 'i386'],
  85         description => "OS architecture type.",
  86         default => 'amd64',
  87     },
  88     ostype => {
  89         optional => 1,
  90         type => 'string',
  91         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  92         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  93     },
  94     console => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Attach a console device (/dev/console) to the container.",
  98         default => 1,
  99     },
 100     tty => {
 101         optional => 1,
 102         type => 'integer',
 103         description => "Specify the number of tty available to the container",
 104         minimum => 0,
 105         maximum => 6,
 106         default => 2,
 107     },
 108     cpulimit => {
 109         optional => 1,
 110         type => 'number',
 111         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 112         minimum => 0,
 113         maximum => 128,
 114         default => 0,
 115     },
 116     cpuunits => {
 117         optional => 1,
 118         type => 'integer',
 119         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 120         minimum => 0,
 121         maximum => 500000,
 122         default => 1024,
 123     },
 124     memory => {
 125         optional => 1,
 126         type => 'integer',
 127         description => "Amount of RAM for the VM in MB.",
 128         minimum => 16,
 129         default => 512,
 130     },
 131     swap => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "Amount of SWAP for the VM in MB.",
 135         minimum => 0,
 136         default => 512,
 137     },
 138     hostname => {
 139         optional => 1,
 140         description => "Set a host name for the container.",
 141         type => 'string',
 142         maxLength => 255,
 143     },
 144     description => {
 145         optional => 1,
 146         type => 'string',
 147         description => "Container description. Only used on the configuration web interface.",
 148     },
 149     searchdomain => {
 150         optional => 1,
 151         type => 'string',
 152         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 153     },
 154     nameserver => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 158     },
 159     rootfs => get_standard_option('pve-ct-rootfs'),
 160     parent => {
 161         optional => 1,
 162         type => 'string', format => 'pve-configid',
 163         maxLength => 40,
 164         description => "Parent snapshot name. This is used internally, and should not be modified.",
 165     },
 166     snaptime => {
 167         optional => 1,
 168         description => "Timestamp for snapshots.",
 169         type => 'integer',
 170         minimum => 0,
 171     },
 172     cmode => {
 173         optional => 1,
 174         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 175         type => 'string',
 176         enum => ['shell', 'console', 'tty'],
 177         default => 'tty',
 178     },
 179 };
 180
 181 my $valid_lxc_conf_keys = {
 182     'lxc.include' => 1,
 183     'lxc.arch' => 1,
 184     'lxc.utsname' => 1,
 185     'lxc.haltsignal' => 1,
 186     'lxc.rebootsignal' => 1,
 187     'lxc.stopsignal' => 1,
 188     'lxc.init_cmd' => 1,
 189     'lxc.network.type' => 1,
 190     'lxc.network.flags' => 1,
 191     'lxc.network.link' => 1,
 192     'lxc.network.mtu' => 1,
 193     'lxc.network.name' => 1,
 194     'lxc.network.hwaddr' => 1,
 195     'lxc.network.ipv4' => 1,
 196     'lxc.network.ipv4.gateway' => 1,
 197     'lxc.network.ipv6' => 1,
 198     'lxc.network.ipv6.gateway' => 1,
 199     'lxc.network.script.up' => 1,
 200     'lxc.network.script.down' => 1,
 201     'lxc.pts' => 1,
 202     'lxc.console.logfile' => 1,
 203     'lxc.console' => 1,
 204     'lxc.tty' => 1,
 205     'lxc.devttydir' => 1,
 206     'lxc.hook.autodev' => 1,
 207     'lxc.autodev' => 1,
 208     'lxc.kmsg' => 1,
 209     'lxc.mount' => 1,
 210     'lxc.mount.entry' => 1,
 211     'lxc.mount.auto' => 1,
 212     'lxc.rootfs' => 1,
 213     'lxc.rootfs.mount' => 1,
 214     'lxc.rootfs.options' => 1,
 215     # lxc.cgroup.*
 216     'lxc.cap.drop' => 1,
 217     'lxc.cap.keep' => 1,
 218     'lxc.aa_profile' => 1,
 219     'lxc.aa_allow_incomplete' => 1,
 220     'lxc.se_context' => 1,
 221     'lxc.seccomp' => 1,
 222     'lxc.id_map' => 1,
 223     'lxc.hook.pre-start' => 1,
 224     'lxc.hook.pre-mount' => 1,
 225     'lxc.hook.mount' => 1,
 226     'lxc.hook.start' => 1,
 227     'lxc.hook.post-stop' => 1,
 228     'lxc.hook.clone' => 1,
 229     'lxc.hook.destroy' => 1,
 230     'lxc.loglevel' => 1,
 231     'lxc.logfile' => 1,
 232     'lxc.start.auto' => 1,
 233     'lxc.start.delay' => 1,
 234     'lxc.start.order' => 1,
 235     'lxc.group' => 1,
 236     'lxc.environment' => 1,
 237     'lxc.' => 1,
 238     'lxc.' => 1,
 239     'lxc.' => 1,
 240     'lxc.' => 1,
 241 };
 242
 243 my $MAX_LXC_NETWORKS = 10;
 244 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 245     $confdesc->{"net$i"} = {
 246         optional => 1,
 247         type => 'string', format => 'pve-lxc-network',
 248         description => "Specifies network interfaces for the container.\n\n".
 249             "The string should have the follow format:\n\n".
 250             "-net<[0-9]> bridge=<vmbr<Nummber>>[,hwaddr=<MAC>]\n".
 251             "[,mtu=<Number>][,name=<String>][,ip=<IPv4Format/CIDR>]\n".
 252             ",ip6=<IPv6Format/CIDR>][,gw=<GatwayIPv4>]\n".
 253             ",gw6=<GatwayIPv6>][,firewall=<[1|0]>][,tag=<VlanNo>]",
 254     };
 255 }
 256
 257 my $MAX_MOUNT_POINTS = 10;
 258 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 259     $confdesc->{"mp$i"} = {
 260         optional => 1,
 261         type => 'string', format => 'pve-ct-mountpoint',
 262         typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+] [,mp=mountpoint]',
 263         description => "Use volume as container mount point (experimental feature).",
 264         optional => 1,
 265     };
 266 }
 267
 268 sub write_pct_config {
 269     my ($filename, $conf) = @_;
 270
 271     delete $conf->{snapstate}; # just to be sure
 272
 273     my $generate_raw_config = sub {
 274         my ($conf) = @_;
 275
 276         my $raw = '';
 277
 278         # add description as comment to top of file
 279         my $descr = $conf->{description} || '';
 280         foreach my $cl (split(/\n/, $descr)) {
 281             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 282         }
 283
 284         foreach my $key (sort keys %$conf) {
 285             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 286                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 287             $raw .= "$key: $conf->{$key}\n";
 288         }
 289
 290         if (my $lxcconf = $conf->{lxc}) {
 291             foreach my $entry (@$lxcconf) {
 292                 my ($k, $v) = @$entry;
 293                 $raw .= "$k: $v\n";
 294             }
 295         }
 296
 297         return $raw;
 298     };
 299
 300     my $raw = &$generate_raw_config($conf);
 301
 302     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 303         $raw .= "\n[$snapname]\n";
 304         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 305     }
 306
 307     return $raw;
 308 }
 309
 310 sub check_type {
 311     my ($key, $value) = @_;
 312
 313     die "unknown setting '$key'\n" if !$confdesc->{$key};
 314
 315     my $type = $confdesc->{$key}->{type};
 316
 317     if (!defined($value)) {
 318         die "got undefined value\n";
 319     }
 320
 321     if ($value =~ m/[\n\r]/) {
 322         die "property contains a line feed\n";
 323     }
 324
 325     if ($type eq 'boolean') {
 326         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 327         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 328         die "type check ('boolean') failed - got '$value'\n";
 329     } elsif ($type eq 'integer') {
 330         return int($1) if $value =~ m/^(\d+)$/;
 331         die "type check ('integer') failed - got '$value'\n";
 332     } elsif ($type eq 'number') {
 333         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 334         die "type check ('number') failed - got '$value'\n";
 335     } elsif ($type eq 'string') {
 336         if (my $fmt = $confdesc->{$key}->{format}) {
 337             PVE::JSONSchema::check_format($fmt, $value);
 338             return $value;
 339         }
 340         return $value;
 341     } else {
 342         die "internal error"
 343     }
 344 }
 345
 346 sub parse_pct_config {
 347     my ($filename, $raw) = @_;
 348
 349     return undef if !defined($raw);
 350
 351     my $res = {
 352         digest => Digest::SHA::sha1_hex($raw),
 353         snapshots => {},
 354     };
 355
 356     $filename =~ m|/lxc/(\d+).conf$|
 357         || die "got strange filename '$filename'";
 358
 359     my $vmid = $1;
 360
 361     my $conf = $res;
 362     my $descr = '';
 363     my $section = '';
 364
 365     my @lines = split(/\n/, $raw);
 366     foreach my $line (@lines) {
 367         next if $line =~ m/^\s*$/;
 368
 369         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 370             $section = $1;
 371             $conf->{description} = $descr if $descr;
 372             $descr = '';
 373             $conf = $res->{snapshots}->{$section} = {};
 374             next;
 375         }
 376
 377         if ($line =~ m/^\#(.*)\s*$/) {
 378             $descr .= PVE::Tools::decode_text($1) . "\n";
 379             next;
 380         }
 381
 382         if ($line =~ m/^(lxc\.[a-z0-9_\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 383             my $key = $1;
 384             my $value = $3;
 385             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 386                 push @{$conf->{lxc}}, [$key, $value];
 387             } else {
 388                 warn "vm $vmid - unable to parse config: $line\n";
 389             }
 390         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 391             $descr .= PVE::Tools::decode_text($2);
 392         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 393             $conf->{snapstate} = $1;
 394         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
 395             my $key = $1;
 396             my $value = $2;
 397             eval { $value = check_type($key, $value); };
 398             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 399             $conf->{$key} = $value;
 400         } else {
 401             warn "vm $vmid - unable to parse config: $line\n";
 402         }
 403     }
 404
 405     $conf->{description} = $descr if $descr;
 406
 407     delete $res->{snapstate}; # just to be sure
 408
 409     return $res;
 410 }
 411
 412 sub config_list {
 413     my $vmlist = PVE::Cluster::get_vmlist();
 414     my $res = {};
 415     return $res if !$vmlist || !$vmlist->{ids};
 416     my $ids = $vmlist->{ids};
 417
 418     foreach my $vmid (keys %$ids) {
 419         next if !$vmid; # skip CT0
 420         my $d = $ids->{$vmid};
 421         next if !$d->{node} || $d->{node} ne $nodename;
 422         next if !$d->{type} || $d->{type} ne 'lxc';
 423         $res->{$vmid}->{type} = 'lxc';
 424     }
 425     return $res;
 426 }
 427
 428 sub cfs_config_path {
 429     my ($vmid, $node) = @_;
 430
 431     $node = $nodename if !$node;
 432     return "nodes/$node/lxc/$vmid.conf";
 433 }
 434
 435 sub config_file {
 436     my ($vmid, $node) = @_;
 437
 438     my $cfspath = cfs_config_path($vmid, $node);
 439     return "/etc/pve/$cfspath";
 440 }
 441
 442 sub load_config {
 443     my ($vmid, $node) = @_;
 444
 445     $node = $nodename if !$node;
 446     my $cfspath = cfs_config_path($vmid, $node);
 447
 448     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 449     die "container $vmid does not exists\n" if !defined($conf);
 450
 451     return $conf;
 452 }
 453
 454 sub create_config {
 455     my ($vmid, $conf) = @_;
 456
 457     my $dir = "/etc/pve/nodes/$nodename/lxc";
 458     mkdir $dir;
 459
 460     write_config($vmid, $conf);
 461 }
 462
 463 sub destroy_config {
 464     my ($vmid) = @_;
 465
 466     unlink config_file($vmid, $nodename);
 467 }
 468
 469 sub write_config {
 470     my ($vmid, $conf) = @_;
 471
 472     my $cfspath = cfs_config_path($vmid);
 473
 474     PVE::Cluster::cfs_write_file($cfspath, $conf);
 475 }
 476
 477 # flock: we use one file handle per process, so lock file
 478 # can be called multiple times and succeeds for the same process.
 479
 480 my $lock_handles =  {};
 481 my $lockdir = "/run/lock/lxc";
 482
 483 sub lock_filename {
 484     my ($vmid) = @_;
 485
 486     return "$lockdir/pve-config-${vmid}.lock";
 487 }
 488
 489 sub lock_aquire {
 490     my ($vmid, $timeout) = @_;
 491
 492     $timeout = 10 if !$timeout;
 493     my $mode = LOCK_EX;
 494
 495     my $filename = lock_filename($vmid);
 496
 497     mkdir $lockdir if !-d $lockdir;
 498
 499     my $lock_func = sub {
 500         if (!$lock_handles->{$$}->{$filename}) {
 501             my $fh = new IO::File(">>$filename") ||
 502                 die "can't open file - $!\n";
 503             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 504         }
 505
 506         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 507             print STDERR "trying to aquire lock...";
 508             my $success;
 509             while(1) {
 510                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 511                 # try again on EINTR (see bug #273)
 512                 if ($success || ($! != EINTR)) {
 513                     last;
 514                 }
 515             }
 516             if (!$success) {
 517                 print STDERR " failed\n";
 518                 die "can't aquire lock - $!\n";
 519             }
 520
 521             print STDERR " OK\n";
 522         }
 523
 524         $lock_handles->{$$}->{$filename}->{refcount}++;
 525     };
 526
 527     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 528     my $err = $@;
 529     if ($err) {
 530         die "can't lock file '$filename' - $err";
 531     }
 532 }
 533
 534 sub lock_release {
 535     my ($vmid) = @_;
 536
 537     my $filename = lock_filename($vmid);
 538
 539     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 540         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 541         if ($refcount <= 0) {
 542             $lock_handles->{$$}->{$filename} = undef;
 543             close ($fh);
 544         }
 545     }
 546 }
 547
 548 sub lock_container {
 549     my ($vmid, $timeout, $code, @param) = @_;
 550
 551     my $res;
 552
 553     lock_aquire($vmid, $timeout);
 554     eval { $res = &$code(@param) };
 555     my $err = $@;
 556     lock_release($vmid);
 557
 558     die $err if $err;
 559
 560     return $res;
 561 }
 562
 563 sub option_exists {
 564     my ($name) = @_;
 565
 566     return defined($confdesc->{$name});
 567 }
 568
 569 # add JSON properties for create and set function
 570 sub json_config_properties {
 571     my $prop = shift;
 572
 573     foreach my $opt (keys %$confdesc) {
 574         next if $opt eq 'parent' || $opt eq 'snaptime';
 575         next if $prop->{$opt};
 576         $prop->{$opt} = $confdesc->{$opt};
 577     }
 578
 579     return $prop;
 580 }
 581
 582 sub json_config_properties_no_rootfs {
 583     my $prop = shift;
 584
 585     foreach my $opt (keys %$confdesc) {
 586         next if $prop->{$opt};
 587         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 588         $prop->{$opt} = $confdesc->{$opt};
 589     }
 590
 591     return $prop;
 592 }
 593
 594 # container status helpers
 595
 596 sub list_active_containers {
 597
 598     my $filename = "/proc/net/unix";
 599
 600     # similar test is used by lcxcontainers.c: list_active_containers
 601     my $res = {};
 602
 603     my $fh = IO::File->new ($filename, "r");
 604     return $res if !$fh;
 605
 606     while (defined(my $line = <$fh>)) {
 607         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 608             my $path = $1;
 609             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 610                 $res->{$1} = 1;
 611             }
 612         }
 613     }
 614
 615     close($fh);
 616
 617     return $res;
 618 }
 619
 620 # warning: this is slow
 621 sub check_running {
 622     my ($vmid) = @_;
 623
 624     my $active_hash = list_active_containers();
 625
 626     return 1 if defined($active_hash->{$vmid});
 627
 628     return undef;
 629 }
 630
 631 sub get_container_disk_usage {
 632     my ($vmid) = @_;
 633
 634     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 635
 636     my $res = {
 637         total => 0,
 638         used => 0,
 639         avail => 0,
 640     };
 641
 642     my $parser = sub {
 643         my $line = shift;
 644         if (my ($fsid, $total, $used, $avail) = $line =~
 645             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 646             $res = {
 647                 total => $total,
 648                 used => $used,
 649                 avail => $avail,
 650             };
 651         }
 652     };
 653     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 654     warn $@ if $@;
 655
 656     return $res;
 657 }
 658
 659 sub vmstatus {
 660     my ($opt_vmid) = @_;
 661
 662     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 663
 664     my $active_hash = list_active_containers();
 665
 666     foreach my $vmid (keys %$list) {
 667         my $d = $list->{$vmid};
 668
 669         my $running = defined($active_hash->{$vmid});
 670
 671         $d->{status} = $running ? 'running' : 'stopped';
 672
 673         my $cfspath = cfs_config_path($vmid);
 674         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 675
 676         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 677         $d->{name} =~ s/[\s]//g;
 678
 679         $d->{cpus} = $conf->{cpulimit} // 0;
 680
 681         if ($running) {
 682             my $res = get_container_disk_usage($vmid);
 683             $d->{disk} = $res->{used};
 684             $d->{maxdisk} = $res->{total};
 685         } else {
 686             $d->{disk} = 0;
 687             # use 4GB by default ??
 688             if (my $rootfs = $conf->{rootfs}) {
 689                 my $rootinfo = parse_ct_mountpoint($rootfs);
 690                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 691             } else {
 692                 $d->{maxdisk} = 4*1024*1024*1024;
 693             }
 694         }
 695
 696         $d->{mem} = 0;
 697         $d->{swap} = 0;
 698         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 699         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 700
 701         $d->{uptime} = 0;
 702         $d->{cpu} = 0;
 703
 704         $d->{netout} = 0;
 705         $d->{netin} = 0;
 706
 707         $d->{diskread} = 0;
 708         $d->{diskwrite} = 0;
 709
 710         $d->{template} = is_template($conf);
 711     }
 712
 713     foreach my $vmid (keys %$list) {
 714         my $d = $list->{$vmid};
 715         next if $d->{status} ne 'running';
 716
 717         $d->{uptime} = 100; # fixme:
 718
 719         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 720         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 721
 722         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 723         my @bytes = split(/\n/, $blkio_bytes);
 724         foreach my $byte (@bytes) {
 725             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 726                 $d->{diskread} = $2 if $key eq 'Read';
 727                 $d->{diskwrite} = $2 if $key eq 'Write';
 728             }
 729         }
 730     }
 731
 732     return $list;
 733 }
 734
 735 my $parse_size = sub {
 736     my ($value) = @_;
 737
 738     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 739     my ($size, $unit) = ($1, $3);
 740     if ($unit) {
 741         if ($unit eq 'K') {
 742             $size = $size * 1024;
 743         } elsif ($unit eq 'M') {
 744             $size = $size * 1024 * 1024;
 745         } elsif ($unit eq 'G') {
 746             $size = $size * 1024 * 1024 * 1024;
 747         }
 748     }
 749     return int($size);
 750 };
 751
 752 sub parse_ct_mountpoint {
 753     my ($data) = @_;
 754
 755     $data //= '';
 756
 757     my $res = {};
 758
 759     foreach my $p (split (/,/, $data)) {
 760         next if $p =~ m/^\s*$/;
 761
 762         if ($p =~ m/^(volume|backup|size|mp)=(.+)$/) {
 763             my ($k, $v) = ($1, $2);
 764             return undef if defined($res->{$k});
 765             $res->{$k} = $v;
 766         } else {
 767             if (!$res->{volume} && $p !~ m/=/) {
 768                 $res->{volume} = $p;
 769             } else {
 770                 return undef;
 771             }
 772         }
 773     }
 774
 775     return undef if !defined($res->{volume});
 776
 777     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 778
 779     if ($res->{size}) {
 780         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 781     }
 782
 783     return $res;
 784 }
 785
 786 sub print_ct_mountpoint {
 787     my ($info, $nomp) = @_;
 788
 789     my $opts = '';
 790
 791     die "missing volume\n" if !$info->{volume};
 792
 793     foreach my $o ('size', 'backup') {
 794         $opts .= ",$o=$info->{$o}" if defined($info->{$o});
 795     }
 796     $opts .= ",mp=$info->{mp}" if !$nomp;
 797
 798     return "$info->{volume}$opts";
 799 }
 800
 801 sub print_lxc_network {
 802     my $net = shift;
 803
 804     die "no network name defined\n" if !$net->{name};
 805
 806     my $res = "name=$net->{name}";
 807
 808     foreach my $k (qw(hwaddr mtu bridge ip gw ip6 gw6 firewall tag)) {
 809         next if !defined($net->{$k});
 810         $res .= ",$k=$net->{$k}";
 811     }
 812
 813     return $res;
 814 }
 815
 816 sub parse_lxc_network {
 817     my ($data) = @_;
 818
 819     my $res = {};
 820
 821     return $res if !$data;
 822
 823     foreach my $pv (split (/,/, $data)) {
 824         if ($pv =~ m/^(bridge|hwaddr|mtu|name|ip|ip6|gw|gw6|firewall|tag)=(\S+)$/) {
 825             $res->{$1} = $2;
 826         } else {
 827             return undef;
 828         }
 829     }
 830
 831     $res->{type} = 'veth';
 832     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 833
 834     return $res;
 835 }
 836
 837 sub read_cgroup_value {
 838     my ($group, $vmid, $name, $full) = @_;
 839
 840     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 841
 842     return PVE::Tools::file_get_contents($path) if $full;
 843
 844     return PVE::Tools::file_read_firstline($path);
 845 }
 846
 847 sub write_cgroup_value {
 848    my ($group, $vmid, $name, $value) = @_;
 849
 850    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 851    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 852
 853 }
 854
 855 sub find_lxc_console_pids {
 856
 857     my $res = {};
 858
 859     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 860         my ($pid) = @_;
 861
 862         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 863         return if !$cmdline;
 864
 865         my @args = split(/\0/, $cmdline);
 866
 867         # serach for lxc-console -n <vmid>
 868         return if scalar(@args) != 3;
 869         return if $args[1] ne '-n';
 870         return if $args[2] !~ m/^\d+$/;
 871         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 872
 873         my $vmid = $args[2];
 874
 875         push @{$res->{$vmid}}, $pid;
 876     });
 877
 878     return $res;
 879 }
 880
 881 sub find_lxc_pid {
 882     my ($vmid) = @_;
 883
 884     my $pid = undef;
 885     my $parser = sub {
 886         my $line = shift;
 887         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 888     };
 889     PVE::Tools::run_command(['lxc-info', '-n', $vmid], outfunc => $parser);
 890
 891     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 892
 893     return $pid;
 894 }
 895
 896 my $ipv4_reverse_mask = [
 897     '0.0.0.0',
 898     '128.0.0.0',
 899     '192.0.0.0',
 900     '224.0.0.0',
 901     '240.0.0.0',
 902     '248.0.0.0',
 903     '252.0.0.0',
 904     '254.0.0.0',
 905     '255.0.0.0',
 906     '255.128.0.0',
 907     '255.192.0.0',
 908     '255.224.0.0',
 909     '255.240.0.0',
 910     '255.248.0.0',
 911     '255.252.0.0',
 912     '255.254.0.0',
 913     '255.255.0.0',
 914     '255.255.128.0',
 915     '255.255.192.0',
 916     '255.255.224.0',
 917     '255.255.240.0',
 918     '255.255.248.0',
 919     '255.255.252.0',
 920     '255.255.254.0',
 921     '255.255.255.0',
 922     '255.255.255.128',
 923     '255.255.255.192',
 924     '255.255.255.224',
 925     '255.255.255.240',
 926     '255.255.255.248',
 927     '255.255.255.252',
 928     '255.255.255.254',
 929     '255.255.255.255',
 930 ];
 931
 932 # Note: we cannot use Net:IP, because that only allows strict
 933 # CIDR networks
 934 sub parse_ipv4_cidr {
 935     my ($cidr, $noerr) = @_;
 936
 937     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
 938         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
 939     }
 940
 941     return undef if $noerr;
 942
 943     die "unable to parse ipv4 address/mask\n";
 944 }
 945
 946 sub check_lock {
 947     my ($conf) = @_;
 948
 949     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
 950 }
 951
 952 sub update_lxc_config {
 953     my ($storage_cfg, $vmid, $conf) = @_;
 954
 955     my $dir = "/var/lib/lxc/$vmid";
 956
 957     if ($conf->{template}) {
 958
 959         unlink "$dir/config";
 960
 961         return;
 962     }
 963
 964     my $raw = '';
 965
 966     die "missing 'arch' - internal error" if !$conf->{arch};
 967     $raw .= "lxc.arch = $conf->{arch}\n";
 968
 969     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
 970     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
 971         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
 972     } else {
 973         die "implement me";
 974     }
 975
 976     if (!has_dev_console($conf)) {
 977         $raw .= "lxc.console = none\n";
 978         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
 979     }
 980
 981     my $ttycount = get_tty_count($conf);
 982     $raw .= "lxc.tty = $ttycount\n";
 983
 984     my $utsname = $conf->{hostname} || "CT$vmid";
 985     $raw .= "lxc.utsname = $utsname\n";
 986
 987     my $memory = $conf->{memory} || 512;
 988     my $swap = $conf->{swap} // 0;
 989
 990     my $lxcmem = int($memory*1024*1024);
 991     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
 992
 993     my $lxcswap = int(($memory + $swap)*1024*1024);
 994     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
 995
 996     if (my $cpulimit = $conf->{cpulimit}) {
 997         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
 998         my $value = int(100000*$cpulimit);
 999         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1000     }
1001
1002     my $shares = $conf->{cpuunits} || 1024;
1003     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1004
1005     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1006     $mountpoint->{mp} = '/';
1007     my $volid = $mountpoint->{volume};
1008     my $path = mountpoint_mount_path($mountpoint, $storage_cfg);
1009
1010     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1011
1012     if ($storage) {
1013         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1014         $path = "loop:$path" if $scfg->{path};
1015     }
1016
1017     $raw .= "lxc.rootfs = $path\n";
1018
1019     my $netcount = 0;
1020     foreach my $k (keys %$conf) {
1021         next if $k !~ m/^net(\d+)$/;
1022         my $ind = $1;
1023         my $d = parse_lxc_network($conf->{$k});
1024         $netcount++;
1025         $raw .= "lxc.network.type = veth\n";
1026         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1027         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1028         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1029         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1030     }
1031
1032     if (my $lxcconf = $conf->{lxc}) {
1033         foreach my $entry (@$lxcconf) {
1034             my ($k, $v) = @$entry;
1035             $netcount++ if $k eq 'lxc.network.type';
1036             $raw .= "$k = $v\n";
1037         }
1038     }
1039
1040     $raw .= "lxc.network.type = empty\n" if !$netcount;
1041
1042     File::Path::mkpath("$dir/rootfs");
1043
1044     PVE::Tools::file_set_contents("$dir/config", $raw);
1045 }
1046
1047 # verify and cleanup nameserver list (replace \0 with ' ')
1048 sub verify_nameserver_list {
1049     my ($nameserver_list) = @_;
1050
1051     my @list = ();
1052     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1053         PVE::JSONSchema::pve_verify_ip($server);
1054         push @list, $server;
1055     }
1056
1057     return join(' ', @list);
1058 }
1059
1060 sub verify_searchdomain_list {
1061     my ($searchdomain_list) = @_;
1062
1063     my @list = ();
1064     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1065         # todo: should we add checks for valid dns domains?
1066         push @list, $server;
1067     }
1068
1069     return join(' ', @list);
1070 }
1071
1072 sub update_pct_config {
1073     my ($vmid, $conf, $running, $param, $delete) = @_;
1074
1075     my @nohotplug;
1076
1077     my $new_disks = [];
1078
1079     my $rootdir;
1080     if ($running) {
1081         my $pid = find_lxc_pid($vmid);
1082         $rootdir = "/proc/$pid/root";
1083     }
1084
1085     if (defined($delete)) {
1086         foreach my $opt (@$delete) {
1087             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1088                 die "unable to delete required option '$opt'\n";
1089             } elsif ($opt eq 'swap') {
1090                 delete $conf->{$opt};
1091                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1092             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1093                 delete $conf->{$opt};
1094             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1095                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1096                 delete $conf->{$opt};
1097                 push @nohotplug, $opt;
1098                 next if $running;
1099             } elsif ($opt =~ m/^net(\d)$/) {
1100                 delete $conf->{$opt};
1101                 next if !$running;
1102                 my $netid = $1;
1103                 PVE::Network::veth_delete("veth${vmid}i$netid");
1104             } elsif ($opt =~ m/^mp(\d+)$/) {
1105                 delete $conf->{$opt};
1106                 push @nohotplug, $opt;
1107                 next if $running;
1108             } elsif ($opt eq 'rootfs') {
1109                 die "implement me"
1110             } else {
1111                 die "implement me"
1112             }
1113             write_config($vmid, $conf) if $running;
1114         }
1115     }
1116
1117     # There's no separate swap size to configure, there's memory and "total"
1118     # memory (iow. memory+swap). This means we have to change them together.
1119     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1120     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1121     if (defined($wanted_memory) || defined($wanted_swap)) {
1122
1123         $wanted_memory //= ($conf->{memory} || 512);
1124         $wanted_swap //=  ($conf->{swap} || 0);
1125
1126         my $total = $wanted_memory + $wanted_swap;
1127         if ($running) {
1128             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1129             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1130         }
1131         $conf->{memory} = $wanted_memory;
1132         $conf->{swap} = $wanted_swap;
1133
1134         write_config($vmid, $conf) if $running;
1135     }
1136
1137     foreach my $opt (keys %$param) {
1138         my $value = $param->{$opt};
1139         if ($opt eq 'hostname') {
1140             $conf->{$opt} = $value;
1141         } elsif ($opt eq 'onboot') {
1142             $conf->{$opt} = $value ? 1 : 0;
1143         } elsif ($opt eq 'startup') {
1144             $conf->{$opt} = $value;
1145         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1146             $conf->{$opt} = $value;
1147             push @nohotplug, $opt;
1148             next if $running;
1149         } elsif ($opt eq 'nameserver') {
1150             my $list = verify_nameserver_list($value);
1151             $conf->{$opt} = $list;
1152             push @nohotplug, $opt;
1153             next if $running;
1154         } elsif ($opt eq 'searchdomain') {
1155             my $list = verify_searchdomain_list($value);
1156             $conf->{$opt} = $list;
1157             push @nohotplug, $opt;
1158             next if $running;
1159         } elsif ($opt eq 'cpulimit') {
1160             $conf->{$opt} = $value;
1161             push @nohotplug, $opt; # fixme: hotplug
1162             next;
1163         } elsif ($opt eq 'cpuunits') {
1164             $conf->{$opt} = $value;
1165             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1166         } elsif ($opt eq 'description') {
1167             $conf->{$opt} = PVE::Tools::encode_text($value);
1168         } elsif ($opt =~ m/^net(\d+)$/) {
1169             my $netid = $1;
1170             my $net = parse_lxc_network($value);
1171             if (!$running) {
1172                 $conf->{$opt} = print_lxc_network($net);
1173             } else {
1174                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1175             }
1176         } elsif ($opt =~ m/^mp(\d+)$/) {
1177             $conf->{$opt} = $value;
1178             push @$new_disks, $opt;
1179             push @nohotplug, $opt;
1180             next;
1181         } elsif ($opt eq 'rootfs') {
1182             die "implement me: $opt";
1183         } else {
1184             die "implement me: $opt";
1185         }
1186         write_config($vmid, $conf) if $running;
1187     }
1188
1189     if ($running && scalar(@nohotplug)) {
1190         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1191     }
1192
1193     if (@$new_disks) {
1194         my $storage_cfg = PVE::Storage::config();
1195         create_disks($storage_cfg, $vmid, $conf, $conf);
1196         mount_all($vmid, $storage_cfg, $conf, $new_disks, 1);
1197         umount_all($vmid, $storage_cfg, $conf, 0);
1198     }
1199 }
1200
1201 sub has_dev_console {
1202     my ($conf) = @_;
1203
1204     return !(defined($conf->{console}) && !$conf->{console});
1205 }
1206
1207 sub get_tty_count {
1208     my ($conf) = @_;
1209
1210     return $conf->{tty} // $confdesc->{tty}->{default};
1211 }
1212
1213 sub get_cmode {
1214     my ($conf) = @_;
1215
1216     return $conf->{cmode} // $confdesc->{cmode}->{default};
1217 }
1218
1219 sub get_console_command {
1220     my ($vmid, $conf) = @_;
1221
1222     my $cmode = get_cmode($conf);
1223
1224     if ($cmode eq 'console') {
1225         return ['lxc-console', '-n',  $vmid, '-t', 0];
1226     } elsif ($cmode eq 'tty') {
1227         return ['lxc-console', '-n',  $vmid];
1228     } elsif ($cmode eq 'shell') {
1229         return ['lxc-attach', '--clear-env', '-n', $vmid];
1230     } else {
1231         die "internal error";
1232     }
1233 }
1234
1235 sub get_primary_ips {
1236     my ($conf) = @_;
1237
1238     # return data from net0
1239
1240     return undef if !defined($conf->{net0});
1241     my $net = parse_lxc_network($conf->{net0});
1242
1243     my $ipv4 = $net->{ip};
1244     if ($ipv4) {
1245         if ($ipv4 =~ /^(dhcp|manual)$/) {
1246             $ipv4 = undef
1247         } else {
1248             $ipv4 =~ s!/\d+$!!;
1249         }
1250     }
1251     my $ipv6 = $net->{ip6};
1252     if ($ipv6) {
1253         if ($ipv6 =~ /^(dhcp|manual)$/) {
1254             $ipv6 = undef;
1255         } else {
1256             $ipv6 =~ s!/\d+$!!;
1257         }
1258     }
1259
1260     return ($ipv4, $ipv6);
1261 }
1262
1263
1264 sub destroy_lxc_container {
1265     my ($storage_cfg, $vmid, $conf) = @_;
1266
1267     foreach_mountpoint($conf, sub {
1268         my ($ms, $mountpoint) = @_;
1269         my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $mountpoint->{volume});
1270         PVE::Storage::vdisk_free($storage_cfg, $mountpoint->{volume}) if $vmid == $owner;
1271     });
1272
1273     rmdir "/var/lib/lxc/$vmid/rootfs";
1274     unlink "/var/lib/lxc/$vmid/config";
1275     rmdir "/var/lib/lxc/$vmid";
1276     destroy_config($vmid);
1277
1278     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1279     #PVE::Tools::run_command($cmd);
1280 }
1281
1282 sub vm_stop_cleanup {
1283     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1284
1285     eval {
1286         if (!$keepActive) {
1287
1288             my $vollist = get_vm_volumes($conf);
1289             my $loopdevlist = get_vm_volumes($conf, 'rootfs');
1290
1291             detach_loops($storage_cfg, $loopdevlist);
1292             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1293         }
1294     };
1295     warn $@ if $@; # avoid errors - just warn
1296 }
1297
1298 my $safe_num_ne = sub {
1299     my ($a, $b) = @_;
1300
1301     return 0 if !defined($a) && !defined($b);
1302     return 1 if !defined($a);
1303     return 1 if !defined($b);
1304
1305     return $a != $b;
1306 };
1307
1308 my $safe_string_ne = sub {
1309     my ($a, $b) = @_;
1310
1311     return 0 if !defined($a) && !defined($b);
1312     return 1 if !defined($a);
1313     return 1 if !defined($b);
1314
1315     return $a ne $b;
1316 };
1317
1318 sub update_net {
1319     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1320
1321     if ($newnet->{type} ne 'veth') {
1322         # for when there are physical interfaces
1323         die "cannot update interface of type $newnet->{type}";
1324     }
1325
1326     my $veth = "veth${vmid}i${netid}";
1327     my $eth = $newnet->{name};
1328
1329     if (my $oldnetcfg = $conf->{$opt}) {
1330         my $oldnet = parse_lxc_network($oldnetcfg);
1331
1332         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1333             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1334
1335             PVE::Network::veth_delete($veth);
1336             delete $conf->{$opt};
1337             write_config($vmid, $conf);
1338
1339             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1340
1341         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1342                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1343                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1344
1345                 if ($oldnet->{bridge}) {
1346                     PVE::Network::tap_unplug($veth);
1347                     foreach (qw(bridge tag firewall)) {
1348                         delete $oldnet->{$_};
1349                     }
1350                     $conf->{$opt} = print_lxc_network($oldnet);
1351                     write_config($vmid, $conf);
1352                 }
1353
1354                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1355                 foreach (qw(bridge tag firewall)) {
1356                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1357                 }
1358                 $conf->{$opt} = print_lxc_network($oldnet);
1359                 write_config($vmid, $conf);
1360         }
1361     } else {
1362         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1363     }
1364
1365     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1366 }
1367
1368 sub hotplug_net {
1369     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1370
1371     my $veth = "veth${vmid}i${netid}";
1372     my $vethpeer = $veth . "p";
1373     my $eth = $newnet->{name};
1374
1375     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1376     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1377
1378     # attach peer in container
1379     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1380     PVE::Tools::run_command($cmd);
1381
1382     # link up peer in container
1383     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1384     PVE::Tools::run_command($cmd);
1385
1386     my $done = { type => 'veth' };
1387     foreach (qw(bridge tag firewall hwaddr name)) {
1388         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1389     }
1390     $conf->{$opt} = print_lxc_network($done);
1391
1392     write_config($vmid, $conf);
1393 }
1394
1395 sub update_ipconfig {
1396     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1397
1398     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1399
1400     my $optdata = parse_lxc_network($conf->{$opt});
1401     my $deleted = [];
1402     my $added = [];
1403     my $nscmd = sub {
1404         my $cmdargs = shift;
1405         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1406     };
1407     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1408
1409     my $change_ip_config = sub {
1410         my ($ipversion) = @_;
1411
1412         my $family_opt = "-$ipversion";
1413         my $suffix = $ipversion == 4 ? '' : $ipversion;
1414         my $gw= "gw$suffix";
1415         my $ip= "ip$suffix";
1416
1417         my $newip = $newnet->{$ip};
1418         my $newgw = $newnet->{$gw};
1419         my $oldip = $optdata->{$ip};
1420
1421         my $change_ip = &$safe_string_ne($oldip, $newip);
1422         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1423
1424         return if !$change_ip && !$change_gw;
1425
1426         # step 1: add new IP, if this fails we cancel
1427         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1428             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1429             if (my $err = $@) {
1430                 warn $err;
1431                 return;
1432             }
1433         }
1434
1435         # step 2: replace gateway
1436         #   If this fails we delete the added IP and cancel.
1437         #   If it succeeds we save the config and delete the old IP, ignoring
1438         #   errors. The config is then saved.
1439         # Note: 'ip route replace' can add
1440         if ($change_gw) {
1441             if ($newgw) {
1442                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1443                 if (my $err = $@) {
1444                     warn $err;
1445                     # the route was not replaced, the old IP is still available
1446                     # rollback (delete new IP) and cancel
1447                     if ($change_ip) {
1448                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1449                         warn $@ if $@; # no need to die here
1450                     }
1451                     return;
1452                 }
1453             } else {
1454                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1455                 # if the route was not deleted, the guest might have deleted it manually
1456                 # warn and continue
1457                 warn $@ if $@;
1458             }
1459         }
1460
1461         # from this point on we save the configuration
1462         # step 3: delete old IP ignoring errors
1463         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1464             # We need to enable promote_secondaries, otherwise our newly added
1465             # address will be removed along with the old one.
1466             my $promote = 0;
1467             eval {
1468                 if ($ipversion == 4) {
1469                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1470                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1471                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1472                 }
1473                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1474             };
1475             warn $@ if $@; # no need to die here
1476
1477             if ($ipversion == 4) {
1478                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1479             }
1480         }
1481
1482         foreach my $property ($ip, $gw) {
1483             if ($newnet->{$property}) {
1484                 $optdata->{$property} = $newnet->{$property};
1485             } else {
1486                 delete $optdata->{$property};
1487             }
1488         }
1489         $conf->{$opt} = print_lxc_network($optdata);
1490         write_config($vmid, $conf);
1491         $lxc_setup->setup_network($conf);
1492     };
1493
1494     &$change_ip_config(4);
1495     &$change_ip_config(6);
1496
1497 }
1498
1499 # Internal snapshots
1500
1501 # NOTE: Snapshot create/delete involves several non-atomic
1502 # action, and can take a long time.
1503 # So we try to avoid locking the file and use 'lock' variable
1504 # inside the config file instead.
1505
1506 my $snapshot_copy_config = sub {
1507     my ($source, $dest) = @_;
1508
1509     foreach my $k (keys %$source) {
1510         next if $k eq 'snapshots';
1511         next if $k eq 'snapstate';
1512         next if $k eq 'snaptime';
1513         next if $k eq 'vmstate';
1514         next if $k eq 'lock';
1515         next if $k eq 'digest';
1516         next if $k eq 'description';
1517
1518         $dest->{$k} = $source->{$k};
1519     }
1520 };
1521
1522 my $snapshot_prepare = sub {
1523     my ($vmid, $snapname, $comment) = @_;
1524
1525     my $snap;
1526
1527     my $updatefn =  sub {
1528
1529         my $conf = load_config($vmid);
1530
1531         die "you can't take a snapshot if it's a template\n"
1532             if is_template($conf);
1533
1534         check_lock($conf);
1535
1536         $conf->{lock} = 'snapshot';
1537
1538         die "snapshot name '$snapname' already used\n"
1539             if defined($conf->{snapshots}->{$snapname});
1540
1541         my $storecfg = PVE::Storage::config();
1542         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1543
1544         $snap = $conf->{snapshots}->{$snapname} = {};
1545
1546         &$snapshot_copy_config($conf, $snap);
1547
1548         $snap->{'snapstate'} = "prepare";
1549         $snap->{'snaptime'} = time();
1550         $snap->{'description'} = $comment if $comment;
1551         $conf->{snapshots}->{$snapname} = $snap;
1552
1553         write_config($vmid, $conf);
1554     };
1555
1556     lock_container($vmid, 10, $updatefn);
1557
1558     return $snap;
1559 };
1560
1561 my $snapshot_commit = sub {
1562     my ($vmid, $snapname) = @_;
1563
1564     my $updatefn = sub {
1565
1566         my $conf = load_config($vmid);
1567
1568         die "missing snapshot lock\n"
1569             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1570
1571         die "snapshot '$snapname' does not exist\n"
1572             if !defined($conf->{snapshots}->{$snapname});
1573
1574         die "wrong snapshot state\n"
1575             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1576                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1577
1578         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1579         delete $conf->{lock};
1580         $conf->{parent} = $snapname;
1581
1582         write_config($vmid, $conf);
1583     };
1584
1585     lock_container($vmid, 10 ,$updatefn);
1586 };
1587
1588 sub has_feature {
1589     my ($feature, $conf, $storecfg, $snapname) = @_;
1590
1591     my $err;
1592
1593     foreach_mountpoint($conf, sub {
1594         my ($ms, $mountpoint) = @_;
1595
1596         return if $err; # skip further test
1597
1598         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1599
1600         # TODO: implement support for mountpoints
1601         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1602             if $ms ne 'rootfs';
1603     });
1604
1605     return $err ? 0 : 1;
1606 }
1607
1608 sub snapshot_create {
1609     my ($vmid, $snapname, $comment) = @_;
1610
1611     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1612
1613     my $conf = load_config($vmid);
1614
1615     my $cmd = "/usr/bin/lxc-freeze -n $vmid";
1616     my $running = check_running($vmid);
1617     eval {
1618         if ($running) {
1619             PVE::Tools::run_command($cmd);
1620         };
1621
1622         my $storecfg = PVE::Storage::config();
1623         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1624         my $volid = $rootinfo->{volume};
1625
1626         $cmd = "/usr/bin/lxc-unfreeze -n $vmid";
1627         if ($running) {
1628             PVE::Tools::run_command($cmd);
1629         };
1630
1631         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1632         &$snapshot_commit($vmid, $snapname);
1633     };
1634     if(my $err = $@) {
1635         snapshot_delete($vmid, $snapname, 1);
1636         die "$err\n";
1637     }
1638 }
1639
1640 sub snapshot_delete {
1641     my ($vmid, $snapname, $force) = @_;
1642
1643     my $snap;
1644
1645     my $conf;
1646
1647     my $updatefn =  sub {
1648
1649         $conf = load_config($vmid);
1650
1651         die "you can't delete a snapshot if vm is a template\n"
1652             if is_template($conf);
1653
1654         $snap = $conf->{snapshots}->{$snapname};
1655
1656         check_lock($conf);
1657
1658         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1659
1660         $snap->{snapstate} = 'delete';
1661
1662         write_config($vmid, $conf);
1663     };
1664
1665     lock_container($vmid, 10, $updatefn);
1666
1667     my $storecfg = PVE::Storage::config();
1668
1669     my $del_snap =  sub {
1670
1671         check_lock($conf);
1672
1673         if ($conf->{parent} eq $snapname) {
1674             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1675                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1676             } else {
1677                 delete $conf->{parent};
1678             }
1679         }
1680
1681         delete $conf->{snapshots}->{$snapname};
1682
1683         write_config($vmid, $conf);
1684     };
1685
1686     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1687     my $rootinfo = parse_ct_mountpoint($rootfs);
1688     my $volid = $rootinfo->{volume};
1689
1690     eval {
1691         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1692     };
1693     my $err = $@;
1694
1695     if(!$err || ($err && $force)) {
1696         lock_container($vmid, 10, $del_snap);
1697         if ($err) {
1698             die "Can't delete snapshot: $vmid $snapname $err\n";
1699         }
1700     }
1701 }
1702
1703 sub snapshot_rollback {
1704     my ($vmid, $snapname) = @_;
1705
1706     my $storecfg = PVE::Storage::config();
1707
1708     my $conf = load_config($vmid);
1709
1710     die "you can't rollback if vm is a template\n" if is_template($conf);
1711
1712     my $snap = $conf->{snapshots}->{$snapname};
1713
1714     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1715
1716     my $rootfs = $snap->{rootfs};
1717     my $rootinfo = parse_ct_mountpoint($rootfs);
1718     my $volid = $rootinfo->{volume};
1719
1720     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1721
1722     my $updatefn = sub {
1723
1724         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1725             if $snap->{snapstate};
1726
1727         check_lock($conf);
1728
1729         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1730
1731         die "unable to rollback vm $vmid: vm is running\n"
1732             if check_running($vmid);
1733
1734         $conf->{lock} = 'rollback';
1735
1736         my $forcemachine;
1737
1738         # copy snapshot config to current config
1739
1740         my $tmp_conf = $conf;
1741         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1742         $conf->{snapshots} = $tmp_conf->{snapshots};
1743         delete $conf->{snaptime};
1744         delete $conf->{snapname};
1745         $conf->{parent} = $snapname;
1746
1747         write_config($vmid, $conf);
1748     };
1749
1750     my $unlockfn = sub {
1751         delete $conf->{lock};
1752         write_config($vmid, $conf);
1753     };
1754
1755     lock_container($vmid, 10, $updatefn);
1756
1757     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1758
1759     lock_container($vmid, 5, $unlockfn);
1760 }
1761
1762 sub template_create {
1763     my ($vmid, $conf) = @_;
1764
1765     my $storecfg = PVE::Storage::config();
1766
1767     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1768     my $volid = $rootinfo->{volume};
1769
1770     die "Template feature is not available for '$volid'\n"
1771         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1772
1773     PVE::Storage::activate_volumes($storecfg, [$volid]);
1774
1775     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1776     $rootinfo->{volume} = $template_volid;
1777     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1778
1779     write_config($vmid, $conf);
1780 }
1781
1782 sub is_template {
1783     my ($conf) = @_;
1784
1785     return 1 if defined $conf->{template} && $conf->{template} == 1;
1786 }
1787
1788 sub mountpoint_names {
1789     my ($reverse) = @_;
1790
1791     my @names = ('rootfs');
1792
1793     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1794         push @names, "mp$i";
1795     }
1796
1797     return $reverse ? reverse @names : @names;
1798 }
1799
1800 sub foreach_mountpoint_full {
1801     my ($conf, $reverse, $func) = @_;
1802
1803     foreach my $key (mountpoint_names($reverse)) {
1804         my $value = $conf->{$key};
1805         next if !defined($value);
1806         my $mountpoint = parse_ct_mountpoint($value);
1807         $mountpoint->{mp} = '/' if $key eq 'rootfs'; # just to be sure
1808         &$func($key, $mountpoint);
1809     }
1810 }
1811
1812 sub foreach_mountpoint {
1813     my ($conf, $func) = @_;
1814
1815     foreach_mountpoint_full($conf, 0, $func);
1816 }
1817
1818 sub foreach_mountpoint_reverse {
1819     my ($conf, $func) = @_;
1820
1821     foreach_mountpoint_full($conf, 1, $func);
1822 }
1823
1824 sub loopdevices_list {
1825
1826     my $loopdev = {};
1827     my $parser = sub {
1828         my $line = shift;
1829         if ($line =~ m/^(\/dev\/loop\d+)\s+\d\s+\d\s+\d\s+\d\s(\S+)$/) {
1830             $loopdev->{$1} = $2;
1831         }
1832     };
1833
1834     PVE::Tools::run_command(['losetup'], outfunc => $parser);
1835
1836     return $loopdev;
1837 }
1838
1839 sub blockdevices_list {
1840
1841     my $bdevs = {};
1842     dir_glob_foreach("/sys/dev/block/", '(\d+):(\d+)', sub {
1843         my (undef, $major, $minor) = @_;
1844         my $bdev = readlink("/sys/dev/block/$major:$minor");
1845         $bdev =~ s/\.\.\/\.\.\/devices\/virtual\/block\//\/dev\//;
1846         $bdevs->{$bdev}->{major} = $major;
1847         $bdevs->{$bdev}->{minor} = $minor;
1848     });
1849     return $bdevs;
1850 }
1851
1852 sub find_loopdev {
1853     my ($loopdevs, $path) = @_;
1854
1855     foreach my $dev (keys %$loopdevs){
1856         return $dev if $loopdevs->{$dev} eq $path;
1857     }
1858 }
1859
1860 sub check_ct_modify_config_perm {
1861     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1862
1863     return 1 if $authuser ne 'root@pam';
1864
1865     foreach my $opt (@$key_list) {
1866
1867         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1868             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1869         } elsif ($opt eq 'disk') {
1870             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1871         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1872             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1873         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1874                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1875             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1876         } else {
1877             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1878         }
1879     }
1880
1881     return 1;
1882 }
1883
1884 sub attach_loops {
1885     my ($storage_cfg, $vollist, $snapname) = @_;
1886
1887     my $loopdevs = {};
1888
1889     foreach my $volid (@$vollist) {
1890
1891         my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
1892         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1893
1894         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1895             PVE::Storage::parse_volname($storage_cfg, $volid);
1896
1897         if ($format eq 'raw' && $scfg->{path}) {
1898             my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
1899             my $loopdev;
1900
1901             my $parser = sub {
1902                 my $line = shift;
1903                 $loopdev = $line if $line =~m|^/dev/loop\d+$|;
1904                 $loopdevs->{$loopdev} = $path;
1905             };
1906
1907             PVE::Tools::run_command(['losetup', '--find', '--show', $path], outfunc => $parser);
1908         }
1909     }
1910
1911     return $loopdevs;
1912 }
1913
1914 sub detach_loops {
1915     my ($storage_cfg, $vollist, $snapname) = @_;
1916
1917     my $loopdevs = loopdevices_list();
1918
1919     foreach my $volid (@$vollist) {
1920
1921         my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
1922         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1923
1924         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1925             PVE::Storage::parse_volname($storage_cfg, $volid);
1926
1927         if ($format eq 'raw' && $scfg->{path}) {
1928             my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
1929             foreach my $dev (keys %$loopdevs){
1930                 PVE::Tools::run_command(['losetup', '-d', $dev]) if $loopdevs->{$dev} eq $path;
1931             }
1932         }
1933     }
1934 }
1935
1936 sub umount_all {
1937     my ($vmid, $storage_cfg, $conf, $noerr, $loopdevs) = @_;
1938
1939     $loopdevs ||= loopdevices_list();
1940
1941     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1942     my $volid_list = get_vm_volumes($conf);
1943
1944     foreach_mountpoint_reverse($conf, sub {
1945         my ($ms, $mountpoint) = @_;
1946
1947         my $volid = $mountpoint->{volume};
1948         my $mount = $mountpoint->{mp};
1949
1950         return if !$volid || !$mount;
1951
1952         my $mount_path = "$rootdir/$mount";
1953         $mount_path =~ s!/+!/!g;
1954
1955         return if !PVE::ProcFSTools::is_mounted($mount_path);
1956
1957         eval {
1958             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1959         };
1960         if (my $err = $@) {
1961             if ($noerr) {
1962                 warn $err;
1963             } else {
1964                 die $err;
1965             }
1966         }
1967     });
1968
1969     PVE::LXC::detach_loops($storage_cfg, $volid_list);
1970 }
1971
1972 sub mount_all {
1973     my ($vmid, $storage_cfg, $conf, $mkdirs) = @_;
1974
1975     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1976
1977     my $volid_list = get_vm_volumes($conf);
1978     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
1979
1980     my $loopdevs;
1981     eval {
1982         $loopdevs = attach_loops($storage_cfg, $volid_list);
1983
1984         foreach_mountpoint($conf, sub {
1985             my ($ms, $mountpoint) = @_;
1986
1987             my $volid = $mountpoint->{volume};
1988             my $mount = $mountpoint->{mp};
1989
1990             return if !$volid || !$mount;
1991
1992             my $image_path = PVE::Storage::path($storage_cfg, $volid);
1993             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1994                 PVE::Storage::parse_volname($storage_cfg, $volid);
1995
1996             die "unable to mount base volume - internal error" if $isBase;
1997
1998             File::Path::make_path "$rootdir/$mount" if $mkdirs;
1999             mountpoint_mount($mountpoint, $rootdir, $storage_cfg, $loopdevs);
2000         });
2001     };
2002     if (my $err = $@) {
2003         warn "mounting container failed - $err";
2004         umount_all($vmid, $storage_cfg, $conf, 1);
2005     }
2006
2007     return wantarray ? ($rootdir, $loopdevs) : $rootdir;
2008 }
2009
2010
2011 sub mountpoint_mount_path {
2012     my ($mountpoint, $storage_cfg, $loopdevs, $snapname) = @_;
2013
2014     return mountpoint_mount($mountpoint, undef, $storage_cfg, $loopdevs, $snapname);
2015 }
2016
2017 # use $rootdir = undef to just return the corresponding mount path
2018 sub mountpoint_mount {
2019     my ($mountpoint, $rootdir, $storage_cfg, $loopdevs, $snapname) = @_;
2020
2021     my $volid = $mountpoint->{volume};
2022     my $mount = $mountpoint->{mp};
2023
2024     return if !$volid || !$mount;
2025
2026     my $mount_path;
2027
2028     if (defined($rootdir)) {
2029         $rootdir =~ s!/+$!!;
2030         $mount_path = "$rootdir/$mount";
2031         $mount_path =~ s!/+!/!g;
2032         File::Path::mkpath($mount_path);
2033     }
2034
2035     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2036
2037     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2038
2039     if ($storage) {
2040
2041         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2042         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2043
2044         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2045             PVE::Storage::parse_volname($storage_cfg, $volid);
2046
2047         if ($format eq 'subvol') {
2048
2049             if ($mount_path) {
2050                 if ($snapname) {
2051                     if ($scfg->{type} eq 'zfspool') {
2052                         my $path_arg = $path;
2053                         $path_arg =~ s!^/+!!;
2054                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2055                     } else {
2056                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2057                     }
2058                 } else {
2059                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2060                 }
2061             }
2062             return $path;
2063
2064         } elsif ($format eq 'raw') {
2065
2066             if ($scfg->{path}) {
2067                 $path = find_loopdev($loopdevs, $path) if $loopdevs;
2068             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'rbd') {
2069                 # do nothing
2070             } else {
2071                 die "unsupported storage type '$scfg->{type}'\n";
2072             }
2073             if ($mount_path) {
2074                 if ($isBase || defined($snapname)) {
2075                     PVE::Tools::run_command(['mount', '-o', 'ro', $path, $mount_path]);
2076                 } else {
2077                     PVE::Tools::run_command(['mount', $path, $mount_path]);
2078                 }
2079             }
2080             return $path;
2081         } else {
2082             die "unsupported image format '$format'\n";
2083         }
2084     } elsif ($volid =~ m|^/dev/.+|) {
2085         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2086         return $volid;
2087     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2088         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2089         return $volid;
2090     }
2091
2092     die "unsupported storage";
2093 }
2094
2095 sub get_vm_volumes {
2096     my ($conf, $excludes) = @_;
2097
2098     my $vollist = [];
2099
2100     foreach_mountpoint($conf, sub {
2101         my ($ms, $mountpoint) = @_;
2102
2103         return if $excludes && $ms eq $excludes;
2104
2105         my $volid = $mountpoint->{volume};
2106
2107         return if !$volid || $volid =~ m|^/|;
2108
2109         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2110         return if !$sid;
2111
2112         push @$vollist, $volid;
2113     });
2114
2115     return $vollist;
2116 }
2117
2118 sub mkfs {
2119     my ($dev) = @_;
2120
2121     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2122 }
2123
2124 sub format_disk {
2125     my ($storage_cfg, $volid) = @_;
2126
2127     if ($volid =~ m!^/dev/.+!) {
2128         mkfs($volid);
2129         return;
2130     }
2131
2132     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2133
2134     die "cannot format volume '$volid' with no storage\n" if !$storage;
2135
2136     my $path = PVE::Storage::path($storage_cfg, $volid);
2137
2138     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2139         PVE::Storage::parse_volname($storage_cfg, $volid);
2140
2141     die "cannot format volume '$volid' (format == $format)\n"
2142         if $format ne 'raw';
2143
2144     mkfs($path);
2145 }
2146
2147 sub destroy_disks {
2148     my ($storecfg, $vollist) = @_;
2149
2150     foreach my $volid (@$vollist) {
2151         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2152         warn $@ if $@;
2153     }
2154 }
2155
2156 sub create_disks {
2157     my ($storecfg, $vmid, $settings, $conf) = @_;
2158
2159     my $vollist = [];
2160
2161     eval {
2162         foreach_mountpoint($settings, sub {
2163             my ($ms, $mountpoint) = @_;
2164
2165             my $volid = $mountpoint->{volume};
2166             my $mp = $mountpoint->{mp};
2167
2168             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2169
2170             return if !$storage;
2171
2172             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2173                 my ($storeid, $size) = ($1, $2);
2174
2175                 $size = int($size*1024) * 1024;
2176
2177                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2178                 # fixme: use better naming ct-$vmid-disk-X.raw?
2179
2180                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2181                     if ($size > 0) {
2182                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2183                                                            undef, $size);
2184                         format_disk($storecfg, $volid);
2185                     } else {
2186                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2187                                                            undef, 0);
2188                     }
2189                 } elsif ($scfg->{type} eq 'zfspool') {
2190
2191                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2192                                                undef, $size);
2193                 } elsif ($scfg->{type} eq 'drbd') {
2194
2195                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size);
2196                     format_disk($storecfg, $volid);
2197
2198                 } elsif ($scfg->{type} eq 'rbd') {
2199
2200                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2201                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size);
2202                     format_disk($storecfg, $volid);
2203                 } else {
2204                     die "unable to create containers on storage type '$scfg->{type}'\n";
2205                 }
2206                 push @$vollist, $volid;
2207                 my $new_mountpoint = { volume => $volid, size => $size, mp => $mp };
2208                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2209             } else {
2210                 # use specified/existing volid
2211             }
2212         });
2213     };
2214     # free allocated images on error
2215     if (my $err = $@) {
2216         destroy_disks($storecfg, $vollist);
2217         die $err;
2218     }
2219     return $vollist;
2220 }
2221
2222 # bash completion helper
2223
2224 sub complete_os_templates {
2225     my ($cmdname, $pname, $cvalue) = @_;
2226
2227     my $cfg = PVE::Storage::config();
2228
2229     my $storeid;
2230
2231     if ($cvalue =~ m/^([^:]+):/) {
2232         $storeid = $1;
2233     }
2234
2235     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2236     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2237
2238     my $res = [];
2239     foreach my $id (keys %$data) {
2240         foreach my $item (@{$data->{$id}}) {
2241             push @$res, $item->{volid} if defined($item->{volid});
2242         }
2243     }
2244
2245     return $res;
2246 }
2247
2248 sub complete_migration_target {
2249
2250     my $res = [];
2251
2252     my $nodelist = PVE::Cluster::get_nodelist();
2253     foreach my $node (@$nodelist) {
2254         next if $node eq $nodename;
2255         push @$res, $node;
2256     }
2257
2258     return $res;
2259 }
2260
2261 my $complete_ctid_full = sub {
2262     my ($running) = @_;
2263
2264     my $idlist = vmstatus();
2265
2266     my $active_hash = list_active_containers();
2267
2268     my $res = [];
2269
2270     foreach my $id (keys %$idlist) {
2271         my $d = $idlist->{$id};
2272         if (defined($running)) {
2273             next if $d->{template};
2274             next if $running && !$active_hash->{$id};
2275             next if !$running && $active_hash->{$id};
2276         }
2277         push @$res, $id;
2278
2279     }
2280     return $res;
2281 };
2282
2283 sub complete_ctid {
2284     return &$complete_ctid_full();
2285 }
2286
2287 sub complete_ctid_stopped {
2288     return &$complete_ctid_full(0);
2289 }
2290
2291 sub complete_ctid_running {
2292     return &$complete_ctid_full(1);
2293 }
2294
2295 1;