src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use Fcntl ':flock';
   9
  10 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  11 use PVE::Storage;
  12 use PVE::SafeSyslog;
  13 use PVE::INotify;
  14 use PVE::JSONSchema qw(get_standard_option);
  15 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  16 use PVE::Network;
  17 use PVE::AccessControl;
  18
  19 use Data::Dumper;
  20
  21 my $nodename = PVE::INotify::nodename();
  22
  23 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  24
  25 PVE::JSONSchema::register_format('pve-lxc-network', \&verify_lxc_network);
  26 sub verify_lxc_network {
  27     my ($value, $noerr) = @_;
  28
  29     return $value if parse_lxc_network($value);
  30
  31     return undef if $noerr;
  32
  33     die "unable to parse network setting\n";
  34 }
  35
  36 PVE::JSONSchema::register_format('pve-ct-mountpoint', \&verify_ct_mountpoint);
  37 sub verify_ct_mountpoint {
  38     my ($value, $noerr) = @_;
  39
  40     return $value if parse_ct_mountpoint($value);
  41
  42     return undef if $noerr;
  43
  44     die "unable to parse CT mountpoint options\n";
  45 }
  46
  47 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  48     type => 'string', format => 'pve-ct-mountpoint',
  49     typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+]',
  50     description => "Use volume as container root.",
  51     optional => 1,
  52 });
  53
  54 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  55     description => "The name of the snapshot.",
  56     type => 'string', format => 'pve-configid',
  57     maxLength => 40,
  58 });
  59
  60 my $confdesc = {
  61     lock => {
  62         optional => 1,
  63         type => 'string',
  64         description => "Lock/unlock the VM.",
  65         enum => [qw(migrate backup snapshot rollback)],
  66     },
  67     onboot => {
  68         optional => 1,
  69         type => 'boolean',
  70         description => "Specifies whether a VM will be started during system bootup.",
  71         default => 0,
  72     },
  73     startup => get_standard_option('pve-startup-order'),
  74     template => {
  75         optional => 1,
  76         type => 'boolean',
  77         description => "Enable/disable Template.",
  78         default => 0,
  79     },
  80     arch => {
  81         optional => 1,
  82         type => 'string',
  83         enum => ['amd64', 'i386'],
  84         description => "OS architecture type.",
  85         default => 'amd64',
  86     },
  87     ostype => {
  88         optional => 1,
  89         type => 'string',
  90         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  91         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  92     },
  93     console => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Attach a console device (/dev/console) to the container.",
  97         default => 1,
  98     },
  99     tty => {
 100         optional => 1,
 101         type => 'integer',
 102         description => "Specify the number of tty available to the container",
 103         minimum => 0,
 104         maximum => 6,
 105         default => 2,
 106     },
 107     cpulimit => {
 108         optional => 1,
 109         type => 'number',
 110         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 111         minimum => 0,
 112         maximum => 128,
 113         default => 0,
 114     },
 115     cpuunits => {
 116         optional => 1,
 117         type => 'integer',
 118         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 119         minimum => 0,
 120         maximum => 500000,
 121         default => 1024,
 122     },
 123     memory => {
 124         optional => 1,
 125         type => 'integer',
 126         description => "Amount of RAM for the VM in MB.",
 127         minimum => 16,
 128         default => 512,
 129     },
 130     swap => {
 131         optional => 1,
 132         type => 'integer',
 133         description => "Amount of SWAP for the VM in MB.",
 134         minimum => 0,
 135         default => 512,
 136     },
 137     hostname => {
 138         optional => 1,
 139         description => "Set a host name for the container.",
 140         type => 'string',
 141         maxLength => 255,
 142     },
 143     description => {
 144         optional => 1,
 145         type => 'string',
 146         description => "Container description. Only used on the configuration web interface.",
 147     },
 148     searchdomain => {
 149         optional => 1,
 150         type => 'string',
 151         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 152     },
 153     nameserver => {
 154         optional => 1,
 155         type => 'string',
 156         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 157     },
 158     rootfs => get_standard_option('pve-ct-rootfs'),
 159     parent => {
 160         optional => 1,
 161         type => 'string', format => 'pve-configid',
 162         maxLength => 40,
 163         description => "Parent snapshot name. This is used internally, and should not be modified.",
 164     },
 165     snaptime => {
 166         optional => 1,
 167         description => "Timestamp for snapshots.",
 168         type => 'integer',
 169         minimum => 0,
 170     },
 171     cmode => {
 172         optional => 1,
 173         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 174         type => 'string',
 175         enum => ['shell', 'console', 'tty'],
 176         default => 'tty',
 177     },
 178 };
 179
 180 my $valid_lxc_conf_keys = {
 181     'lxc.include' => 1,
 182     'lxc.arch' => 1,
 183     'lxc.utsname' => 1,
 184     'lxc.haltsignal' => 1,
 185     'lxc.rebootsignal' => 1,
 186     'lxc.stopsignal' => 1,
 187     'lxc.init_cmd' => 1,
 188     'lxc.network.type' => 1,
 189     'lxc.network.flags' => 1,
 190     'lxc.network.link' => 1,
 191     'lxc.network.mtu' => 1,
 192     'lxc.network.name' => 1,
 193     'lxc.network.hwaddr' => 1,
 194     'lxc.network.ipv4' => 1,
 195     'lxc.network.ipv4.gateway' => 1,
 196     'lxc.network.ipv6' => 1,
 197     'lxc.network.ipv6.gateway' => 1,
 198     'lxc.network.script.up' => 1,
 199     'lxc.network.script.down' => 1,
 200     'lxc.pts' => 1,
 201     'lxc.console.logfile' => 1,
 202     'lxc.console' => 1,
 203     'lxc.tty' => 1,
 204     'lxc.devttydir' => 1,
 205     'lxc.hook.autodev' => 1,
 206     'lxc.autodev' => 1,
 207     'lxc.kmsg' => 1,
 208     'lxc.mount' => 1,
 209     'lxc.mount.entry' => 1,
 210     'lxc.mount.auto' => 1,
 211     'lxc.rootfs' => 1,
 212     'lxc.rootfs.mount' => 1,
 213     'lxc.rootfs.options' => 1,
 214     # lxc.cgroup.*
 215     'lxc.cap.drop' => 1,
 216     'lxc.cap.keep' => 1,
 217     'lxc.aa_profile' => 1,
 218     'lxc.aa_allow_incomplete' => 1,
 219     'lxc.se_context' => 1,
 220     'lxc.seccomp' => 1,
 221     'lxc.id_map' => 1,
 222     'lxc.hook.pre-start' => 1,
 223     'lxc.hook.pre-mount' => 1,
 224     'lxc.hook.mount' => 1,
 225     'lxc.hook.start' => 1,
 226     'lxc.hook.post-stop' => 1,
 227     'lxc.hook.clone' => 1,
 228     'lxc.hook.destroy' => 1,
 229     'lxc.loglevel' => 1,
 230     'lxc.logfile' => 1,
 231     'lxc.start.auto' => 1,
 232     'lxc.start.delay' => 1,
 233     'lxc.start.order' => 1,
 234     'lxc.group' => 1,
 235     'lxc.environment' => 1,
 236     'lxc.' => 1,
 237     'lxc.' => 1,
 238     'lxc.' => 1,
 239     'lxc.' => 1,
 240 };
 241
 242 my $MAX_LXC_NETWORKS = 10;
 243 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 244     $confdesc->{"net$i"} = {
 245         optional => 1,
 246         type => 'string', format => 'pve-lxc-network',
 247         description => "Specifies network interfaces for the container.\n\n".
 248             "The string should have the follow format:\n\n".
 249             "-net<[0-9]> bridge=<vmbr<Nummber>>[,hwaddr=<MAC>]\n".
 250             "[,mtu=<Number>][,name=<String>][,ip=<IPv4Format/CIDR>]\n".
 251             ",ip6=<IPv6Format/CIDR>][,gw=<GatwayIPv4>]\n".
 252             ",gw6=<GatwayIPv6>][,firewall=<[1|0]>][,tag=<VlanNo>]",
 253     };
 254 }
 255
 256 my $MAX_MOUNT_POINTS = 10;
 257 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 258     $confdesc->{"mp$i"} = {
 259         optional => 1,
 260         type => 'string', format => 'pve-ct-mountpoint',
 261         typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+] [,mp=mountpoint]',
 262         description => "Use volume as container mount point (experimental feature).",
 263         optional => 1,
 264     };
 265 }
 266
 267 sub write_pct_config {
 268     my ($filename, $conf) = @_;
 269
 270     delete $conf->{snapstate}; # just to be sure
 271
 272     my $generate_raw_config = sub {
 273         my ($conf) = @_;
 274
 275         my $raw = '';
 276
 277         # add description as comment to top of file
 278         my $descr = $conf->{description} || '';
 279         foreach my $cl (split(/\n/, $descr)) {
 280             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 281         }
 282
 283         foreach my $key (sort keys %$conf) {
 284             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 285                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 286             $raw .= "$key: $conf->{$key}\n";
 287         }
 288
 289         if (my $lxcconf = $conf->{lxc}) {
 290             foreach my $entry (@$lxcconf) {
 291                 my ($k, $v) = @$entry;
 292                 $raw .= "$k: $v\n";
 293             }
 294         }
 295
 296         return $raw;
 297     };
 298
 299     my $raw = &$generate_raw_config($conf);
 300
 301     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 302         $raw .= "\n[$snapname]\n";
 303         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 304     }
 305
 306     return $raw;
 307 }
 308
 309 sub check_type {
 310     my ($key, $value) = @_;
 311
 312     die "unknown setting '$key'\n" if !$confdesc->{$key};
 313
 314     my $type = $confdesc->{$key}->{type};
 315
 316     if (!defined($value)) {
 317         die "got undefined value\n";
 318     }
 319
 320     if ($value =~ m/[\n\r]/) {
 321         die "property contains a line feed\n";
 322     }
 323
 324     if ($type eq 'boolean') {
 325         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 326         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 327         die "type check ('boolean') failed - got '$value'\n";
 328     } elsif ($type eq 'integer') {
 329         return int($1) if $value =~ m/^(\d+)$/;
 330         die "type check ('integer') failed - got '$value'\n";
 331     } elsif ($type eq 'number') {
 332         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 333         die "type check ('number') failed - got '$value'\n";
 334     } elsif ($type eq 'string') {
 335         if (my $fmt = $confdesc->{$key}->{format}) {
 336             PVE::JSONSchema::check_format($fmt, $value);
 337             return $value;
 338         }
 339         return $value;
 340     } else {
 341         die "internal error"
 342     }
 343 }
 344
 345 sub parse_pct_config {
 346     my ($filename, $raw) = @_;
 347
 348     return undef if !defined($raw);
 349
 350     my $res = {
 351         digest => Digest::SHA::sha1_hex($raw),
 352         snapshots => {},
 353     };
 354
 355     $filename =~ m|/lxc/(\d+).conf$|
 356         || die "got strange filename '$filename'";
 357
 358     my $vmid = $1;
 359
 360     my $conf = $res;
 361     my $descr = '';
 362     my $section = '';
 363
 364     my @lines = split(/\n/, $raw);
 365     foreach my $line (@lines) {
 366         next if $line =~ m/^\s*$/;
 367
 368         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 369             $section = $1;
 370             $conf->{description} = $descr if $descr;
 371             $descr = '';
 372             $conf = $res->{snapshots}->{$section} = {};
 373             next;
 374         }
 375
 376         if ($line =~ m/^\#(.*)\s*$/) {
 377             $descr .= PVE::Tools::decode_text($1) . "\n";
 378             next;
 379         }
 380
 381         if ($line =~ m/^(lxc\.[a-z0-9_\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 382             my $key = $1;
 383             my $value = $3;
 384             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 385                 push @{$conf->{lxc}}, [$key, $value];
 386             } else {
 387                 warn "vm $vmid - unable to parse config: $line\n";
 388             }
 389         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 390             $descr .= PVE::Tools::decode_text($2);
 391         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 392             $conf->{snapstate} = $1;
 393         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
 394             my $key = $1;
 395             my $value = $2;
 396             eval { $value = check_type($key, $value); };
 397             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 398             $conf->{$key} = $value;
 399         } else {
 400             warn "vm $vmid - unable to parse config: $line\n";
 401         }
 402     }
 403
 404     $conf->{description} = $descr if $descr;
 405
 406     delete $res->{snapstate}; # just to be sure
 407
 408     return $res;
 409 }
 410
 411 sub config_list {
 412     my $vmlist = PVE::Cluster::get_vmlist();
 413     my $res = {};
 414     return $res if !$vmlist || !$vmlist->{ids};
 415     my $ids = $vmlist->{ids};
 416
 417     foreach my $vmid (keys %$ids) {
 418         next if !$vmid; # skip CT0
 419         my $d = $ids->{$vmid};
 420         next if !$d->{node} || $d->{node} ne $nodename;
 421         next if !$d->{type} || $d->{type} ne 'lxc';
 422         $res->{$vmid}->{type} = 'lxc';
 423     }
 424     return $res;
 425 }
 426
 427 sub cfs_config_path {
 428     my ($vmid, $node) = @_;
 429
 430     $node = $nodename if !$node;
 431     return "nodes/$node/lxc/$vmid.conf";
 432 }
 433
 434 sub config_file {
 435     my ($vmid, $node) = @_;
 436
 437     my $cfspath = cfs_config_path($vmid, $node);
 438     return "/etc/pve/$cfspath";
 439 }
 440
 441 sub load_config {
 442     my ($vmid, $node) = @_;
 443
 444     $node = $nodename if !$node;
 445     my $cfspath = cfs_config_path($vmid, $node);
 446
 447     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 448     die "container $vmid does not exists\n" if !defined($conf);
 449
 450     return $conf;
 451 }
 452
 453 sub create_config {
 454     my ($vmid, $conf) = @_;
 455
 456     my $dir = "/etc/pve/nodes/$nodename/lxc";
 457     mkdir $dir;
 458
 459     write_config($vmid, $conf);
 460 }
 461
 462 sub destroy_config {
 463     my ($vmid) = @_;
 464
 465     unlink config_file($vmid, $nodename);
 466 }
 467
 468 sub write_config {
 469     my ($vmid, $conf) = @_;
 470
 471     my $cfspath = cfs_config_path($vmid);
 472
 473     PVE::Cluster::cfs_write_file($cfspath, $conf);
 474 }
 475
 476 # flock: we use one file handle per process, so lock file
 477 # can be called multiple times and succeeds for the same process.
 478
 479 my $lock_handles =  {};
 480 my $lockdir = "/run/lock/lxc";
 481
 482 sub lock_filename {
 483     my ($vmid) = @_;
 484
 485     return "$lockdir/pve-config-${vmid}.lock";
 486 }
 487
 488 sub lock_aquire {
 489     my ($vmid, $timeout) = @_;
 490
 491     $timeout = 10 if !$timeout;
 492     my $mode = LOCK_EX;
 493
 494     my $filename = lock_filename($vmid);
 495
 496     mkdir $lockdir if !-d $lockdir;
 497
 498     my $lock_func = sub {
 499         if (!$lock_handles->{$$}->{$filename}) {
 500             my $fh = new IO::File(">>$filename") ||
 501                 die "can't open file - $!\n";
 502             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 503         }
 504
 505         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 506             print STDERR "trying to aquire lock...";
 507             my $success;
 508             while(1) {
 509                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 510                 # try again on EINTR (see bug #273)
 511                 if ($success || ($! != EINTR)) {
 512                     last;
 513                 }
 514             }
 515             if (!$success) {
 516                 print STDERR " failed\n";
 517                 die "can't aquire lock - $!\n";
 518             }
 519
 520             print STDERR " OK\n";
 521         }
 522
 523         $lock_handles->{$$}->{$filename}->{refcount}++;
 524     };
 525
 526     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 527     my $err = $@;
 528     if ($err) {
 529         die "can't lock file '$filename' - $err";
 530     }
 531 }
 532
 533 sub lock_release {
 534     my ($vmid) = @_;
 535
 536     my $filename = lock_filename($vmid);
 537
 538     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 539         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 540         if ($refcount <= 0) {
 541             $lock_handles->{$$}->{$filename} = undef;
 542             close ($fh);
 543         }
 544     }
 545 }
 546
 547 sub lock_container {
 548     my ($vmid, $timeout, $code, @param) = @_;
 549
 550     my $res;
 551
 552     lock_aquire($vmid, $timeout);
 553     eval { $res = &$code(@param) };
 554     my $err = $@;
 555     lock_release($vmid);
 556
 557     die $err if $err;
 558
 559     return $res;
 560 }
 561
 562 sub option_exists {
 563     my ($name) = @_;
 564
 565     return defined($confdesc->{$name});
 566 }
 567
 568 # add JSON properties for create and set function
 569 sub json_config_properties {
 570     my $prop = shift;
 571
 572     foreach my $opt (keys %$confdesc) {
 573         next if $opt eq 'parent' || $opt eq 'snaptime';
 574         next if $prop->{$opt};
 575         $prop->{$opt} = $confdesc->{$opt};
 576     }
 577
 578     return $prop;
 579 }
 580
 581 sub json_config_properties_no_rootfs {
 582     my $prop = shift;
 583
 584     foreach my $opt (keys %$confdesc) {
 585         next if $prop->{$opt};
 586         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 587         $prop->{$opt} = $confdesc->{$opt};
 588     }
 589
 590     return $prop;
 591 }
 592
 593 # container status helpers
 594
 595 sub list_active_containers {
 596
 597     my $filename = "/proc/net/unix";
 598
 599     # similar test is used by lcxcontainers.c: list_active_containers
 600     my $res = {};
 601
 602     my $fh = IO::File->new ($filename, "r");
 603     return $res if !$fh;
 604
 605     while (defined(my $line = <$fh>)) {
 606         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 607             my $path = $1;
 608             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 609                 $res->{$1} = 1;
 610             }
 611         }
 612     }
 613
 614     close($fh);
 615
 616     return $res;
 617 }
 618
 619 # warning: this is slow
 620 sub check_running {
 621     my ($vmid) = @_;
 622
 623     my $active_hash = list_active_containers();
 624
 625     return 1 if defined($active_hash->{$vmid});
 626
 627     return undef;
 628 }
 629
 630 sub get_container_disk_usage {
 631     my ($vmid) = @_;
 632
 633     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 634
 635     my $res = {
 636         total => 0,
 637         used => 0,
 638         avail => 0,
 639     };
 640
 641     my $parser = sub {
 642         my $line = shift;
 643         if (my ($fsid, $total, $used, $avail) = $line =~
 644             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 645             $res = {
 646                 total => $total,
 647                 used => $used,
 648                 avail => $avail,
 649             };
 650         }
 651     };
 652     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 653     warn $@ if $@;
 654
 655     return $res;
 656 }
 657
 658 sub vmstatus {
 659     my ($opt_vmid) = @_;
 660
 661     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 662
 663     my $active_hash = list_active_containers();
 664
 665     foreach my $vmid (keys %$list) {
 666         my $d = $list->{$vmid};
 667
 668         my $running = defined($active_hash->{$vmid});
 669
 670         $d->{status} = $running ? 'running' : 'stopped';
 671
 672         my $cfspath = cfs_config_path($vmid);
 673         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 674
 675         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 676         $d->{name} =~ s/[\s]//g;
 677
 678         $d->{cpus} = $conf->{cpulimit} // 0;
 679
 680         if ($running) {
 681             my $res = get_container_disk_usage($vmid);
 682             $d->{disk} = $res->{used};
 683             $d->{maxdisk} = $res->{total};
 684         } else {
 685             $d->{disk} = 0;
 686             # use 4GB by default ??
 687             if (my $rootfs = $conf->{rootfs}) {
 688                 my $rootinfo = parse_ct_mountpoint($rootfs);
 689                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 690             } else {
 691                 $d->{maxdisk} = 4*1024*1024*1024;
 692             }
 693         }
 694
 695         $d->{mem} = 0;
 696         $d->{swap} = 0;
 697         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 698         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 699
 700         $d->{uptime} = 0;
 701         $d->{cpu} = 0;
 702
 703         $d->{netout} = 0;
 704         $d->{netin} = 0;
 705
 706         $d->{diskread} = 0;
 707         $d->{diskwrite} = 0;
 708
 709         $d->{template} = is_template($conf);
 710     }
 711
 712     foreach my $vmid (keys %$list) {
 713         my $d = $list->{$vmid};
 714         next if $d->{status} ne 'running';
 715
 716         $d->{uptime} = 100; # fixme:
 717
 718         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 719         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 720
 721         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 722         my @bytes = split(/\n/, $blkio_bytes);
 723         foreach my $byte (@bytes) {
 724             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 725                 $d->{diskread} = $2 if $key eq 'Read';
 726                 $d->{diskwrite} = $2 if $key eq 'Write';
 727             }
 728         }
 729     }
 730
 731     return $list;
 732 }
 733
 734 my $parse_size = sub {
 735     my ($value) = @_;
 736
 737     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 738     my ($size, $unit) = ($1, $3);
 739     if ($unit) {
 740         if ($unit eq 'K') {
 741             $size = $size * 1024;
 742         } elsif ($unit eq 'M') {
 743             $size = $size * 1024 * 1024;
 744         } elsif ($unit eq 'G') {
 745             $size = $size * 1024 * 1024 * 1024;
 746         }
 747     }
 748     return int($size);
 749 };
 750
 751 sub parse_ct_mountpoint {
 752     my ($data) = @_;
 753
 754     $data //= '';
 755
 756     my $res = {};
 757
 758     foreach my $p (split (/,/, $data)) {
 759         next if $p =~ m/^\s*$/;
 760
 761         if ($p =~ m/^(volume|backup|size|mp)=(.+)$/) {
 762             my ($k, $v) = ($1, $2);
 763             return undef if defined($res->{$k});
 764             $res->{$k} = $v;
 765         } else {
 766             if (!$res->{volume} && $p !~ m/=/) {
 767                 $res->{volume} = $p;
 768             } else {
 769                 return undef;
 770             }
 771         }
 772     }
 773
 774     return undef if !defined($res->{volume});
 775
 776     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 777
 778     if ($res->{size}) {
 779         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 780     }
 781
 782     return $res;
 783 }
 784
 785 sub print_ct_mountpoint {
 786     my ($info) = @_;
 787
 788     my $opts = '';
 789
 790     die "missing volume\n" if !$info->{volume};
 791
 792     foreach my $o ('size', 'backup') {
 793         $opts .= ",$o=$info->{$o}" if defined($info->{$o});
 794     }
 795
 796     return "$info->{volume}$opts";
 797 }
 798
 799 sub print_lxc_network {
 800     my $net = shift;
 801
 802     die "no network name defined\n" if !$net->{name};
 803
 804     my $res = "name=$net->{name}";
 805
 806     foreach my $k (qw(hwaddr mtu bridge ip gw ip6 gw6 firewall tag)) {
 807         next if !defined($net->{$k});
 808         $res .= ",$k=$net->{$k}";
 809     }
 810
 811     return $res;
 812 }
 813
 814 sub parse_lxc_network {
 815     my ($data) = @_;
 816
 817     my $res = {};
 818
 819     return $res if !$data;
 820
 821     foreach my $pv (split (/,/, $data)) {
 822         if ($pv =~ m/^(bridge|hwaddr|mtu|name|ip|ip6|gw|gw6|firewall|tag)=(\S+)$/) {
 823             $res->{$1} = $2;
 824         } else {
 825             return undef;
 826         }
 827     }
 828
 829     $res->{type} = 'veth';
 830     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 831
 832     return $res;
 833 }
 834
 835 sub read_cgroup_value {
 836     my ($group, $vmid, $name, $full) = @_;
 837
 838     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 839
 840     return PVE::Tools::file_get_contents($path) if $full;
 841
 842     return PVE::Tools::file_read_firstline($path);
 843 }
 844
 845 sub write_cgroup_value {
 846    my ($group, $vmid, $name, $value) = @_;
 847
 848    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 849    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 850
 851 }
 852
 853 sub find_lxc_console_pids {
 854
 855     my $res = {};
 856
 857     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 858         my ($pid) = @_;
 859
 860         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 861         return if !$cmdline;
 862
 863         my @args = split(/\0/, $cmdline);
 864
 865         # serach for lxc-console -n <vmid>
 866         return if scalar(@args) != 3;
 867         return if $args[1] ne '-n';
 868         return if $args[2] !~ m/^\d+$/;
 869         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 870
 871         my $vmid = $args[2];
 872
 873         push @{$res->{$vmid}}, $pid;
 874     });
 875
 876     return $res;
 877 }
 878
 879 sub find_lxc_pid {
 880     my ($vmid) = @_;
 881
 882     my $pid = undef;
 883     my $parser = sub {
 884         my $line = shift;
 885         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 886     };
 887     PVE::Tools::run_command(['lxc-info', '-n', $vmid], outfunc => $parser);
 888
 889     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 890
 891     return $pid;
 892 }
 893
 894 my $ipv4_reverse_mask = [
 895     '0.0.0.0',
 896     '128.0.0.0',
 897     '192.0.0.0',
 898     '224.0.0.0',
 899     '240.0.0.0',
 900     '248.0.0.0',
 901     '252.0.0.0',
 902     '254.0.0.0',
 903     '255.0.0.0',
 904     '255.128.0.0',
 905     '255.192.0.0',
 906     '255.224.0.0',
 907     '255.240.0.0',
 908     '255.248.0.0',
 909     '255.252.0.0',
 910     '255.254.0.0',
 911     '255.255.0.0',
 912     '255.255.128.0',
 913     '255.255.192.0',
 914     '255.255.224.0',
 915     '255.255.240.0',
 916     '255.255.248.0',
 917     '255.255.252.0',
 918     '255.255.254.0',
 919     '255.255.255.0',
 920     '255.255.255.128',
 921     '255.255.255.192',
 922     '255.255.255.224',
 923     '255.255.255.240',
 924     '255.255.255.248',
 925     '255.255.255.252',
 926     '255.255.255.254',
 927     '255.255.255.255',
 928 ];
 929
 930 # Note: we cannot use Net:IP, because that only allows strict
 931 # CIDR networks
 932 sub parse_ipv4_cidr {
 933     my ($cidr, $noerr) = @_;
 934
 935     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
 936         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
 937     }
 938
 939     return undef if $noerr;
 940
 941     die "unable to parse ipv4 address/mask\n";
 942 }
 943
 944 sub check_lock {
 945     my ($conf) = @_;
 946
 947     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
 948 }
 949
 950 sub update_lxc_config {
 951     my ($storage_cfg, $vmid, $conf) = @_;
 952
 953     my $dir = "/var/lib/lxc/$vmid";
 954
 955     if ($conf->{template}) {
 956
 957         unlink "$dir/config";
 958
 959         return;
 960     }
 961
 962     my $raw = '';
 963
 964     die "missing 'arch' - internal error" if !$conf->{arch};
 965     $raw .= "lxc.arch = $conf->{arch}\n";
 966
 967     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
 968     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
 969         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
 970     } else {
 971         die "implement me";
 972     }
 973
 974     if (!has_dev_console($conf)) {
 975         $raw .= "lxc.console = none\n";
 976         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
 977     }
 978
 979     my $ttycount = get_tty_count($conf);
 980     $raw .= "lxc.tty = $ttycount\n";
 981
 982     my $utsname = $conf->{hostname} || "CT$vmid";
 983     $raw .= "lxc.utsname = $utsname\n";
 984
 985     my $memory = $conf->{memory} || 512;
 986     my $swap = $conf->{swap} // 0;
 987
 988     my $lxcmem = int($memory*1024*1024);
 989     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
 990
 991     my $lxcswap = int(($memory + $swap)*1024*1024);
 992     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
 993
 994     if (my $cpulimit = $conf->{cpulimit}) {
 995         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
 996         my $value = int(100000*$cpulimit);
 997         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
 998     }
 999
1000     my $shares = $conf->{cpuunits} || 1024;
1001     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1002
1003     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1004     $mountpoint->{mp} = '/';
1005     my $volid = $mountpoint->{volume};
1006     my $path = mountpoint_mount_path($mountpoint, $storage_cfg);
1007
1008     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1009
1010     if ($storage) {
1011         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1012         $path = "loop:$path" if $scfg->{path};
1013     }
1014
1015     $raw .= "lxc.rootfs = $path\n";
1016
1017     my $netcount = 0;
1018     foreach my $k (keys %$conf) {
1019         next if $k !~ m/^net(\d+)$/;
1020         my $ind = $1;
1021         my $d = parse_lxc_network($conf->{$k});
1022         $netcount++;
1023         $raw .= "lxc.network.type = veth\n";
1024         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1025         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1026         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1027         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1028     }
1029
1030     if (my $lxcconf = $conf->{lxc}) {
1031         foreach my $entry (@$lxcconf) {
1032             my ($k, $v) = @$entry;
1033             $netcount++ if $k eq 'lxc.network.type';
1034             $raw .= "$k = $v\n";
1035         }
1036     }
1037
1038     $raw .= "lxc.network.type = empty\n" if !$netcount;
1039
1040     File::Path::mkpath("$dir/rootfs");
1041
1042     PVE::Tools::file_set_contents("$dir/config", $raw);
1043 }
1044
1045 # verify and cleanup nameserver list (replace \0 with ' ')
1046 sub verify_nameserver_list {
1047     my ($nameserver_list) = @_;
1048
1049     my @list = ();
1050     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1051         PVE::JSONSchema::pve_verify_ip($server);
1052         push @list, $server;
1053     }
1054
1055     return join(' ', @list);
1056 }
1057
1058 sub verify_searchdomain_list {
1059     my ($searchdomain_list) = @_;
1060
1061     my @list = ();
1062     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1063         # todo: should we add checks for valid dns domains?
1064         push @list, $server;
1065     }
1066
1067     return join(' ', @list);
1068 }
1069
1070 sub update_pct_config {
1071     my ($vmid, $conf, $running, $param, $delete) = @_;
1072
1073     my @nohotplug;
1074
1075     my $rootdir;
1076     if ($running) {
1077         my $pid = find_lxc_pid($vmid);
1078         $rootdir = "/proc/$pid/root";
1079     }
1080
1081     if (defined($delete)) {
1082         foreach my $opt (@$delete) {
1083             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1084                 die "unable to delete required option '$opt'\n";
1085             } elsif ($opt eq 'swap') {
1086                 delete $conf->{$opt};
1087                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1088             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1089                 delete $conf->{$opt};
1090             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1091                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1092                 delete $conf->{$opt};
1093                 push @nohotplug, $opt;
1094                 next if $running;
1095             } elsif ($opt =~ m/^net(\d)$/) {
1096                 delete $conf->{$opt};
1097                 next if !$running;
1098                 my $netid = $1;
1099                 PVE::Network::veth_delete("veth${vmid}i$netid");
1100             } elsif ($opt eq 'rootfs' || $opt =~ m/^mp(\d+)$/) {
1101                 die "implement me"
1102             } else {
1103                 die "implement me"
1104             }
1105             write_config($vmid, $conf) if $running;
1106         }
1107     }
1108
1109     # There's no separate swap size to configure, there's memory and "total"
1110     # memory (iow. memory+swap). This means we have to change them together.
1111     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1112     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1113     if (defined($wanted_memory) || defined($wanted_swap)) {
1114
1115         $wanted_memory //= ($conf->{memory} || 512);
1116         $wanted_swap //=  ($conf->{swap} || 0);
1117
1118         my $total = $wanted_memory + $wanted_swap;
1119         if ($running) {
1120             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1121             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1122         }
1123         $conf->{memory} = $wanted_memory;
1124         $conf->{swap} = $wanted_swap;
1125
1126         write_config($vmid, $conf) if $running;
1127     }
1128
1129     foreach my $opt (keys %$param) {
1130         my $value = $param->{$opt};
1131         if ($opt eq 'hostname') {
1132             $conf->{$opt} = $value;
1133         } elsif ($opt eq 'onboot') {
1134             $conf->{$opt} = $value ? 1 : 0;
1135         } elsif ($opt eq 'startup') {
1136             $conf->{$opt} = $value;
1137         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1138             $conf->{$opt} = $value;
1139             push @nohotplug, $opt;
1140             next if $running;
1141         } elsif ($opt eq 'nameserver') {
1142             my $list = verify_nameserver_list($value);
1143             $conf->{$opt} = $list;
1144             push @nohotplug, $opt;
1145             next if $running;
1146         } elsif ($opt eq 'searchdomain') {
1147             my $list = verify_searchdomain_list($value);
1148             $conf->{$opt} = $list;
1149             push @nohotplug, $opt;
1150             next if $running;
1151         } elsif ($opt eq 'cpulimit') {
1152             $conf->{$opt} = $value;
1153             push @nohotplug, $opt; # fixme: hotplug
1154             next;
1155         } elsif ($opt eq 'cpuunits') {
1156             $conf->{$opt} = $value;
1157             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1158         } elsif ($opt eq 'description') {
1159             $conf->{$opt} = PVE::Tools::encode_text($value);
1160         } elsif ($opt =~ m/^net(\d+)$/) {
1161             my $netid = $1;
1162             my $net = parse_lxc_network($value);
1163             if (!$running) {
1164                 $conf->{$opt} = print_lxc_network($net);
1165             } else {
1166                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1167             }
1168         } elsif ($opt eq 'rootfs' || $opt =~ m/^mp(\d+)$/) {
1169             die "implement me: $opt";
1170         } else {
1171             die "implement me: $opt";
1172         }
1173         write_config($vmid, $conf) if $running;
1174     }
1175
1176     if ($running && scalar(@nohotplug)) {
1177         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1178     }
1179 }
1180
1181 sub has_dev_console {
1182     my ($conf) = @_;
1183
1184     return !(defined($conf->{console}) && !$conf->{console});
1185 }
1186
1187 sub get_tty_count {
1188     my ($conf) = @_;
1189
1190     return $conf->{tty} // $confdesc->{tty}->{default};
1191 }
1192
1193 sub get_cmode {
1194     my ($conf) = @_;
1195
1196     return $conf->{cmode} // $confdesc->{cmode}->{default};
1197 }
1198
1199 sub get_console_command {
1200     my ($vmid, $conf) = @_;
1201
1202     my $cmode = get_cmode($conf);
1203
1204     if ($cmode eq 'console') {
1205         return ['lxc-console', '-n',  $vmid, '-t', 0];
1206     } elsif ($cmode eq 'tty') {
1207         return ['lxc-console', '-n',  $vmid];
1208     } elsif ($cmode eq 'shell') {
1209         return ['lxc-attach', '--clear-env', '-n', $vmid];
1210     } else {
1211         die "internal error";
1212     }
1213 }
1214
1215 sub get_primary_ips {
1216     my ($conf) = @_;
1217
1218     # return data from net0
1219
1220     return undef if !defined($conf->{net0});
1221     my $net = parse_lxc_network($conf->{net0});
1222
1223     my $ipv4 = $net->{ip};
1224     if ($ipv4) {
1225         if ($ipv4 =~ /^(dhcp|manual)$/) {
1226             $ipv4 = undef
1227         } else {
1228             $ipv4 =~ s!/\d+$!!;
1229         }
1230     }
1231     my $ipv6 = $net->{ip6};
1232     if ($ipv6) {
1233         if ($ipv6 =~ /^(dhcp|manual)$/) {
1234             $ipv6 = undef;
1235         } else {
1236             $ipv6 =~ s!/\d+$!!;
1237         }
1238     }
1239
1240     return ($ipv4, $ipv6);
1241 }
1242
1243
1244 sub destroy_lxc_container {
1245     my ($storage_cfg, $vmid, $conf) = @_;
1246
1247     foreach_mountpoint($conf, sub {
1248         my ($ms, $mountpoint) = @_;
1249         my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $mountpoint->{volume});
1250         PVE::Storage::vdisk_free($storage_cfg, $mountpoint->{volume}) if $vmid == $owner;
1251     });
1252
1253     rmdir "/var/lib/lxc/$vmid/rootfs";
1254     unlink "/var/lib/lxc/$vmid/config";
1255     rmdir "/var/lib/lxc/$vmid";
1256     destroy_config($vmid);
1257
1258     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1259     #PVE::Tools::run_command($cmd);
1260 }
1261
1262 sub vm_stop_cleanup {
1263     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1264
1265     eval {
1266         if (!$keepActive) {
1267
1268             my $vollist = get_vm_volumes($conf);
1269             my $loopdevlist = get_vm_volumes($conf, 'rootfs');
1270
1271             dettach_loops($storage_cfg, $loopdevlist);
1272             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1273         }
1274     };
1275     warn $@ if $@; # avoid errors - just warn
1276 }
1277
1278 my $safe_num_ne = sub {
1279     my ($a, $b) = @_;
1280
1281     return 0 if !defined($a) && !defined($b);
1282     return 1 if !defined($a);
1283     return 1 if !defined($b);
1284
1285     return $a != $b;
1286 };
1287
1288 my $safe_string_ne = sub {
1289     my ($a, $b) = @_;
1290
1291     return 0 if !defined($a) && !defined($b);
1292     return 1 if !defined($a);
1293     return 1 if !defined($b);
1294
1295     return $a ne $b;
1296 };
1297
1298 sub update_net {
1299     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1300
1301     if ($newnet->{type} ne 'veth') {
1302         # for when there are physical interfaces
1303         die "cannot update interface of type $newnet->{type}";
1304     }
1305
1306     my $veth = "veth${vmid}i${netid}";
1307     my $eth = $newnet->{name};
1308
1309     if (my $oldnetcfg = $conf->{$opt}) {
1310         my $oldnet = parse_lxc_network($oldnetcfg);
1311
1312         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1313             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1314
1315             PVE::Network::veth_delete($veth);
1316             delete $conf->{$opt};
1317             write_config($vmid, $conf);
1318
1319             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1320
1321         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1322                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1323                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1324
1325                 if ($oldnet->{bridge}) {
1326                     PVE::Network::tap_unplug($veth);
1327                     foreach (qw(bridge tag firewall)) {
1328                         delete $oldnet->{$_};
1329                     }
1330                     $conf->{$opt} = print_lxc_network($oldnet);
1331                     write_config($vmid, $conf);
1332                 }
1333
1334                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1335                 foreach (qw(bridge tag firewall)) {
1336                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1337                 }
1338                 $conf->{$opt} = print_lxc_network($oldnet);
1339                 write_config($vmid, $conf);
1340         }
1341     } else {
1342         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1343     }
1344
1345     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1346 }
1347
1348 sub hotplug_net {
1349     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1350
1351     my $veth = "veth${vmid}i${netid}";
1352     my $vethpeer = $veth . "p";
1353     my $eth = $newnet->{name};
1354
1355     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1356     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1357
1358     # attach peer in container
1359     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1360     PVE::Tools::run_command($cmd);
1361
1362     # link up peer in container
1363     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1364     PVE::Tools::run_command($cmd);
1365
1366     my $done = { type => 'veth' };
1367     foreach (qw(bridge tag firewall hwaddr name)) {
1368         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1369     }
1370     $conf->{$opt} = print_lxc_network($done);
1371
1372     write_config($vmid, $conf);
1373 }
1374
1375 sub update_ipconfig {
1376     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1377
1378     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1379
1380     my $optdata = parse_lxc_network($conf->{$opt});
1381     my $deleted = [];
1382     my $added = [];
1383     my $nscmd = sub {
1384         my $cmdargs = shift;
1385         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1386     };
1387     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1388
1389     my $change_ip_config = sub {
1390         my ($ipversion) = @_;
1391
1392         my $family_opt = "-$ipversion";
1393         my $suffix = $ipversion == 4 ? '' : $ipversion;
1394         my $gw= "gw$suffix";
1395         my $ip= "ip$suffix";
1396
1397         my $newip = $newnet->{$ip};
1398         my $newgw = $newnet->{$gw};
1399         my $oldip = $optdata->{$ip};
1400
1401         my $change_ip = &$safe_string_ne($oldip, $newip);
1402         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1403
1404         return if !$change_ip && !$change_gw;
1405
1406         # step 1: add new IP, if this fails we cancel
1407         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1408             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1409             if (my $err = $@) {
1410                 warn $err;
1411                 return;
1412             }
1413         }
1414
1415         # step 2: replace gateway
1416         #   If this fails we delete the added IP and cancel.
1417         #   If it succeeds we save the config and delete the old IP, ignoring
1418         #   errors. The config is then saved.
1419         # Note: 'ip route replace' can add
1420         if ($change_gw) {
1421             if ($newgw) {
1422                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1423                 if (my $err = $@) {
1424                     warn $err;
1425                     # the route was not replaced, the old IP is still available
1426                     # rollback (delete new IP) and cancel
1427                     if ($change_ip) {
1428                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1429                         warn $@ if $@; # no need to die here
1430                     }
1431                     return;
1432                 }
1433             } else {
1434                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1435                 # if the route was not deleted, the guest might have deleted it manually
1436                 # warn and continue
1437                 warn $@ if $@;
1438             }
1439         }
1440
1441         # from this point on we save the configuration
1442         # step 3: delete old IP ignoring errors
1443         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1444             # We need to enable promote_secondaries, otherwise our newly added
1445             # address will be removed along with the old one.
1446             my $promote = 0;
1447             eval {
1448                 if ($ipversion == 4) {
1449                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1450                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1451                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1452                 }
1453                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1454             };
1455             warn $@ if $@; # no need to die here
1456
1457             if ($ipversion == 4) {
1458                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1459             }
1460         }
1461
1462         foreach my $property ($ip, $gw) {
1463             if ($newnet->{$property}) {
1464                 $optdata->{$property} = $newnet->{$property};
1465             } else {
1466                 delete $optdata->{$property};
1467             }
1468         }
1469         $conf->{$opt} = print_lxc_network($optdata);
1470         write_config($vmid, $conf);
1471         $lxc_setup->setup_network($conf);
1472     };
1473
1474     &$change_ip_config(4);
1475     &$change_ip_config(6);
1476
1477 }
1478
1479 # Internal snapshots
1480
1481 # NOTE: Snapshot create/delete involves several non-atomic
1482 # action, and can take a long time.
1483 # So we try to avoid locking the file and use 'lock' variable
1484 # inside the config file instead.
1485
1486 my $snapshot_copy_config = sub {
1487     my ($source, $dest) = @_;
1488
1489     foreach my $k (keys %$source) {
1490         next if $k eq 'snapshots';
1491         next if $k eq 'snapstate';
1492         next if $k eq 'snaptime';
1493         next if $k eq 'vmstate';
1494         next if $k eq 'lock';
1495         next if $k eq 'digest';
1496         next if $k eq 'description';
1497
1498         $dest->{$k} = $source->{$k};
1499     }
1500 };
1501
1502 my $snapshot_prepare = sub {
1503     my ($vmid, $snapname, $comment) = @_;
1504
1505     my $snap;
1506
1507     my $updatefn =  sub {
1508
1509         my $conf = load_config($vmid);
1510
1511         die "you can't take a snapshot if it's a template\n"
1512             if is_template($conf);
1513
1514         check_lock($conf);
1515
1516         $conf->{lock} = 'snapshot';
1517
1518         die "snapshot name '$snapname' already used\n"
1519             if defined($conf->{snapshots}->{$snapname});
1520
1521         my $storecfg = PVE::Storage::config();
1522         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1523
1524         $snap = $conf->{snapshots}->{$snapname} = {};
1525
1526         &$snapshot_copy_config($conf, $snap);
1527
1528         $snap->{'snapstate'} = "prepare";
1529         $snap->{'snaptime'} = time();
1530         $snap->{'description'} = $comment if $comment;
1531         $conf->{snapshots}->{$snapname} = $snap;
1532
1533         write_config($vmid, $conf);
1534     };
1535
1536     lock_container($vmid, 10, $updatefn);
1537
1538     return $snap;
1539 };
1540
1541 my $snapshot_commit = sub {
1542     my ($vmid, $snapname) = @_;
1543
1544     my $updatefn = sub {
1545
1546         my $conf = load_config($vmid);
1547
1548         die "missing snapshot lock\n"
1549             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1550
1551         die "snapshot '$snapname' does not exist\n"
1552             if !defined($conf->{snapshots}->{$snapname});
1553
1554         die "wrong snapshot state\n"
1555             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1556                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1557
1558         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1559         delete $conf->{lock};
1560         $conf->{parent} = $snapname;
1561
1562         write_config($vmid, $conf);
1563     };
1564
1565     lock_container($vmid, 10 ,$updatefn);
1566 };
1567
1568 sub has_feature {
1569     my ($feature, $conf, $storecfg, $snapname) = @_;
1570
1571     my $err;
1572
1573     foreach_mountpoint($conf, sub {
1574         my ($ms, $mountpoint) = @_;
1575
1576         return if $err; # skip further test
1577
1578         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1579
1580         # TODO: implement support for mountpoints
1581         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1582             if $ms ne 'rootfs';
1583     });
1584
1585     return $err ? 0 : 1;
1586 }
1587
1588 sub snapshot_create {
1589     my ($vmid, $snapname, $comment) = @_;
1590
1591     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1592
1593     my $conf = load_config($vmid);
1594
1595     my $cmd = "/usr/bin/lxc-freeze -n $vmid";
1596     my $running = check_running($vmid);
1597     eval {
1598         if ($running) {
1599             PVE::Tools::run_command($cmd);
1600         };
1601
1602         my $storecfg = PVE::Storage::config();
1603         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1604         my $volid = $rootinfo->{volume};
1605
1606         $cmd = "/usr/bin/lxc-unfreeze -n $vmid";
1607         if ($running) {
1608             PVE::Tools::run_command($cmd);
1609         };
1610
1611         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1612         &$snapshot_commit($vmid, $snapname);
1613     };
1614     if(my $err = $@) {
1615         snapshot_delete($vmid, $snapname, 1);
1616         die "$err\n";
1617     }
1618 }
1619
1620 sub snapshot_delete {
1621     my ($vmid, $snapname, $force) = @_;
1622
1623     my $snap;
1624
1625     my $conf;
1626
1627     my $updatefn =  sub {
1628
1629         $conf = load_config($vmid);
1630
1631         die "you can't delete a snapshot if vm is a template\n"
1632             if is_template($conf);
1633
1634         $snap = $conf->{snapshots}->{$snapname};
1635
1636         check_lock($conf);
1637
1638         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1639
1640         $snap->{snapstate} = 'delete';
1641
1642         write_config($vmid, $conf);
1643     };
1644
1645     lock_container($vmid, 10, $updatefn);
1646
1647     my $storecfg = PVE::Storage::config();
1648
1649     my $del_snap =  sub {
1650
1651         check_lock($conf);
1652
1653         if ($conf->{parent} eq $snapname) {
1654             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1655                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1656             } else {
1657                 delete $conf->{parent};
1658             }
1659         }
1660
1661         delete $conf->{snapshots}->{$snapname};
1662
1663         write_config($vmid, $conf);
1664     };
1665
1666     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1667     my $rootinfo = parse_ct_mountpoint($rootfs);
1668     my $volid = $rootinfo->{volume};
1669
1670     eval {
1671         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1672     };
1673     my $err = $@;
1674
1675     if(!$err || ($err && $force)) {
1676         lock_container($vmid, 10, $del_snap);
1677         if ($err) {
1678             die "Can't delete snapshot: $vmid $snapname $err\n";
1679         }
1680     }
1681 }
1682
1683 sub snapshot_rollback {
1684     my ($vmid, $snapname) = @_;
1685
1686     my $storecfg = PVE::Storage::config();
1687
1688     my $conf = load_config($vmid);
1689
1690     die "you can't rollback if vm is a template\n" if is_template($conf);
1691
1692     my $snap = $conf->{snapshots}->{$snapname};
1693
1694     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1695
1696     my $rootfs = $snap->{rootfs};
1697     my $rootinfo = parse_ct_mountpoint($rootfs);
1698     my $volid = $rootinfo->{volume};
1699
1700     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1701
1702     my $updatefn = sub {
1703
1704         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1705             if $snap->{snapstate};
1706
1707         check_lock($conf);
1708
1709         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1710
1711         die "unable to rollback vm $vmid: vm is running\n"
1712             if check_running($vmid);
1713
1714         $conf->{lock} = 'rollback';
1715
1716         my $forcemachine;
1717
1718         # copy snapshot config to current config
1719
1720         my $tmp_conf = $conf;
1721         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1722         $conf->{snapshots} = $tmp_conf->{snapshots};
1723         delete $conf->{snaptime};
1724         delete $conf->{snapname};
1725         $conf->{parent} = $snapname;
1726
1727         write_config($vmid, $conf);
1728     };
1729
1730     my $unlockfn = sub {
1731         delete $conf->{lock};
1732         write_config($vmid, $conf);
1733     };
1734
1735     lock_container($vmid, 10, $updatefn);
1736
1737     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1738
1739     lock_container($vmid, 5, $unlockfn);
1740 }
1741
1742 sub template_create {
1743     my ($vmid, $conf) = @_;
1744
1745     my $storecfg = PVE::Storage::config();
1746
1747     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1748     my $volid = $rootinfo->{volume};
1749
1750     die "Template feature is not available for '$volid'\n"
1751         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1752
1753     PVE::Storage::activate_volumes($storecfg, [$volid]);
1754
1755     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1756     $rootinfo->{volume} = $template_volid;
1757     $conf->{rootfs} = print_ct_mountpoint($rootinfo);
1758
1759     write_config($vmid, $conf);
1760 }
1761
1762 sub is_template {
1763     my ($conf) = @_;
1764
1765     return 1 if defined $conf->{template} && $conf->{template} == 1;
1766 }
1767
1768 sub mountpoint_names {
1769     my ($reverse) = @_;
1770
1771     my @names = ('rootfs');
1772
1773     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1774         push @names, "mp$i";
1775     }
1776
1777     return $reverse ? reverse @names : @names;
1778 }
1779
1780 sub foreach_mountpoint_full {
1781     my ($conf, $reverse, $func) = @_;
1782
1783     foreach my $key (mountpoint_names($reverse)) {
1784         my $value = $conf->{$key};
1785         next if !defined($value);
1786         my $mountpoint = parse_ct_mountpoint($value);
1787         $mountpoint->{mp} = '/' if $key eq 'rootfs'; # just to be sure
1788         &$func($key, $mountpoint);
1789     }
1790 }
1791
1792 sub foreach_mountpoint {
1793     my ($conf, $func) = @_;
1794
1795     foreach_mountpoint_full($conf, 0, $func);
1796 }
1797
1798 sub foreach_mountpoint_reverse {
1799     my ($conf, $func) = @_;
1800
1801     foreach_mountpoint_full($conf, 1, $func);
1802 }
1803
1804 sub loopdevices_list {
1805
1806     my $loopdev = {};
1807     my $parser = sub {
1808         my $line = shift;
1809         if ($line =~ m/^(\/dev\/loop\d+)\s+\d\s+\d\s+\d\s+\d\s(\S+)$/) {
1810             $loopdev->{$1} = $2;
1811         }
1812     };
1813
1814     PVE::Tools::run_command(['losetup'], outfunc => $parser);
1815
1816     return $loopdev;
1817 }
1818
1819 sub blockdevices_list {
1820
1821     my $bdevs = {};
1822     dir_glob_foreach("/sys/dev/block/", '(\d+):(\d+)', sub {
1823         my (undef, $major, $minor) = @_;
1824         my $bdev = readlink("/sys/dev/block/$major:$minor");
1825         $bdev =~ s/\.\.\/\.\.\/devices\/virtual\/block\//\/dev\//;
1826         $bdevs->{$bdev}->{major} = $major;
1827         $bdevs->{$bdev}->{minor} = $minor;
1828     });
1829     return $bdevs;
1830 }
1831
1832 sub find_loopdev {
1833     my ($loopdevs, $path) = @_;
1834
1835     foreach my $dev (keys %$loopdevs){
1836         return $dev if $loopdevs->{$dev} eq $path;
1837     }
1838 }
1839
1840 sub check_ct_modify_config_perm {
1841     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1842
1843     return 1 if $authuser ne 'root@pam';
1844
1845     foreach my $opt (@$key_list) {
1846
1847         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1848             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1849         } elsif ($opt eq 'disk') {
1850             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1851         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1852             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1853         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1854                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1855             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1856         } else {
1857             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1858         }
1859     }
1860
1861     return 1;
1862 }
1863
1864 sub attach_loops {
1865     my ($storage_cfg, $vollist, $snapname) = @_;
1866
1867     my $loopdevs = {};
1868
1869     foreach my $volid (@$vollist) {
1870
1871         my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
1872         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1873
1874         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1875             PVE::Storage::parse_volname($storage_cfg, $volid);
1876
1877         if ($format eq 'raw' && $scfg->{path}) {
1878             my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
1879             my $loopdev;
1880
1881             my $parser = sub {
1882                 my $line = shift;
1883                 $loopdev = $line if $line =~m|^/dev/loop\d+$|;
1884                 $loopdevs->{$loopdev} = $path;
1885             };
1886
1887             PVE::Tools::run_command(['losetup', '--find', '--show', $path], outfunc => $parser);
1888         }
1889     }
1890
1891     return $loopdevs;
1892 }
1893
1894 sub dettach_loops {
1895     my ($storage_cfg, $vollist, $snapname) = @_;
1896
1897     my $loopdevs = loopdevices_list();
1898
1899     foreach my $volid (@$vollist) {
1900
1901         my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
1902         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1903
1904         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1905             PVE::Storage::parse_volname($storage_cfg, $volid);
1906
1907         if ($format eq 'raw' && $scfg->{path}) {
1908             my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
1909             foreach my $dev (keys %$loopdevs){
1910                 PVE::Tools::run_command(['losetup', '-d', $dev]) if $loopdevs->{$dev} eq $path;
1911             }
1912         }
1913     }
1914 }
1915
1916 sub umount_all {
1917     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
1918
1919     my $loopdevs = loopdevices_list();
1920
1921     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1922     my $volid_list = get_vm_volumes($conf);
1923
1924     foreach_mountpoint_reverse($conf, sub {
1925         my ($ms, $mountpoint) = @_;
1926
1927         my $volid = $mountpoint->{volume};
1928         my $mount = $mountpoint->{mp};
1929
1930         return if !$volid || !$mount;
1931
1932         my $mount_path = "$rootdir/$mount";
1933         $mount_path =~ s!/+!/!g;
1934
1935         # fixme: test if mounted?
1936         eval {
1937             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1938         };
1939         if (my $err = $@) {
1940             if ($noerr) {
1941                 warn $err;
1942             } else {
1943                 die $err;
1944             }
1945         }
1946     });
1947
1948     PVE::LXC::dettach_loops($storage_cfg, $volid_list);
1949 }
1950
1951 sub mount_all {
1952     my ($vmid, $storage_cfg, $conf, $format_raw_images) = @_;
1953
1954     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1955
1956     my $volid_list = get_vm_volumes($conf);
1957     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
1958
1959     eval {
1960         my $loopdevs = attach_loops($storage_cfg, $volid_list);
1961
1962         foreach_mountpoint($conf, sub {
1963             my ($ms, $mountpoint) = @_;
1964
1965             my $volid = $mountpoint->{volume};
1966             my $mount = $mountpoint->{mp};
1967
1968             return if !$volid || !$mount;
1969
1970             my $image_path = PVE::Storage::path($storage_cfg, $volid);
1971             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1972                 PVE::Storage::parse_volname($storage_cfg, $volid);
1973
1974             die "unable to mount base volume - internal error" if $isBase;
1975
1976             if ($format_raw_images && $format eq 'raw') {
1977                 my $cmd = ['mkfs.ext4', '-O', 'mmp', $image_path];
1978                 PVE::Tools::run_command($cmd);
1979             }
1980
1981             mountpoint_mount($mountpoint, $rootdir, $storage_cfg, $loopdevs);
1982         });
1983     };
1984     if (my $err = $@) {
1985         warn "mounting container failed - $err";
1986         umount_all($vmid, $storage_cfg, $conf, 1);
1987     }
1988
1989     return $rootdir;
1990 }
1991
1992
1993 sub mountpoint_mount_path {
1994     my ($mountpoint, $storage_cfg, $loopdevs, $snapname) = @_;
1995
1996     return mountpoint_mount($mountpoint, undef, $storage_cfg, $loopdevs, $snapname);
1997 }
1998
1999 # use $rootdir = undef to just return the corresponding mount path
2000 sub mountpoint_mount {
2001     my ($mountpoint, $rootdir, $storage_cfg, $loopdevs, $snapname) = @_;
2002
2003     my $volid = $mountpoint->{volume};
2004     my $mount = $mountpoint->{mp};
2005
2006     return if !$volid || !$mount;
2007
2008     my $mount_path;
2009
2010     if (defined($rootdir)) {
2011         $rootdir =~ s!/+$!!;
2012         $mount_path = "$rootdir/$mount";
2013         $mount_path =~ s!/+!/!g;
2014         File::Path::mkpath($mount_path);
2015     }
2016
2017     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2018
2019     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2020
2021     if ($storage) {
2022
2023         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2024         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2025
2026         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2027             PVE::Storage::parse_volname($storage_cfg, $volid);
2028
2029         if ($format eq 'subvol') {
2030
2031             if ($mount_path) {
2032                 if ($snapname) {
2033                     if ($scfg->{type} eq 'zfspool') {
2034                         my $path_arg = $path;
2035                         $path_arg =~ s!^/+!!;
2036                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2037                     } else {
2038                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2039                     }
2040                 } else {
2041                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2042                 }
2043             }
2044             return $path;
2045
2046         } elsif ($format eq 'raw') {
2047
2048             if ($scfg->{path}) {
2049                 $path = find_loopdev($loopdevs, $path) if $loopdevs;
2050             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'rbd') {
2051                 # do nothing
2052             } else {
2053                 die "unsupported storage type '$scfg->{type}'\n";
2054             }
2055             if ($mount_path) {
2056                 if ($isBase || defined($snapname)) {
2057                     PVE::Tools::run_command(['mount', '-o', 'ro', $path, $mount_path]);
2058                 } else {
2059                     PVE::Tools::run_command(['mount', $path, $mount_path]);
2060                 }
2061             }
2062             return $path;
2063         } else {
2064             die "unsupported image format '$format'\n";
2065         }
2066     } elsif ($volid =~ m|^/dev/.+|) {
2067         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2068         return $volid;
2069     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2070         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2071         return $volid;
2072     }
2073
2074     die "unsupported storage";
2075 }
2076
2077 sub get_vm_volumes {
2078     my ($conf, $excludes) = @_;
2079
2080     my $vollist = [];
2081
2082     foreach_mountpoint($conf, sub {
2083         my ($ms, $mountpoint) = @_;
2084
2085         return if $excludes && $ms eq $excludes;
2086
2087         my $volid = $mountpoint->{volume};
2088
2089         return if !$volid || $volid =~ m|^/|;
2090
2091         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2092         return if !$sid;
2093
2094         push @$vollist, $volid;
2095     });
2096
2097     return $vollist;
2098 }
2099
2100 # bash completion helper
2101
2102 sub complete_os_templates {
2103     my ($cmdname, $pname, $cvalue) = @_;
2104
2105     my $cfg = PVE::Storage::config();
2106
2107      my $storeid;
2108
2109     if ($cvalue =~ m/^([^:]+):/) {
2110         $storeid = $1;
2111     }
2112
2113     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2114     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2115
2116     my $res = [];
2117     foreach my $id (keys %$data) {
2118         foreach my $item (@{$data->{$id}}) {
2119             push @$res, $item->{volid} if defined($item->{volid});
2120         }
2121     }
2122
2123     return $res;
2124 }
2125
2126 sub complete_migration_target {
2127
2128     my $res = [];
2129
2130     my $nodelist = PVE::Cluster::get_nodelist();
2131     foreach my $node (@$nodelist) {
2132         next if $node eq $nodename;
2133         push @$res, $node;
2134     }
2135
2136     return $res;
2137 }
2138
2139 sub complete_next_vmid {
2140
2141     my $vmlist = PVE::Cluster::get_vmlist() || {};
2142     my $idlist = $vmlist->{ids} || {};
2143
2144     for (my $i = 100; $i < 10000; $i++) {
2145         return [$i] if !defined($idlist->{$i});
2146     }
2147
2148     return [];
2149 }
2150
2151 my $complete_ctid_full = sub {
2152     my ($running) = @_;
2153
2154     my $idlist = vmstatus();
2155
2156     my $active_hash = list_active_containers();
2157
2158     my $res = [];
2159
2160     foreach my $id (keys %$idlist) {
2161         my $d = $idlist->{$id};
2162         if (defined($running)) {
2163             next if $d->{template};
2164             next if $running && !$active_hash->{$id};
2165             next if !$running && $active_hash->{$id};
2166         }
2167         push @$res, $id;
2168
2169     }
2170     return $res;
2171 };
2172
2173 sub complete_ctid {
2174     return &$complete_ctid_full();
2175 }
2176
2177 sub complete_ctid_stopped {
2178     return &$complete_ctid_full(0);
2179 }
2180
2181 sub complete_ctid_running {
2182     return &$complete_ctid_full(1);
2183 }
2184
2185 1;