src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use Fcntl ':flock';
   9
  10 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  11 use PVE::Storage;
  12 use PVE::SafeSyslog;
  13 use PVE::INotify;
  14 use PVE::JSONSchema qw(get_standard_option);
  15 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  16 use PVE::Network;
  17 use PVE::AccessControl;
  18
  19 use Data::Dumper;
  20
  21 my $nodename = PVE::INotify::nodename();
  22
  23 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  24
  25 PVE::JSONSchema::register_format('pve-lxc-network', \&verify_lxc_network);
  26 sub verify_lxc_network {
  27     my ($value, $noerr) = @_;
  28
  29     return $value if parse_lxc_network($value);
  30
  31     return undef if $noerr;
  32
  33     die "unable to parse network setting\n";
  34 }
  35
  36 PVE::JSONSchema::register_format('pve-ct-mountpoint', \&verify_ct_mountpoint);
  37 sub verify_ct_mountpoint {
  38     my ($value, $noerr) = @_;
  39
  40     return $value if parse_ct_mountpoint($value);
  41
  42     return undef if $noerr;
  43
  44     die "unable to parse CT mountpoint options\n";
  45 }
  46
  47 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  48     type => 'string', format => 'pve-ct-mountpoint',
  49     typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+]',
  50     description => "Use volume as container root.",
  51     optional => 1,
  52 });
  53
  54 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  55     description => "The name of the snapshot.",
  56     type => 'string', format => 'pve-configid',
  57     maxLength => 40,
  58 });
  59
  60 my $confdesc = {
  61     lock => {
  62         optional => 1,
  63         type => 'string',
  64         description => "Lock/unlock the VM.",
  65         enum => [qw(migrate backup snapshot rollback)],
  66     },
  67     onboot => {
  68         optional => 1,
  69         type => 'boolean',
  70         description => "Specifies whether a VM will be started during system bootup.",
  71         default => 0,
  72     },
  73     startup => get_standard_option('pve-startup-order'),
  74     template => {
  75         optional => 1,
  76         type => 'boolean',
  77         description => "Enable/disable Template.",
  78         default => 0,
  79     },
  80     arch => {
  81         optional => 1,
  82         type => 'string',
  83         enum => ['amd64', 'i386'],
  84         description => "OS architecture type.",
  85         default => 'amd64',
  86     },
  87     ostype => {
  88         optional => 1,
  89         type => 'string',
  90         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  91         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  92     },
  93     console => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Attach a console device (/dev/console) to the container.",
  97         default => 1,
  98     },
  99     tty => {
 100         optional => 1,
 101         type => 'integer',
 102         description => "Specify the number of tty available to the container",
 103         minimum => 0,
 104         maximum => 6,
 105         default => 2,
 106     },
 107     cpulimit => {
 108         optional => 1,
 109         type => 'number',
 110         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 111         minimum => 0,
 112         maximum => 128,
 113         default => 0,
 114     },
 115     cpuunits => {
 116         optional => 1,
 117         type => 'integer',
 118         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 119         minimum => 0,
 120         maximum => 500000,
 121         default => 1024,
 122     },
 123     memory => {
 124         optional => 1,
 125         type => 'integer',
 126         description => "Amount of RAM for the VM in MB.",
 127         minimum => 16,
 128         default => 512,
 129     },
 130     swap => {
 131         optional => 1,
 132         type => 'integer',
 133         description => "Amount of SWAP for the VM in MB.",
 134         minimum => 0,
 135         default => 512,
 136     },
 137     hostname => {
 138         optional => 1,
 139         description => "Set a host name for the container.",
 140         type => 'string',
 141         maxLength => 255,
 142     },
 143     description => {
 144         optional => 1,
 145         type => 'string',
 146         description => "Container description. Only used on the configuration web interface.",
 147     },
 148     searchdomain => {
 149         optional => 1,
 150         type => 'string',
 151         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 152     },
 153     nameserver => {
 154         optional => 1,
 155         type => 'string',
 156         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 157     },
 158     rootfs => get_standard_option('pve-ct-rootfs'),
 159     parent => {
 160         optional => 1,
 161         type => 'string', format => 'pve-configid',
 162         maxLength => 40,
 163         description => "Parent snapshot name. This is used internally, and should not be modified.",
 164     },
 165     snaptime => {
 166         optional => 1,
 167         description => "Timestamp for snapshots.",
 168         type => 'integer',
 169         minimum => 0,
 170     },
 171     cmode => {
 172         optional => 1,
 173         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 174         type => 'string',
 175         enum => ['shell', 'console', 'tty'],
 176         default => 'tty',
 177     },
 178 };
 179
 180 my $valid_lxc_conf_keys = {
 181     'lxc.include' => 1,
 182     'lxc.arch' => 1,
 183     'lxc.utsname' => 1,
 184     'lxc.haltsignal' => 1,
 185     'lxc.rebootsignal' => 1,
 186     'lxc.stopsignal' => 1,
 187     'lxc.init_cmd' => 1,
 188     'lxc.network.type' => 1,
 189     'lxc.network.flags' => 1,
 190     'lxc.network.link' => 1,
 191     'lxc.network.mtu' => 1,
 192     'lxc.network.name' => 1,
 193     'lxc.network.hwaddr' => 1,
 194     'lxc.network.ipv4' => 1,
 195     'lxc.network.ipv4.gateway' => 1,
 196     'lxc.network.ipv6' => 1,
 197     'lxc.network.ipv6.gateway' => 1,
 198     'lxc.network.script.up' => 1,
 199     'lxc.network.script.down' => 1,
 200     'lxc.pts' => 1,
 201     'lxc.console.logfile' => 1,
 202     'lxc.console' => 1,
 203     'lxc.tty' => 1,
 204     'lxc.devttydir' => 1,
 205     'lxc.hook.autodev' => 1,
 206     'lxc.autodev' => 1,
 207     'lxc.kmsg' => 1,
 208     'lxc.mount' => 1,
 209     'lxc.mount.entry' => 1,
 210     'lxc.mount.auto' => 1,
 211     'lxc.rootfs' => 1,
 212     'lxc.rootfs.mount' => 1,
 213     'lxc.rootfs.options' => 1,
 214     # lxc.cgroup.*
 215     'lxc.cap.drop' => 1,
 216     'lxc.cap.keep' => 1,
 217     'lxc.aa_profile' => 1,
 218     'lxc.aa_allow_incomplete' => 1,
 219     'lxc.se_context' => 1,
 220     'lxc.seccomp' => 1,
 221     'lxc.id_map' => 1,
 222     'lxc.hook.pre-start' => 1,
 223     'lxc.hook.pre-mount' => 1,
 224     'lxc.hook.mount' => 1,
 225     'lxc.hook.start' => 1,
 226     'lxc.hook.post-stop' => 1,
 227     'lxc.hook.clone' => 1,
 228     'lxc.hook.destroy' => 1,
 229     'lxc.loglevel' => 1,
 230     'lxc.logfile' => 1,
 231     'lxc.start.auto' => 1,
 232     'lxc.start.delay' => 1,
 233     'lxc.start.order' => 1,
 234     'lxc.group' => 1,
 235     'lxc.environment' => 1,
 236     'lxc.' => 1,
 237     'lxc.' => 1,
 238     'lxc.' => 1,
 239     'lxc.' => 1,
 240 };
 241
 242 my $MAX_LXC_NETWORKS = 10;
 243 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 244     $confdesc->{"net$i"} = {
 245         optional => 1,
 246         type => 'string', format => 'pve-lxc-network',
 247         description => "Specifies network interfaces for the container.\n\n".
 248             "The string should have the follow format:\n\n".
 249             "-net<[0-9]> bridge=<vmbr<Nummber>>[,hwaddr=<MAC>]\n".
 250             "[,mtu=<Number>][,name=<String>][,ip=<IPv4Format/CIDR>]\n".
 251             ",ip6=<IPv6Format/CIDR>][,gw=<GatwayIPv4>]\n".
 252             ",gw6=<GatwayIPv6>][,firewall=<[1|0]>][,tag=<VlanNo>]",
 253     };
 254 }
 255
 256 my $MAX_MOUNT_POINTS = 10;
 257 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 258     $confdesc->{"mp$i"} = {
 259         optional => 1,
 260         type => 'string', format => 'pve-ct-mountpoint',
 261         typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+] [,mp=mountpoint]',
 262         description => "Use volume as container mount point (experimental feature).",
 263         optional => 1,
 264     };
 265 }
 266
 267 sub write_pct_config {
 268     my ($filename, $conf) = @_;
 269
 270     delete $conf->{snapstate}; # just to be sure
 271
 272     my $generate_raw_config = sub {
 273         my ($conf) = @_;
 274
 275         my $raw = '';
 276
 277         # add description as comment to top of file
 278         my $descr = $conf->{description} || '';
 279         foreach my $cl (split(/\n/, $descr)) {
 280             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 281         }
 282
 283         foreach my $key (sort keys %$conf) {
 284             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 285                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 286             $raw .= "$key: $conf->{$key}\n";
 287         }
 288
 289         if (my $lxcconf = $conf->{lxc}) {
 290             foreach my $entry (@$lxcconf) {
 291                 my ($k, $v) = @$entry;
 292                 $raw .= "$k: $v\n";
 293             }
 294         }
 295
 296         return $raw;
 297     };
 298
 299     my $raw = &$generate_raw_config($conf);
 300
 301     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 302         $raw .= "\n[$snapname]\n";
 303         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 304     }
 305
 306     return $raw;
 307 }
 308
 309 sub check_type {
 310     my ($key, $value) = @_;
 311
 312     die "unknown setting '$key'\n" if !$confdesc->{$key};
 313
 314     my $type = $confdesc->{$key}->{type};
 315
 316     if (!defined($value)) {
 317         die "got undefined value\n";
 318     }
 319
 320     if ($value =~ m/[\n\r]/) {
 321         die "property contains a line feed\n";
 322     }
 323
 324     if ($type eq 'boolean') {
 325         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 326         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 327         die "type check ('boolean') failed - got '$value'\n";
 328     } elsif ($type eq 'integer') {
 329         return int($1) if $value =~ m/^(\d+)$/;
 330         die "type check ('integer') failed - got '$value'\n";
 331     } elsif ($type eq 'number') {
 332         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 333         die "type check ('number') failed - got '$value'\n";
 334     } elsif ($type eq 'string') {
 335         if (my $fmt = $confdesc->{$key}->{format}) {
 336             PVE::JSONSchema::check_format($fmt, $value);
 337             return $value;
 338         }
 339         return $value;
 340     } else {
 341         die "internal error"
 342     }
 343 }
 344
 345 sub parse_pct_config {
 346     my ($filename, $raw) = @_;
 347
 348     return undef if !defined($raw);
 349
 350     my $res = {
 351         digest => Digest::SHA::sha1_hex($raw),
 352         snapshots => {},
 353     };
 354
 355     $filename =~ m|/lxc/(\d+).conf$|
 356         || die "got strange filename '$filename'";
 357
 358     my $vmid = $1;
 359
 360     my $conf = $res;
 361     my $descr = '';
 362     my $section = '';
 363
 364     my @lines = split(/\n/, $raw);
 365     foreach my $line (@lines) {
 366         next if $line =~ m/^\s*$/;
 367
 368         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 369             $section = $1;
 370             $conf->{description} = $descr if $descr;
 371             $descr = '';
 372             $conf = $res->{snapshots}->{$section} = {};
 373             next;
 374         }
 375
 376         if ($line =~ m/^\#(.*)\s*$/) {
 377             $descr .= PVE::Tools::decode_text($1) . "\n";
 378             next;
 379         }
 380
 381         if ($line =~ m/^(lxc\.[a-z0-9_\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 382             my $key = $1;
 383             my $value = $3;
 384             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 385                 push @{$conf->{lxc}}, [$key, $value];
 386             } else {
 387                 warn "vm $vmid - unable to parse config: $line\n";
 388             }
 389         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 390             $descr .= PVE::Tools::decode_text($2);
 391         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 392             $conf->{snapstate} = $1;
 393         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
 394             my $key = $1;
 395             my $value = $2;
 396             eval { $value = check_type($key, $value); };
 397             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 398             $conf->{$key} = $value;
 399         } else {
 400             warn "vm $vmid - unable to parse config: $line\n";
 401         }
 402     }
 403
 404     $conf->{description} = $descr if $descr;
 405
 406     delete $res->{snapstate}; # just to be sure
 407
 408     return $res;
 409 }
 410
 411 sub config_list {
 412     my $vmlist = PVE::Cluster::get_vmlist();
 413     my $res = {};
 414     return $res if !$vmlist || !$vmlist->{ids};
 415     my $ids = $vmlist->{ids};
 416
 417     foreach my $vmid (keys %$ids) {
 418         next if !$vmid; # skip CT0
 419         my $d = $ids->{$vmid};
 420         next if !$d->{node} || $d->{node} ne $nodename;
 421         next if !$d->{type} || $d->{type} ne 'lxc';
 422         $res->{$vmid}->{type} = 'lxc';
 423     }
 424     return $res;
 425 }
 426
 427 sub cfs_config_path {
 428     my ($vmid, $node) = @_;
 429
 430     $node = $nodename if !$node;
 431     return "nodes/$node/lxc/$vmid.conf";
 432 }
 433
 434 sub config_file {
 435     my ($vmid, $node) = @_;
 436
 437     my $cfspath = cfs_config_path($vmid, $node);
 438     return "/etc/pve/$cfspath";
 439 }
 440
 441 sub load_config {
 442     my ($vmid, $node) = @_;
 443
 444     $node = $nodename if !$node;
 445     my $cfspath = cfs_config_path($vmid, $node);
 446
 447     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 448     die "container $vmid does not exists\n" if !defined($conf);
 449
 450     return $conf;
 451 }
 452
 453 sub create_config {
 454     my ($vmid, $conf) = @_;
 455
 456     my $dir = "/etc/pve/nodes/$nodename/lxc";
 457     mkdir $dir;
 458
 459     write_config($vmid, $conf);
 460 }
 461
 462 sub destroy_config {
 463     my ($vmid) = @_;
 464
 465     unlink config_file($vmid, $nodename);
 466 }
 467
 468 sub write_config {
 469     my ($vmid, $conf) = @_;
 470
 471     my $cfspath = cfs_config_path($vmid);
 472
 473     PVE::Cluster::cfs_write_file($cfspath, $conf);
 474 }
 475
 476 # flock: we use one file handle per process, so lock file
 477 # can be called multiple times and succeeds for the same process.
 478
 479 my $lock_handles =  {};
 480 my $lockdir = "/run/lock/lxc";
 481
 482 sub lock_filename {
 483     my ($vmid) = @_;
 484
 485     return "$lockdir/pve-config-${vmid}.lock";
 486 }
 487
 488 sub lock_aquire {
 489     my ($vmid, $timeout) = @_;
 490
 491     $timeout = 10 if !$timeout;
 492     my $mode = LOCK_EX;
 493
 494     my $filename = lock_filename($vmid);
 495
 496     mkdir $lockdir if !-d $lockdir;
 497
 498     my $lock_func = sub {
 499         if (!$lock_handles->{$$}->{$filename}) {
 500             my $fh = new IO::File(">>$filename") ||
 501                 die "can't open file - $!\n";
 502             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 503         }
 504
 505         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 506             print STDERR "trying to aquire lock...";
 507             my $success;
 508             while(1) {
 509                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 510                 # try again on EINTR (see bug #273)
 511                 if ($success || ($! != EINTR)) {
 512                     last;
 513                 }
 514             }
 515             if (!$success) {
 516                 print STDERR " failed\n";
 517                 die "can't aquire lock - $!\n";
 518             }
 519
 520             print STDERR " OK\n";
 521         }
 522
 523         $lock_handles->{$$}->{$filename}->{refcount}++;
 524     };
 525
 526     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 527     my $err = $@;
 528     if ($err) {
 529         die "can't lock file '$filename' - $err";
 530     }
 531 }
 532
 533 sub lock_release {
 534     my ($vmid) = @_;
 535
 536     my $filename = lock_filename($vmid);
 537
 538     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 539         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 540         if ($refcount <= 0) {
 541             $lock_handles->{$$}->{$filename} = undef;
 542             close ($fh);
 543         }
 544     }
 545 }
 546
 547 sub lock_container {
 548     my ($vmid, $timeout, $code, @param) = @_;
 549
 550     my $res;
 551
 552     lock_aquire($vmid, $timeout);
 553     eval { $res = &$code(@param) };
 554     my $err = $@;
 555     lock_release($vmid);
 556
 557     die $err if $err;
 558
 559     return $res;
 560 }
 561
 562 sub option_exists {
 563     my ($name) = @_;
 564
 565     return defined($confdesc->{$name});
 566 }
 567
 568 # add JSON properties for create and set function
 569 sub json_config_properties {
 570     my $prop = shift;
 571
 572     foreach my $opt (keys %$confdesc) {
 573         next if $opt eq 'parent' || $opt eq 'snaptime';
 574         next if $prop->{$opt};
 575         $prop->{$opt} = $confdesc->{$opt};
 576     }
 577
 578     return $prop;
 579 }
 580
 581 sub json_config_properties_no_rootfs {
 582     my $prop = shift;
 583
 584     foreach my $opt (keys %$confdesc) {
 585         next if $prop->{$opt};
 586         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 587         $prop->{$opt} = $confdesc->{$opt};
 588     }
 589
 590     return $prop;
 591 }
 592
 593 # container status helpers
 594
 595 sub list_active_containers {
 596
 597     my $filename = "/proc/net/unix";
 598
 599     # similar test is used by lcxcontainers.c: list_active_containers
 600     my $res = {};
 601
 602     my $fh = IO::File->new ($filename, "r");
 603     return $res if !$fh;
 604
 605     while (defined(my $line = <$fh>)) {
 606         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 607             my $path = $1;
 608             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 609                 $res->{$1} = 1;
 610             }
 611         }
 612     }
 613
 614     close($fh);
 615
 616     return $res;
 617 }
 618
 619 # warning: this is slow
 620 sub check_running {
 621     my ($vmid) = @_;
 622
 623     my $active_hash = list_active_containers();
 624
 625     return 1 if defined($active_hash->{$vmid});
 626
 627     return undef;
 628 }
 629
 630 sub get_container_disk_usage {
 631     my ($vmid) = @_;
 632
 633     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 634
 635     my $res = {
 636         total => 0,
 637         used => 0,
 638         avail => 0,
 639     };
 640
 641     my $parser = sub {
 642         my $line = shift;
 643         if (my ($fsid, $total, $used, $avail) = $line =~
 644             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 645             $res = {
 646                 total => $total,
 647                 used => $used,
 648                 avail => $avail,
 649             };
 650         }
 651     };
 652     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 653     warn $@ if $@;
 654
 655     return $res;
 656 }
 657
 658 sub vmstatus {
 659     my ($opt_vmid) = @_;
 660
 661     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 662
 663     my $active_hash = list_active_containers();
 664
 665     foreach my $vmid (keys %$list) {
 666         my $d = $list->{$vmid};
 667
 668         my $running = defined($active_hash->{$vmid});
 669
 670         $d->{status} = $running ? 'running' : 'stopped';
 671
 672         my $cfspath = cfs_config_path($vmid);
 673         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 674
 675         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 676         $d->{name} =~ s/[\s]//g;
 677
 678         $d->{cpus} = $conf->{cpulimit} // 0;
 679
 680         if ($running) {
 681             my $res = get_container_disk_usage($vmid);
 682             $d->{disk} = $res->{used};
 683             $d->{maxdisk} = $res->{total};
 684         } else {
 685             $d->{disk} = 0;
 686             # use 4GB by default ??
 687             if (my $rootfs = $conf->{rootfs}) {
 688                 my $rootinfo = parse_ct_mountpoint($rootfs);
 689                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 690             } else {
 691                 $d->{maxdisk} = 4*1024*1024*1024;
 692             }
 693         }
 694
 695         $d->{mem} = 0;
 696         $d->{swap} = 0;
 697         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 698         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 699
 700         $d->{uptime} = 0;
 701         $d->{cpu} = 0;
 702
 703         $d->{netout} = 0;
 704         $d->{netin} = 0;
 705
 706         $d->{diskread} = 0;
 707         $d->{diskwrite} = 0;
 708
 709         $d->{template} = is_template($conf);
 710     }
 711
 712     foreach my $vmid (keys %$list) {
 713         my $d = $list->{$vmid};
 714         next if $d->{status} ne 'running';
 715
 716         $d->{uptime} = 100; # fixme:
 717
 718         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 719         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 720
 721         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 722         my @bytes = split(/\n/, $blkio_bytes);
 723         foreach my $byte (@bytes) {
 724             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 725                 $d->{diskread} = $2 if $key eq 'Read';
 726                 $d->{diskwrite} = $2 if $key eq 'Write';
 727             }
 728         }
 729     }
 730
 731     return $list;
 732 }
 733
 734 my $parse_size = sub {
 735     my ($value) = @_;
 736
 737     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 738     my ($size, $unit) = ($1, $3);
 739     if ($unit) {
 740         if ($unit eq 'K') {
 741             $size = $size * 1024;
 742         } elsif ($unit eq 'M') {
 743             $size = $size * 1024 * 1024;
 744         } elsif ($unit eq 'G') {
 745             $size = $size * 1024 * 1024 * 1024;
 746         }
 747     }
 748     return int($size);
 749 };
 750
 751 sub parse_ct_mountpoint {
 752     my ($data) = @_;
 753
 754     $data //= '';
 755
 756     my $res = {};
 757
 758     foreach my $p (split (/,/, $data)) {
 759         next if $p =~ m/^\s*$/;
 760
 761         if ($p =~ m/^(volume|backup|size|mp)=(.+)$/) {
 762             my ($k, $v) = ($1, $2);
 763             return undef if defined($res->{$k});
 764             $res->{$k} = $v;
 765         } else {
 766             if (!$res->{volume} && $p !~ m/=/) {
 767                 $res->{volume} = $p;
 768             } else {
 769                 return undef;
 770             }
 771         }
 772     }
 773
 774     return undef if !defined($res->{volume});
 775
 776     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 777
 778     if ($res->{size}) {
 779         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 780     }
 781
 782     return $res;
 783 }
 784
 785 sub print_ct_mountpoint {
 786     my ($info, $nomp) = @_;
 787
 788     my $opts = '';
 789
 790     die "missing volume\n" if !$info->{volume};
 791
 792     foreach my $o ('size', 'backup') {
 793         $opts .= ",$o=$info->{$o}" if defined($info->{$o});
 794     }
 795     $opts .= ",mp=$info->{mp}" if !$nomp;
 796
 797     return "$info->{volume}$opts";
 798 }
 799
 800 sub print_lxc_network {
 801     my $net = shift;
 802
 803     die "no network name defined\n" if !$net->{name};
 804
 805     my $res = "name=$net->{name}";
 806
 807     foreach my $k (qw(hwaddr mtu bridge ip gw ip6 gw6 firewall tag)) {
 808         next if !defined($net->{$k});
 809         $res .= ",$k=$net->{$k}";
 810     }
 811
 812     return $res;
 813 }
 814
 815 sub parse_lxc_network {
 816     my ($data) = @_;
 817
 818     my $res = {};
 819
 820     return $res if !$data;
 821
 822     foreach my $pv (split (/,/, $data)) {
 823         if ($pv =~ m/^(bridge|hwaddr|mtu|name|ip|ip6|gw|gw6|firewall|tag)=(\S+)$/) {
 824             $res->{$1} = $2;
 825         } else {
 826             return undef;
 827         }
 828     }
 829
 830     $res->{type} = 'veth';
 831     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 832
 833     return $res;
 834 }
 835
 836 sub read_cgroup_value {
 837     my ($group, $vmid, $name, $full) = @_;
 838
 839     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 840
 841     return PVE::Tools::file_get_contents($path) if $full;
 842
 843     return PVE::Tools::file_read_firstline($path);
 844 }
 845
 846 sub write_cgroup_value {
 847    my ($group, $vmid, $name, $value) = @_;
 848
 849    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 850    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 851
 852 }
 853
 854 sub find_lxc_console_pids {
 855
 856     my $res = {};
 857
 858     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 859         my ($pid) = @_;
 860
 861         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 862         return if !$cmdline;
 863
 864         my @args = split(/\0/, $cmdline);
 865
 866         # serach for lxc-console -n <vmid>
 867         return if scalar(@args) != 3;
 868         return if $args[1] ne '-n';
 869         return if $args[2] !~ m/^\d+$/;
 870         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 871
 872         my $vmid = $args[2];
 873
 874         push @{$res->{$vmid}}, $pid;
 875     });
 876
 877     return $res;
 878 }
 879
 880 sub find_lxc_pid {
 881     my ($vmid) = @_;
 882
 883     my $pid = undef;
 884     my $parser = sub {
 885         my $line = shift;
 886         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 887     };
 888     PVE::Tools::run_command(['lxc-info', '-n', $vmid], outfunc => $parser);
 889
 890     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 891
 892     return $pid;
 893 }
 894
 895 my $ipv4_reverse_mask = [
 896     '0.0.0.0',
 897     '128.0.0.0',
 898     '192.0.0.0',
 899     '224.0.0.0',
 900     '240.0.0.0',
 901     '248.0.0.0',
 902     '252.0.0.0',
 903     '254.0.0.0',
 904     '255.0.0.0',
 905     '255.128.0.0',
 906     '255.192.0.0',
 907     '255.224.0.0',
 908     '255.240.0.0',
 909     '255.248.0.0',
 910     '255.252.0.0',
 911     '255.254.0.0',
 912     '255.255.0.0',
 913     '255.255.128.0',
 914     '255.255.192.0',
 915     '255.255.224.0',
 916     '255.255.240.0',
 917     '255.255.248.0',
 918     '255.255.252.0',
 919     '255.255.254.0',
 920     '255.255.255.0',
 921     '255.255.255.128',
 922     '255.255.255.192',
 923     '255.255.255.224',
 924     '255.255.255.240',
 925     '255.255.255.248',
 926     '255.255.255.252',
 927     '255.255.255.254',
 928     '255.255.255.255',
 929 ];
 930
 931 # Note: we cannot use Net:IP, because that only allows strict
 932 # CIDR networks
 933 sub parse_ipv4_cidr {
 934     my ($cidr, $noerr) = @_;
 935
 936     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
 937         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
 938     }
 939
 940     return undef if $noerr;
 941
 942     die "unable to parse ipv4 address/mask\n";
 943 }
 944
 945 sub check_lock {
 946     my ($conf) = @_;
 947
 948     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
 949 }
 950
 951 sub update_lxc_config {
 952     my ($storage_cfg, $vmid, $conf) = @_;
 953
 954     my $dir = "/var/lib/lxc/$vmid";
 955
 956     if ($conf->{template}) {
 957
 958         unlink "$dir/config";
 959
 960         return;
 961     }
 962
 963     my $raw = '';
 964
 965     die "missing 'arch' - internal error" if !$conf->{arch};
 966     $raw .= "lxc.arch = $conf->{arch}\n";
 967
 968     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
 969     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
 970         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
 971     } else {
 972         die "implement me";
 973     }
 974
 975     if (!has_dev_console($conf)) {
 976         $raw .= "lxc.console = none\n";
 977         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
 978     }
 979
 980     my $ttycount = get_tty_count($conf);
 981     $raw .= "lxc.tty = $ttycount\n";
 982
 983     my $utsname = $conf->{hostname} || "CT$vmid";
 984     $raw .= "lxc.utsname = $utsname\n";
 985
 986     my $memory = $conf->{memory} || 512;
 987     my $swap = $conf->{swap} // 0;
 988
 989     my $lxcmem = int($memory*1024*1024);
 990     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
 991
 992     my $lxcswap = int(($memory + $swap)*1024*1024);
 993     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
 994
 995     if (my $cpulimit = $conf->{cpulimit}) {
 996         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
 997         my $value = int(100000*$cpulimit);
 998         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
 999     }
1000
1001     my $shares = $conf->{cpuunits} || 1024;
1002     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1003
1004     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1005     $mountpoint->{mp} = '/';
1006     my $volid = $mountpoint->{volume};
1007     my $path = mountpoint_mount_path($mountpoint, $storage_cfg);
1008
1009     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1010
1011     if ($storage) {
1012         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1013         $path = "loop:$path" if $scfg->{path};
1014     }
1015
1016     $raw .= "lxc.rootfs = $path\n";
1017
1018     my $netcount = 0;
1019     foreach my $k (keys %$conf) {
1020         next if $k !~ m/^net(\d+)$/;
1021         my $ind = $1;
1022         my $d = parse_lxc_network($conf->{$k});
1023         $netcount++;
1024         $raw .= "lxc.network.type = veth\n";
1025         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1026         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1027         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1028         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1029     }
1030
1031     if (my $lxcconf = $conf->{lxc}) {
1032         foreach my $entry (@$lxcconf) {
1033             my ($k, $v) = @$entry;
1034             $netcount++ if $k eq 'lxc.network.type';
1035             $raw .= "$k = $v\n";
1036         }
1037     }
1038
1039     $raw .= "lxc.network.type = empty\n" if !$netcount;
1040
1041     File::Path::mkpath("$dir/rootfs");
1042
1043     PVE::Tools::file_set_contents("$dir/config", $raw);
1044 }
1045
1046 # verify and cleanup nameserver list (replace \0 with ' ')
1047 sub verify_nameserver_list {
1048     my ($nameserver_list) = @_;
1049
1050     my @list = ();
1051     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1052         PVE::JSONSchema::pve_verify_ip($server);
1053         push @list, $server;
1054     }
1055
1056     return join(' ', @list);
1057 }
1058
1059 sub verify_searchdomain_list {
1060     my ($searchdomain_list) = @_;
1061
1062     my @list = ();
1063     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1064         # todo: should we add checks for valid dns domains?
1065         push @list, $server;
1066     }
1067
1068     return join(' ', @list);
1069 }
1070
1071 sub update_pct_config {
1072     my ($vmid, $conf, $running, $param, $delete) = @_;
1073
1074     my @nohotplug;
1075
1076     my $new_disks = [];
1077
1078     my $rootdir;
1079     if ($running) {
1080         my $pid = find_lxc_pid($vmid);
1081         $rootdir = "/proc/$pid/root";
1082     }
1083
1084     if (defined($delete)) {
1085         foreach my $opt (@$delete) {
1086             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1087                 die "unable to delete required option '$opt'\n";
1088             } elsif ($opt eq 'swap') {
1089                 delete $conf->{$opt};
1090                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1091             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1092                 delete $conf->{$opt};
1093             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1094                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1095                 delete $conf->{$opt};
1096                 push @nohotplug, $opt;
1097                 next if $running;
1098             } elsif ($opt =~ m/^net(\d)$/) {
1099                 delete $conf->{$opt};
1100                 next if !$running;
1101                 my $netid = $1;
1102                 PVE::Network::veth_delete("veth${vmid}i$netid");
1103             } elsif ($opt =~ m/^mp(\d+)$/) {
1104                 delete $conf->{$opt};
1105                 push @nohotplug, $opt;
1106                 next if $running;
1107             } elsif ($opt eq 'rootfs') {
1108                 die "implement me"
1109             } else {
1110                 die "implement me"
1111             }
1112             write_config($vmid, $conf) if $running;
1113         }
1114     }
1115
1116     # There's no separate swap size to configure, there's memory and "total"
1117     # memory (iow. memory+swap). This means we have to change them together.
1118     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1119     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1120     if (defined($wanted_memory) || defined($wanted_swap)) {
1121
1122         $wanted_memory //= ($conf->{memory} || 512);
1123         $wanted_swap //=  ($conf->{swap} || 0);
1124
1125         my $total = $wanted_memory + $wanted_swap;
1126         if ($running) {
1127             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1128             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1129         }
1130         $conf->{memory} = $wanted_memory;
1131         $conf->{swap} = $wanted_swap;
1132
1133         write_config($vmid, $conf) if $running;
1134     }
1135
1136     foreach my $opt (keys %$param) {
1137         my $value = $param->{$opt};
1138         if ($opt eq 'hostname') {
1139             $conf->{$opt} = $value;
1140         } elsif ($opt eq 'onboot') {
1141             $conf->{$opt} = $value ? 1 : 0;
1142         } elsif ($opt eq 'startup') {
1143             $conf->{$opt} = $value;
1144         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1145             $conf->{$opt} = $value;
1146             push @nohotplug, $opt;
1147             next if $running;
1148         } elsif ($opt eq 'nameserver') {
1149             my $list = verify_nameserver_list($value);
1150             $conf->{$opt} = $list;
1151             push @nohotplug, $opt;
1152             next if $running;
1153         } elsif ($opt eq 'searchdomain') {
1154             my $list = verify_searchdomain_list($value);
1155             $conf->{$opt} = $list;
1156             push @nohotplug, $opt;
1157             next if $running;
1158         } elsif ($opt eq 'cpulimit') {
1159             $conf->{$opt} = $value;
1160             push @nohotplug, $opt; # fixme: hotplug
1161             next;
1162         } elsif ($opt eq 'cpuunits') {
1163             $conf->{$opt} = $value;
1164             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1165         } elsif ($opt eq 'description') {
1166             $conf->{$opt} = PVE::Tools::encode_text($value);
1167         } elsif ($opt =~ m/^net(\d+)$/) {
1168             my $netid = $1;
1169             my $net = parse_lxc_network($value);
1170             if (!$running) {
1171                 $conf->{$opt} = print_lxc_network($net);
1172             } else {
1173                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1174             }
1175         } elsif ($opt =~ m/^mp(\d+)$/) {
1176             $conf->{$opt} = $value;
1177             push @$new_disks, $opt;
1178             push @nohotplug, $opt;
1179             next;
1180         } elsif ($opt eq 'rootfs') {
1181             die "implement me: $opt";
1182         } else {
1183             die "implement me: $opt";
1184         }
1185         write_config($vmid, $conf) if $running;
1186     }
1187
1188     if ($running && scalar(@nohotplug)) {
1189         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1190     }
1191
1192     if (@$new_disks) {
1193         my $storage_cfg = PVE::Storage::config();
1194         PVE::API2::LXC::create_disks($storage_cfg, $vmid, $conf, $conf);
1195         mount_all($vmid, $storage_cfg, $conf, $new_disks, 1);
1196         umount_all($vmid, $storage_cfg, $conf, 0);
1197     }
1198 }
1199
1200 sub has_dev_console {
1201     my ($conf) = @_;
1202
1203     return !(defined($conf->{console}) && !$conf->{console});
1204 }
1205
1206 sub get_tty_count {
1207     my ($conf) = @_;
1208
1209     return $conf->{tty} // $confdesc->{tty}->{default};
1210 }
1211
1212 sub get_cmode {
1213     my ($conf) = @_;
1214
1215     return $conf->{cmode} // $confdesc->{cmode}->{default};
1216 }
1217
1218 sub get_console_command {
1219     my ($vmid, $conf) = @_;
1220
1221     my $cmode = get_cmode($conf);
1222
1223     if ($cmode eq 'console') {
1224         return ['lxc-console', '-n',  $vmid, '-t', 0];
1225     } elsif ($cmode eq 'tty') {
1226         return ['lxc-console', '-n',  $vmid];
1227     } elsif ($cmode eq 'shell') {
1228         return ['lxc-attach', '--clear-env', '-n', $vmid];
1229     } else {
1230         die "internal error";
1231     }
1232 }
1233
1234 sub get_primary_ips {
1235     my ($conf) = @_;
1236
1237     # return data from net0
1238
1239     return undef if !defined($conf->{net0});
1240     my $net = parse_lxc_network($conf->{net0});
1241
1242     my $ipv4 = $net->{ip};
1243     if ($ipv4) {
1244         if ($ipv4 =~ /^(dhcp|manual)$/) {
1245             $ipv4 = undef
1246         } else {
1247             $ipv4 =~ s!/\d+$!!;
1248         }
1249     }
1250     my $ipv6 = $net->{ip6};
1251     if ($ipv6) {
1252         if ($ipv6 =~ /^(dhcp|manual)$/) {
1253             $ipv6 = undef;
1254         } else {
1255             $ipv6 =~ s!/\d+$!!;
1256         }
1257     }
1258
1259     return ($ipv4, $ipv6);
1260 }
1261
1262
1263 sub destroy_lxc_container {
1264     my ($storage_cfg, $vmid, $conf) = @_;
1265
1266     foreach_mountpoint($conf, sub {
1267         my ($ms, $mountpoint) = @_;
1268         my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $mountpoint->{volume});
1269         PVE::Storage::vdisk_free($storage_cfg, $mountpoint->{volume}) if $vmid == $owner;
1270     });
1271
1272     rmdir "/var/lib/lxc/$vmid/rootfs";
1273     unlink "/var/lib/lxc/$vmid/config";
1274     rmdir "/var/lib/lxc/$vmid";
1275     destroy_config($vmid);
1276
1277     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1278     #PVE::Tools::run_command($cmd);
1279 }
1280
1281 sub vm_stop_cleanup {
1282     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1283
1284     eval {
1285         if (!$keepActive) {
1286
1287             my $vollist = get_vm_volumes($conf);
1288             my $loopdevlist = get_vm_volumes($conf, 'rootfs');
1289
1290             detach_loops($storage_cfg, $loopdevlist);
1291             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1292         }
1293     };
1294     warn $@ if $@; # avoid errors - just warn
1295 }
1296
1297 my $safe_num_ne = sub {
1298     my ($a, $b) = @_;
1299
1300     return 0 if !defined($a) && !defined($b);
1301     return 1 if !defined($a);
1302     return 1 if !defined($b);
1303
1304     return $a != $b;
1305 };
1306
1307 my $safe_string_ne = sub {
1308     my ($a, $b) = @_;
1309
1310     return 0 if !defined($a) && !defined($b);
1311     return 1 if !defined($a);
1312     return 1 if !defined($b);
1313
1314     return $a ne $b;
1315 };
1316
1317 sub update_net {
1318     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1319
1320     if ($newnet->{type} ne 'veth') {
1321         # for when there are physical interfaces
1322         die "cannot update interface of type $newnet->{type}";
1323     }
1324
1325     my $veth = "veth${vmid}i${netid}";
1326     my $eth = $newnet->{name};
1327
1328     if (my $oldnetcfg = $conf->{$opt}) {
1329         my $oldnet = parse_lxc_network($oldnetcfg);
1330
1331         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1332             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1333
1334             PVE::Network::veth_delete($veth);
1335             delete $conf->{$opt};
1336             write_config($vmid, $conf);
1337
1338             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1339
1340         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1341                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1342                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1343
1344                 if ($oldnet->{bridge}) {
1345                     PVE::Network::tap_unplug($veth);
1346                     foreach (qw(bridge tag firewall)) {
1347                         delete $oldnet->{$_};
1348                     }
1349                     $conf->{$opt} = print_lxc_network($oldnet);
1350                     write_config($vmid, $conf);
1351                 }
1352
1353                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1354                 foreach (qw(bridge tag firewall)) {
1355                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1356                 }
1357                 $conf->{$opt} = print_lxc_network($oldnet);
1358                 write_config($vmid, $conf);
1359         }
1360     } else {
1361         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1362     }
1363
1364     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1365 }
1366
1367 sub hotplug_net {
1368     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1369
1370     my $veth = "veth${vmid}i${netid}";
1371     my $vethpeer = $veth . "p";
1372     my $eth = $newnet->{name};
1373
1374     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1375     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1376
1377     # attach peer in container
1378     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1379     PVE::Tools::run_command($cmd);
1380
1381     # link up peer in container
1382     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1383     PVE::Tools::run_command($cmd);
1384
1385     my $done = { type => 'veth' };
1386     foreach (qw(bridge tag firewall hwaddr name)) {
1387         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1388     }
1389     $conf->{$opt} = print_lxc_network($done);
1390
1391     write_config($vmid, $conf);
1392 }
1393
1394 sub update_ipconfig {
1395     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1396
1397     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1398
1399     my $optdata = parse_lxc_network($conf->{$opt});
1400     my $deleted = [];
1401     my $added = [];
1402     my $nscmd = sub {
1403         my $cmdargs = shift;
1404         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1405     };
1406     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1407
1408     my $change_ip_config = sub {
1409         my ($ipversion) = @_;
1410
1411         my $family_opt = "-$ipversion";
1412         my $suffix = $ipversion == 4 ? '' : $ipversion;
1413         my $gw= "gw$suffix";
1414         my $ip= "ip$suffix";
1415
1416         my $newip = $newnet->{$ip};
1417         my $newgw = $newnet->{$gw};
1418         my $oldip = $optdata->{$ip};
1419
1420         my $change_ip = &$safe_string_ne($oldip, $newip);
1421         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1422
1423         return if !$change_ip && !$change_gw;
1424
1425         # step 1: add new IP, if this fails we cancel
1426         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1427             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1428             if (my $err = $@) {
1429                 warn $err;
1430                 return;
1431             }
1432         }
1433
1434         # step 2: replace gateway
1435         #   If this fails we delete the added IP and cancel.
1436         #   If it succeeds we save the config and delete the old IP, ignoring
1437         #   errors. The config is then saved.
1438         # Note: 'ip route replace' can add
1439         if ($change_gw) {
1440             if ($newgw) {
1441                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1442                 if (my $err = $@) {
1443                     warn $err;
1444                     # the route was not replaced, the old IP is still available
1445                     # rollback (delete new IP) and cancel
1446                     if ($change_ip) {
1447                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1448                         warn $@ if $@; # no need to die here
1449                     }
1450                     return;
1451                 }
1452             } else {
1453                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1454                 # if the route was not deleted, the guest might have deleted it manually
1455                 # warn and continue
1456                 warn $@ if $@;
1457             }
1458         }
1459
1460         # from this point on we save the configuration
1461         # step 3: delete old IP ignoring errors
1462         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1463             # We need to enable promote_secondaries, otherwise our newly added
1464             # address will be removed along with the old one.
1465             my $promote = 0;
1466             eval {
1467                 if ($ipversion == 4) {
1468                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1469                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1470                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1471                 }
1472                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1473             };
1474             warn $@ if $@; # no need to die here
1475
1476             if ($ipversion == 4) {
1477                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1478             }
1479         }
1480
1481         foreach my $property ($ip, $gw) {
1482             if ($newnet->{$property}) {
1483                 $optdata->{$property} = $newnet->{$property};
1484             } else {
1485                 delete $optdata->{$property};
1486             }
1487         }
1488         $conf->{$opt} = print_lxc_network($optdata);
1489         write_config($vmid, $conf);
1490         $lxc_setup->setup_network($conf);
1491     };
1492
1493     &$change_ip_config(4);
1494     &$change_ip_config(6);
1495
1496 }
1497
1498 # Internal snapshots
1499
1500 # NOTE: Snapshot create/delete involves several non-atomic
1501 # action, and can take a long time.
1502 # So we try to avoid locking the file and use 'lock' variable
1503 # inside the config file instead.
1504
1505 my $snapshot_copy_config = sub {
1506     my ($source, $dest) = @_;
1507
1508     foreach my $k (keys %$source) {
1509         next if $k eq 'snapshots';
1510         next if $k eq 'snapstate';
1511         next if $k eq 'snaptime';
1512         next if $k eq 'vmstate';
1513         next if $k eq 'lock';
1514         next if $k eq 'digest';
1515         next if $k eq 'description';
1516
1517         $dest->{$k} = $source->{$k};
1518     }
1519 };
1520
1521 my $snapshot_prepare = sub {
1522     my ($vmid, $snapname, $comment) = @_;
1523
1524     my $snap;
1525
1526     my $updatefn =  sub {
1527
1528         my $conf = load_config($vmid);
1529
1530         die "you can't take a snapshot if it's a template\n"
1531             if is_template($conf);
1532
1533         check_lock($conf);
1534
1535         $conf->{lock} = 'snapshot';
1536
1537         die "snapshot name '$snapname' already used\n"
1538             if defined($conf->{snapshots}->{$snapname});
1539
1540         my $storecfg = PVE::Storage::config();
1541         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1542
1543         $snap = $conf->{snapshots}->{$snapname} = {};
1544
1545         &$snapshot_copy_config($conf, $snap);
1546
1547         $snap->{'snapstate'} = "prepare";
1548         $snap->{'snaptime'} = time();
1549         $snap->{'description'} = $comment if $comment;
1550         $conf->{snapshots}->{$snapname} = $snap;
1551
1552         write_config($vmid, $conf);
1553     };
1554
1555     lock_container($vmid, 10, $updatefn);
1556
1557     return $snap;
1558 };
1559
1560 my $snapshot_commit = sub {
1561     my ($vmid, $snapname) = @_;
1562
1563     my $updatefn = sub {
1564
1565         my $conf = load_config($vmid);
1566
1567         die "missing snapshot lock\n"
1568             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1569
1570         die "snapshot '$snapname' does not exist\n"
1571             if !defined($conf->{snapshots}->{$snapname});
1572
1573         die "wrong snapshot state\n"
1574             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1575                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1576
1577         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1578         delete $conf->{lock};
1579         $conf->{parent} = $snapname;
1580
1581         write_config($vmid, $conf);
1582     };
1583
1584     lock_container($vmid, 10 ,$updatefn);
1585 };
1586
1587 sub has_feature {
1588     my ($feature, $conf, $storecfg, $snapname) = @_;
1589
1590     my $err;
1591
1592     foreach_mountpoint($conf, sub {
1593         my ($ms, $mountpoint) = @_;
1594
1595         return if $err; # skip further test
1596
1597         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1598
1599         # TODO: implement support for mountpoints
1600         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1601             if $ms ne 'rootfs';
1602     });
1603
1604     return $err ? 0 : 1;
1605 }
1606
1607 sub snapshot_create {
1608     my ($vmid, $snapname, $comment) = @_;
1609
1610     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1611
1612     my $conf = load_config($vmid);
1613
1614     my $cmd = "/usr/bin/lxc-freeze -n $vmid";
1615     my $running = check_running($vmid);
1616     eval {
1617         if ($running) {
1618             PVE::Tools::run_command($cmd);
1619         };
1620
1621         my $storecfg = PVE::Storage::config();
1622         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1623         my $volid = $rootinfo->{volume};
1624
1625         $cmd = "/usr/bin/lxc-unfreeze -n $vmid";
1626         if ($running) {
1627             PVE::Tools::run_command($cmd);
1628         };
1629
1630         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1631         &$snapshot_commit($vmid, $snapname);
1632     };
1633     if(my $err = $@) {
1634         snapshot_delete($vmid, $snapname, 1);
1635         die "$err\n";
1636     }
1637 }
1638
1639 sub snapshot_delete {
1640     my ($vmid, $snapname, $force) = @_;
1641
1642     my $snap;
1643
1644     my $conf;
1645
1646     my $updatefn =  sub {
1647
1648         $conf = load_config($vmid);
1649
1650         die "you can't delete a snapshot if vm is a template\n"
1651             if is_template($conf);
1652
1653         $snap = $conf->{snapshots}->{$snapname};
1654
1655         check_lock($conf);
1656
1657         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1658
1659         $snap->{snapstate} = 'delete';
1660
1661         write_config($vmid, $conf);
1662     };
1663
1664     lock_container($vmid, 10, $updatefn);
1665
1666     my $storecfg = PVE::Storage::config();
1667
1668     my $del_snap =  sub {
1669
1670         check_lock($conf);
1671
1672         if ($conf->{parent} eq $snapname) {
1673             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1674                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1675             } else {
1676                 delete $conf->{parent};
1677             }
1678         }
1679
1680         delete $conf->{snapshots}->{$snapname};
1681
1682         write_config($vmid, $conf);
1683     };
1684
1685     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1686     my $rootinfo = parse_ct_mountpoint($rootfs);
1687     my $volid = $rootinfo->{volume};
1688
1689     eval {
1690         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1691     };
1692     my $err = $@;
1693
1694     if(!$err || ($err && $force)) {
1695         lock_container($vmid, 10, $del_snap);
1696         if ($err) {
1697             die "Can't delete snapshot: $vmid $snapname $err\n";
1698         }
1699     }
1700 }
1701
1702 sub snapshot_rollback {
1703     my ($vmid, $snapname) = @_;
1704
1705     my $storecfg = PVE::Storage::config();
1706
1707     my $conf = load_config($vmid);
1708
1709     die "you can't rollback if vm is a template\n" if is_template($conf);
1710
1711     my $snap = $conf->{snapshots}->{$snapname};
1712
1713     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1714
1715     my $rootfs = $snap->{rootfs};
1716     my $rootinfo = parse_ct_mountpoint($rootfs);
1717     my $volid = $rootinfo->{volume};
1718
1719     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1720
1721     my $updatefn = sub {
1722
1723         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1724             if $snap->{snapstate};
1725
1726         check_lock($conf);
1727
1728         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1729
1730         die "unable to rollback vm $vmid: vm is running\n"
1731             if check_running($vmid);
1732
1733         $conf->{lock} = 'rollback';
1734
1735         my $forcemachine;
1736
1737         # copy snapshot config to current config
1738
1739         my $tmp_conf = $conf;
1740         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1741         $conf->{snapshots} = $tmp_conf->{snapshots};
1742         delete $conf->{snaptime};
1743         delete $conf->{snapname};
1744         $conf->{parent} = $snapname;
1745
1746         write_config($vmid, $conf);
1747     };
1748
1749     my $unlockfn = sub {
1750         delete $conf->{lock};
1751         write_config($vmid, $conf);
1752     };
1753
1754     lock_container($vmid, 10, $updatefn);
1755
1756     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1757
1758     lock_container($vmid, 5, $unlockfn);
1759 }
1760
1761 sub template_create {
1762     my ($vmid, $conf) = @_;
1763
1764     my $storecfg = PVE::Storage::config();
1765
1766     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1767     my $volid = $rootinfo->{volume};
1768
1769     die "Template feature is not available for '$volid'\n"
1770         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1771
1772     PVE::Storage::activate_volumes($storecfg, [$volid]);
1773
1774     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1775     $rootinfo->{volume} = $template_volid;
1776     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1777
1778     write_config($vmid, $conf);
1779 }
1780
1781 sub is_template {
1782     my ($conf) = @_;
1783
1784     return 1 if defined $conf->{template} && $conf->{template} == 1;
1785 }
1786
1787 sub mountpoint_names {
1788     my ($reverse) = @_;
1789
1790     my @names = ('rootfs');
1791
1792     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1793         push @names, "mp$i";
1794     }
1795
1796     return $reverse ? reverse @names : @names;
1797 }
1798
1799 sub foreach_mountpoint_full {
1800     my ($conf, $reverse, $func) = @_;
1801
1802     foreach my $key (mountpoint_names($reverse)) {
1803         my $value = $conf->{$key};
1804         next if !defined($value);
1805         my $mountpoint = parse_ct_mountpoint($value);
1806         $mountpoint->{mp} = '/' if $key eq 'rootfs'; # just to be sure
1807         &$func($key, $mountpoint);
1808     }
1809 }
1810
1811 sub foreach_mountpoint {
1812     my ($conf, $func) = @_;
1813
1814     foreach_mountpoint_full($conf, 0, $func);
1815 }
1816
1817 sub foreach_mountpoint_reverse {
1818     my ($conf, $func) = @_;
1819
1820     foreach_mountpoint_full($conf, 1, $func);
1821 }
1822
1823 sub loopdevices_list {
1824
1825     my $loopdev = {};
1826     my $parser = sub {
1827         my $line = shift;
1828         if ($line =~ m/^(\/dev\/loop\d+)\s+\d\s+\d\s+\d\s+\d\s(\S+)$/) {
1829             $loopdev->{$1} = $2;
1830         }
1831     };
1832
1833     PVE::Tools::run_command(['losetup'], outfunc => $parser);
1834
1835     return $loopdev;
1836 }
1837
1838 sub blockdevices_list {
1839
1840     my $bdevs = {};
1841     dir_glob_foreach("/sys/dev/block/", '(\d+):(\d+)', sub {
1842         my (undef, $major, $minor) = @_;
1843         my $bdev = readlink("/sys/dev/block/$major:$minor");
1844         $bdev =~ s/\.\.\/\.\.\/devices\/virtual\/block\//\/dev\//;
1845         $bdevs->{$bdev}->{major} = $major;
1846         $bdevs->{$bdev}->{minor} = $minor;
1847     });
1848     return $bdevs;
1849 }
1850
1851 sub find_loopdev {
1852     my ($loopdevs, $path) = @_;
1853
1854     foreach my $dev (keys %$loopdevs){
1855         return $dev if $loopdevs->{$dev} eq $path;
1856     }
1857 }
1858
1859 sub check_ct_modify_config_perm {
1860     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1861
1862     return 1 if $authuser ne 'root@pam';
1863
1864     foreach my $opt (@$key_list) {
1865
1866         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1867             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1868         } elsif ($opt eq 'disk') {
1869             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1870         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1871             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1872         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1873                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1874             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1875         } else {
1876             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1877         }
1878     }
1879
1880     return 1;
1881 }
1882
1883 sub attach_loops {
1884     my ($storage_cfg, $vollist, $snapname) = @_;
1885
1886     my $loopdevs = {};
1887
1888     foreach my $volid (@$vollist) {
1889
1890         my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
1891         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1892
1893         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1894             PVE::Storage::parse_volname($storage_cfg, $volid);
1895
1896         if ($format eq 'raw' && $scfg->{path}) {
1897             my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
1898             my $loopdev;
1899
1900             my $parser = sub {
1901                 my $line = shift;
1902                 $loopdev = $line if $line =~m|^/dev/loop\d+$|;
1903                 $loopdevs->{$loopdev} = $path;
1904             };
1905
1906             PVE::Tools::run_command(['losetup', '--find', '--show', $path], outfunc => $parser);
1907         }
1908     }
1909
1910     return $loopdevs;
1911 }
1912
1913 sub detach_loops {
1914     my ($storage_cfg, $vollist, $snapname) = @_;
1915
1916     my $loopdevs = loopdevices_list();
1917
1918     foreach my $volid (@$vollist) {
1919
1920         my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
1921         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1922
1923         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1924             PVE::Storage::parse_volname($storage_cfg, $volid);
1925
1926         if ($format eq 'raw' && $scfg->{path}) {
1927             my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
1928             foreach my $dev (keys %$loopdevs){
1929                 PVE::Tools::run_command(['losetup', '-d', $dev]) if $loopdevs->{$dev} eq $path;
1930             }
1931         }
1932     }
1933 }
1934
1935 sub umount_all {
1936     my ($vmid, $storage_cfg, $conf, $noerr, $loopdevs) = @_;
1937
1938     $loopdevs ||= loopdevices_list();
1939
1940     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1941     my $volid_list = get_vm_volumes($conf);
1942
1943     foreach_mountpoint_reverse($conf, sub {
1944         my ($ms, $mountpoint) = @_;
1945
1946         my $volid = $mountpoint->{volume};
1947         my $mount = $mountpoint->{mp};
1948
1949         return if !$volid || !$mount;
1950
1951         my $mount_path = "$rootdir/$mount";
1952         $mount_path =~ s!/+!/!g;
1953
1954         # fixme: test if mounted?
1955         eval {
1956             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1957         };
1958         if (my $err = $@) {
1959             if ($noerr) {
1960                 warn $err;
1961             } else {
1962                 die $err;
1963             }
1964         }
1965     });
1966
1967     PVE::LXC::detach_loops($storage_cfg, $volid_list);
1968 }
1969
1970 sub mount_all {
1971     my ($vmid, $storage_cfg, $conf, $mkdirs) = @_;
1972
1973     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1974
1975     my $volid_list = get_vm_volumes($conf);
1976     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
1977
1978     my $loopdevs;
1979     eval {
1980         $loopdevs = attach_loops($storage_cfg, $volid_list);
1981
1982         foreach_mountpoint($conf, sub {
1983             my ($ms, $mountpoint) = @_;
1984
1985             my $volid = $mountpoint->{volume};
1986             my $mount = $mountpoint->{mp};
1987
1988             return if !$volid || !$mount;
1989
1990             my $image_path = PVE::Storage::path($storage_cfg, $volid);
1991             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1992                 PVE::Storage::parse_volname($storage_cfg, $volid);
1993
1994             die "unable to mount base volume - internal error" if $isBase;
1995
1996             File::Path::make_path "$rootdir/$mount" if $mkdirs;
1997             mountpoint_mount($mountpoint, $rootdir, $storage_cfg, $loopdevs);
1998         });
1999     };
2000     if (my $err = $@) {
2001         warn "mounting container failed - $err";
2002         umount_all($vmid, $storage_cfg, $conf, 1);
2003     }
2004
2005     return wantarray ? ($rootdir, $loopdevs) : $rootdir;
2006 }
2007
2008
2009 sub mountpoint_mount_path {
2010     my ($mountpoint, $storage_cfg, $loopdevs, $snapname) = @_;
2011
2012     return mountpoint_mount($mountpoint, undef, $storage_cfg, $loopdevs, $snapname);
2013 }
2014
2015 # use $rootdir = undef to just return the corresponding mount path
2016 sub mountpoint_mount {
2017     my ($mountpoint, $rootdir, $storage_cfg, $loopdevs, $snapname) = @_;
2018
2019     my $volid = $mountpoint->{volume};
2020     my $mount = $mountpoint->{mp};
2021
2022     return if !$volid || !$mount;
2023
2024     my $mount_path;
2025
2026     if (defined($rootdir)) {
2027         $rootdir =~ s!/+$!!;
2028         $mount_path = "$rootdir/$mount";
2029         $mount_path =~ s!/+!/!g;
2030         File::Path::mkpath($mount_path);
2031     }
2032
2033     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2034
2035     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2036
2037     if ($storage) {
2038
2039         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2040         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2041
2042         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2043             PVE::Storage::parse_volname($storage_cfg, $volid);
2044
2045         if ($format eq 'subvol') {
2046
2047             if ($mount_path) {
2048                 if ($snapname) {
2049                     if ($scfg->{type} eq 'zfspool') {
2050                         my $path_arg = $path;
2051                         $path_arg =~ s!^/+!!;
2052                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2053                     } else {
2054                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2055                     }
2056                 } else {
2057                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2058                 }
2059             }
2060             return $path;
2061
2062         } elsif ($format eq 'raw') {
2063
2064             if ($scfg->{path}) {
2065                 $path = find_loopdev($loopdevs, $path) if $loopdevs;
2066             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'rbd') {
2067                 # do nothing
2068             } else {
2069                 die "unsupported storage type '$scfg->{type}'\n";
2070             }
2071             if ($mount_path) {
2072                 if ($isBase || defined($snapname)) {
2073                     PVE::Tools::run_command(['mount', '-o', 'ro', $path, $mount_path]);
2074                 } else {
2075                     PVE::Tools::run_command(['mount', $path, $mount_path]);
2076                 }
2077             }
2078             return $path;
2079         } else {
2080             die "unsupported image format '$format'\n";
2081         }
2082     } elsif ($volid =~ m|^/dev/.+|) {
2083         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2084         return $volid;
2085     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2086         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2087         return $volid;
2088     }
2089
2090     die "unsupported storage";
2091 }
2092
2093 sub get_vm_volumes {
2094     my ($conf, $excludes) = @_;
2095
2096     my $vollist = [];
2097
2098     foreach_mountpoint($conf, sub {
2099         my ($ms, $mountpoint) = @_;
2100
2101         return if $excludes && $ms eq $excludes;
2102
2103         my $volid = $mountpoint->{volume};
2104
2105         return if !$volid || $volid =~ m|^/|;
2106
2107         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2108         return if !$sid;
2109
2110         push @$vollist, $volid;
2111     });
2112
2113     return $vollist;
2114 }
2115
2116 # bash completion helper
2117
2118 sub complete_os_templates {
2119     my ($cmdname, $pname, $cvalue) = @_;
2120
2121     my $cfg = PVE::Storage::config();
2122
2123     my $storeid;
2124
2125     if ($cvalue =~ m/^([^:]+):/) {
2126         $storeid = $1;
2127     }
2128
2129     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2130     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2131
2132     my $res = [];
2133     foreach my $id (keys %$data) {
2134         foreach my $item (@{$data->{$id}}) {
2135             push @$res, $item->{volid} if defined($item->{volid});
2136         }
2137     }
2138
2139     return $res;
2140 }
2141
2142 sub complete_migration_target {
2143
2144     my $res = [];
2145
2146     my $nodelist = PVE::Cluster::get_nodelist();
2147     foreach my $node (@$nodelist) {
2148         next if $node eq $nodename;
2149         push @$res, $node;
2150     }
2151
2152     return $res;
2153 }
2154
2155 my $complete_ctid_full = sub {
2156     my ($running) = @_;
2157
2158     my $idlist = vmstatus();
2159
2160     my $active_hash = list_active_containers();
2161
2162     my $res = [];
2163
2164     foreach my $id (keys %$idlist) {
2165         my $d = $idlist->{$id};
2166         if (defined($running)) {
2167             next if $d->{template};
2168             next if $running && !$active_hash->{$id};
2169             next if !$running && $active_hash->{$id};
2170         }
2171         push @$res, $id;
2172
2173     }
2174     return $res;
2175 };
2176
2177 sub complete_ctid {
2178     return &$complete_ctid_full();
2179 }
2180
2181 sub complete_ctid_stopped {
2182     return &$complete_ctid_full(0);
2183 }
2184
2185 sub complete_ctid_running {
2186     return &$complete_ctid_full(1);
2187 }
2188
2189 1;