src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21
  22 use Data::Dumper;
  23
  24 my $nodename = PVE::INotify::nodename();
  25
  26 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  27
  28 my $rootfs_desc = {
  29     volume => {
  30         type => 'string',
  31         default_key => 1,
  32         format_description => 'volume',
  33         description => 'Volume, device or directory to mount into the container.',
  34     },
  35     backup => {
  36         type => 'boolean',
  37         format_description => '[1|0]',
  38         description => 'Whether to include the mountpoint in backups.',
  39         optional => 1,
  40     },
  41     size => {
  42         type => 'disk-size',
  43         format_description => 'DiskSize',
  44         description => 'Volume size (read only value).',
  45         optional => 1,
  46     },
  47 };
  48
  49 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  50     type => 'string', format => $rootfs_desc,
  51     description => "Use volume as container root.",
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  56     description => "The name of the snapshot.",
  57     type => 'string', format => 'pve-configid',
  58     maxLength => 40,
  59 });
  60
  61 my $confdesc = {
  62     lock => {
  63         optional => 1,
  64         type => 'string',
  65         description => "Lock/unlock the VM.",
  66         enum => [qw(migrate backup snapshot rollback)],
  67     },
  68     onboot => {
  69         optional => 1,
  70         type => 'boolean',
  71         description => "Specifies whether a VM will be started during system bootup.",
  72         default => 0,
  73     },
  74     startup => get_standard_option('pve-startup-order'),
  75     template => {
  76         optional => 1,
  77         type => 'boolean',
  78         description => "Enable/disable Template.",
  79         default => 0,
  80     },
  81     arch => {
  82         optional => 1,
  83         type => 'string',
  84         enum => ['amd64', 'i386'],
  85         description => "OS architecture type.",
  86         default => 'amd64',
  87     },
  88     ostype => {
  89         optional => 1,
  90         type => 'string',
  91         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  92         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  93     },
  94     console => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Attach a console device (/dev/console) to the container.",
  98         default => 1,
  99     },
 100     tty => {
 101         optional => 1,
 102         type => 'integer',
 103         description => "Specify the number of tty available to the container",
 104         minimum => 0,
 105         maximum => 6,
 106         default => 2,
 107     },
 108     cpulimit => {
 109         optional => 1,
 110         type => 'number',
 111         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 112         minimum => 0,
 113         maximum => 128,
 114         default => 0,
 115     },
 116     cpuunits => {
 117         optional => 1,
 118         type => 'integer',
 119         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 120         minimum => 0,
 121         maximum => 500000,
 122         default => 1024,
 123     },
 124     memory => {
 125         optional => 1,
 126         type => 'integer',
 127         description => "Amount of RAM for the VM in MB.",
 128         minimum => 16,
 129         default => 512,
 130     },
 131     swap => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "Amount of SWAP for the VM in MB.",
 135         minimum => 0,
 136         default => 512,
 137     },
 138     hostname => {
 139         optional => 1,
 140         description => "Set a host name for the container.",
 141         type => 'string', format => 'dns-name',
 142         maxLength => 255,
 143     },
 144     description => {
 145         optional => 1,
 146         type => 'string',
 147         description => "Container description. Only used on the configuration web interface.",
 148     },
 149     searchdomain => {
 150         optional => 1,
 151         type => 'string', format => 'dns-name-list',
 152         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 153     },
 154     nameserver => {
 155         optional => 1,
 156         type => 'string', format => 'address-list',
 157         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 158     },
 159     rootfs => get_standard_option('pve-ct-rootfs'),
 160     parent => {
 161         optional => 1,
 162         type => 'string', format => 'pve-configid',
 163         maxLength => 40,
 164         description => "Parent snapshot name. This is used internally, and should not be modified.",
 165     },
 166     snaptime => {
 167         optional => 1,
 168         description => "Timestamp for snapshots.",
 169         type => 'integer',
 170         minimum => 0,
 171     },
 172     cmode => {
 173         optional => 1,
 174         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 175         type => 'string',
 176         enum => ['shell', 'console', 'tty'],
 177         default => 'tty',
 178     },
 179     protection => {
 180         optional => 1,
 181         type => 'boolean',
 182         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 183         default => 0,
 184     },
 185 };
 186
 187 my $valid_lxc_conf_keys = {
 188     'lxc.include' => 1,
 189     'lxc.arch' => 1,
 190     'lxc.utsname' => 1,
 191     'lxc.haltsignal' => 1,
 192     'lxc.rebootsignal' => 1,
 193     'lxc.stopsignal' => 1,
 194     'lxc.init_cmd' => 1,
 195     'lxc.network.type' => 1,
 196     'lxc.network.flags' => 1,
 197     'lxc.network.link' => 1,
 198     'lxc.network.mtu' => 1,
 199     'lxc.network.name' => 1,
 200     'lxc.network.hwaddr' => 1,
 201     'lxc.network.ipv4' => 1,
 202     'lxc.network.ipv4.gateway' => 1,
 203     'lxc.network.ipv6' => 1,
 204     'lxc.network.ipv6.gateway' => 1,
 205     'lxc.network.script.up' => 1,
 206     'lxc.network.script.down' => 1,
 207     'lxc.pts' => 1,
 208     'lxc.console.logfile' => 1,
 209     'lxc.console' => 1,
 210     'lxc.tty' => 1,
 211     'lxc.devttydir' => 1,
 212     'lxc.hook.autodev' => 1,
 213     'lxc.autodev' => 1,
 214     'lxc.kmsg' => 1,
 215     'lxc.mount' => 1,
 216     'lxc.mount.entry' => 1,
 217     'lxc.mount.auto' => 1,
 218     'lxc.rootfs' => 1,
 219     'lxc.rootfs.mount' => 1,
 220     'lxc.rootfs.options' => 1,
 221     # lxc.cgroup.*
 222     'lxc.cap.drop' => 1,
 223     'lxc.cap.keep' => 1,
 224     'lxc.aa_profile' => 1,
 225     'lxc.aa_allow_incomplete' => 1,
 226     'lxc.se_context' => 1,
 227     'lxc.seccomp' => 1,
 228     'lxc.id_map' => 1,
 229     'lxc.hook.pre-start' => 1,
 230     'lxc.hook.pre-mount' => 1,
 231     'lxc.hook.mount' => 1,
 232     'lxc.hook.start' => 1,
 233     'lxc.hook.post-stop' => 1,
 234     'lxc.hook.clone' => 1,
 235     'lxc.hook.destroy' => 1,
 236     'lxc.loglevel' => 1,
 237     'lxc.logfile' => 1,
 238     'lxc.start.auto' => 1,
 239     'lxc.start.delay' => 1,
 240     'lxc.start.order' => 1,
 241     'lxc.group' => 1,
 242     'lxc.environment' => 1,
 243     'lxc.' => 1,
 244     'lxc.' => 1,
 245     'lxc.' => 1,
 246     'lxc.' => 1,
 247 };
 248
 249 my $netconf_desc = {
 250     type => {
 251         type => 'string',
 252         optional => 1,
 253         description => "Network interface type.",
 254         enum => [qw(veth)],
 255     },
 256     name => {
 257         type => 'string',
 258         format_description => 'String',
 259         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 260         pattern => '[-_.\w\d]+',
 261     },
 262     bridge => {
 263         type => 'string',
 264         format_description => 'vmbr<Number>',
 265         description => 'Bridge to attach the network device to.',
 266         pattern => '[-_.\w\d]+',
 267     },
 268     hwaddr => {
 269         type => 'string',
 270         format_description => 'MAC',
 271         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 272         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 273         optional => 1,
 274     },
 275     mtu => {
 276         type => 'integer',
 277         format_description => 'Number',
 278         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 279         minimum => 64, # minimum ethernet frame is 64 bytes
 280         optional => 1,
 281     },
 282     ip => {
 283         type => 'string',
 284         format => 'pve-ipv4-config',
 285         format_description => 'IPv4Format/CIDR',
 286         description => 'IPv4 address in CIDR format.',
 287         optional => 1,
 288     },
 289     gw => {
 290         type => 'string',
 291         format => 'ipv4',
 292         format_description => 'GatewayIPv4',
 293         description => 'Default gateway for IPv4 traffic.',
 294         optional => 1,
 295     },
 296     ip6 => {
 297         type => 'string',
 298         format => 'pve-ipv6-config',
 299         format_description => 'IPv6Format/CIDR',
 300         description => 'IPv6 address in CIDR format.',
 301         optional => 1,
 302     },
 303     gw6 => {
 304         type => 'string',
 305         format => 'ipv6',
 306         format_description => 'GatewayIPv6',
 307         description => 'Default gateway for IPv6 traffic.',
 308         optional => 1,
 309     },
 310     firewall => {
 311         type => 'boolean',
 312         format_description => '[1|0]',
 313         description => "Controls whether this interface's firewall rules should be used.",
 314         optional => 1,
 315     },
 316     tag => {
 317         type => 'integer',
 318         format_description => 'VlanNo',
 319         minimum => '2',
 320         maximum => '4094',
 321         description => "VLAN tag foro this interface.",
 322         optional => 1,
 323     },
 324 };
 325 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 326
 327 my $MAX_LXC_NETWORKS = 10;
 328 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 329     $confdesc->{"net$i"} = {
 330         optional => 1,
 331         type => 'string', format => $netconf_desc,
 332         description => "Specifies network interfaces for the container.",
 333     };
 334 }
 335
 336 my $mp_desc = {
 337     %$rootfs_desc,
 338     mp => {
 339         type => 'string',
 340         format_description => 'Path',
 341         description => 'Path to the mountpoint as seen from inside the container.',
 342         optional => 1,
 343     },
 344 };
 345 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 346
 347 my $MAX_MOUNT_POINTS = 10;
 348 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 349     $confdesc->{"mp$i"} = {
 350         optional => 1,
 351         type => 'string', format => $mp_desc,
 352         description => "Use volume as container mount point (experimental feature).",
 353         optional => 1,
 354     };
 355 }
 356
 357 sub write_pct_config {
 358     my ($filename, $conf) = @_;
 359
 360     delete $conf->{snapstate}; # just to be sure
 361
 362     my $generate_raw_config = sub {
 363         my ($conf) = @_;
 364
 365         my $raw = '';
 366
 367         # add description as comment to top of file
 368         my $descr = $conf->{description} || '';
 369         foreach my $cl (split(/\n/, $descr)) {
 370             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 371         }
 372
 373         foreach my $key (sort keys %$conf) {
 374             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 375                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 376             my $value = $conf->{$key};
 377             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 378             $raw .= "$key: $value\n";
 379         }
 380
 381         if (my $lxcconf = $conf->{lxc}) {
 382             foreach my $entry (@$lxcconf) {
 383                 my ($k, $v) = @$entry;
 384                 $raw .= "$k: $v\n";
 385             }
 386         }
 387
 388         return $raw;
 389     };
 390
 391     my $raw = &$generate_raw_config($conf);
 392
 393     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 394         $raw .= "\n[$snapname]\n";
 395         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 396     }
 397
 398     return $raw;
 399 }
 400
 401 sub check_type {
 402     my ($key, $value) = @_;
 403
 404     die "unknown setting '$key'\n" if !$confdesc->{$key};
 405
 406     my $type = $confdesc->{$key}->{type};
 407
 408     if (!defined($value)) {
 409         die "got undefined value\n";
 410     }
 411
 412     if ($value =~ m/[\n\r]/) {
 413         die "property contains a line feed\n";
 414     }
 415
 416     if ($type eq 'boolean') {
 417         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 418         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 419         die "type check ('boolean') failed - got '$value'\n";
 420     } elsif ($type eq 'integer') {
 421         return int($1) if $value =~ m/^(\d+)$/;
 422         die "type check ('integer') failed - got '$value'\n";
 423     } elsif ($type eq 'number') {
 424         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 425         die "type check ('number') failed - got '$value'\n";
 426     } elsif ($type eq 'string') {
 427         if (my $fmt = $confdesc->{$key}->{format}) {
 428             PVE::JSONSchema::check_format($fmt, $value);
 429             return $value;
 430         }
 431         return $value;
 432     } else {
 433         die "internal error"
 434     }
 435 }
 436
 437 sub parse_pct_config {
 438     my ($filename, $raw) = @_;
 439
 440     return undef if !defined($raw);
 441
 442     my $res = {
 443         digest => Digest::SHA::sha1_hex($raw),
 444         snapshots => {},
 445     };
 446
 447     $filename =~ m|/lxc/(\d+).conf$|
 448         || die "got strange filename '$filename'";
 449
 450     my $vmid = $1;
 451
 452     my $conf = $res;
 453     my $descr = '';
 454     my $section = '';
 455
 456     my @lines = split(/\n/, $raw);
 457     foreach my $line (@lines) {
 458         next if $line =~ m/^\s*$/;
 459
 460         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 461             $section = $1;
 462             $conf->{description} = $descr if $descr;
 463             $descr = '';
 464             $conf = $res->{snapshots}->{$section} = {};
 465             next;
 466         }
 467
 468         if ($line =~ m/^\#(.*)\s*$/) {
 469             $descr .= PVE::Tools::decode_text($1) . "\n";
 470             next;
 471         }
 472
 473         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 474             my $key = $1;
 475             my $value = $3;
 476             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 477                 push @{$conf->{lxc}}, [$key, $value];
 478             } else {
 479                 warn "vm $vmid - unable to parse config: $line\n";
 480             }
 481         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 482             $descr .= PVE::Tools::decode_text($2);
 483         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 484             $conf->{snapstate} = $1;
 485         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 486             my $key = $1;
 487             my $value = $2;
 488             eval { $value = check_type($key, $value); };
 489             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 490             $conf->{$key} = $value;
 491         } else {
 492             warn "vm $vmid - unable to parse config: $line\n";
 493         }
 494     }
 495
 496     $conf->{description} = $descr if $descr;
 497
 498     delete $res->{snapstate}; # just to be sure
 499
 500     return $res;
 501 }
 502
 503 sub config_list {
 504     my $vmlist = PVE::Cluster::get_vmlist();
 505     my $res = {};
 506     return $res if !$vmlist || !$vmlist->{ids};
 507     my $ids = $vmlist->{ids};
 508
 509     foreach my $vmid (keys %$ids) {
 510         next if !$vmid; # skip CT0
 511         my $d = $ids->{$vmid};
 512         next if !$d->{node} || $d->{node} ne $nodename;
 513         next if !$d->{type} || $d->{type} ne 'lxc';
 514         $res->{$vmid}->{type} = 'lxc';
 515     }
 516     return $res;
 517 }
 518
 519 sub cfs_config_path {
 520     my ($vmid, $node) = @_;
 521
 522     $node = $nodename if !$node;
 523     return "nodes/$node/lxc/$vmid.conf";
 524 }
 525
 526 sub config_file {
 527     my ($vmid, $node) = @_;
 528
 529     my $cfspath = cfs_config_path($vmid, $node);
 530     return "/etc/pve/$cfspath";
 531 }
 532
 533 sub load_config {
 534     my ($vmid, $node) = @_;
 535
 536     $node = $nodename if !$node;
 537     my $cfspath = cfs_config_path($vmid, $node);
 538
 539     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 540     die "container $vmid does not exists\n" if !defined($conf);
 541
 542     return $conf;
 543 }
 544
 545 sub create_config {
 546     my ($vmid, $conf) = @_;
 547
 548     my $dir = "/etc/pve/nodes/$nodename/lxc";
 549     mkdir $dir;
 550
 551     write_config($vmid, $conf);
 552 }
 553
 554 sub destroy_config {
 555     my ($vmid) = @_;
 556
 557     unlink config_file($vmid, $nodename);
 558 }
 559
 560 sub write_config {
 561     my ($vmid, $conf) = @_;
 562
 563     my $cfspath = cfs_config_path($vmid);
 564
 565     PVE::Cluster::cfs_write_file($cfspath, $conf);
 566 }
 567
 568 # flock: we use one file handle per process, so lock file
 569 # can be called multiple times and succeeds for the same process.
 570
 571 my $lock_handles =  {};
 572 my $lockdir = "/run/lock/lxc";
 573
 574 sub lock_filename {
 575     my ($vmid) = @_;
 576
 577     return "$lockdir/pve-config-${vmid}.lock";
 578 }
 579
 580 sub lock_aquire {
 581     my ($vmid, $timeout) = @_;
 582
 583     $timeout = 10 if !$timeout;
 584     my $mode = LOCK_EX;
 585
 586     my $filename = lock_filename($vmid);
 587
 588     mkdir $lockdir if !-d $lockdir;
 589
 590     my $lock_func = sub {
 591         if (!$lock_handles->{$$}->{$filename}) {
 592             my $fh = new IO::File(">>$filename") ||
 593                 die "can't open file - $!\n";
 594             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 595         }
 596
 597         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 598             print STDERR "trying to aquire lock...";
 599             my $success;
 600             while(1) {
 601                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 602                 # try again on EINTR (see bug #273)
 603                 if ($success || ($! != EINTR)) {
 604                     last;
 605                 }
 606             }
 607             if (!$success) {
 608                 print STDERR " failed\n";
 609                 die "can't aquire lock - $!\n";
 610             }
 611
 612             print STDERR " OK\n";
 613         }
 614
 615         $lock_handles->{$$}->{$filename}->{refcount}++;
 616     };
 617
 618     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 619     my $err = $@;
 620     if ($err) {
 621         die "can't lock file '$filename' - $err";
 622     }
 623 }
 624
 625 sub lock_release {
 626     my ($vmid) = @_;
 627
 628     my $filename = lock_filename($vmid);
 629
 630     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 631         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 632         if ($refcount <= 0) {
 633             $lock_handles->{$$}->{$filename} = undef;
 634             close ($fh);
 635         }
 636     }
 637 }
 638
 639 sub lock_container {
 640     my ($vmid, $timeout, $code, @param) = @_;
 641
 642     my $res;
 643
 644     lock_aquire($vmid, $timeout);
 645     eval { $res = &$code(@param) };
 646     my $err = $@;
 647     lock_release($vmid);
 648
 649     die $err if $err;
 650
 651     return $res;
 652 }
 653
 654 sub option_exists {
 655     my ($name) = @_;
 656
 657     return defined($confdesc->{$name});
 658 }
 659
 660 # add JSON properties for create and set function
 661 sub json_config_properties {
 662     my $prop = shift;
 663
 664     foreach my $opt (keys %$confdesc) {
 665         next if $opt eq 'parent' || $opt eq 'snaptime';
 666         next if $prop->{$opt};
 667         $prop->{$opt} = $confdesc->{$opt};
 668     }
 669
 670     return $prop;
 671 }
 672
 673 sub json_config_properties_no_rootfs {
 674     my $prop = shift;
 675
 676     foreach my $opt (keys %$confdesc) {
 677         next if $prop->{$opt};
 678         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 679         $prop->{$opt} = $confdesc->{$opt};
 680     }
 681
 682     return $prop;
 683 }
 684
 685 # container status helpers
 686
 687 sub list_active_containers {
 688
 689     my $filename = "/proc/net/unix";
 690
 691     # similar test is used by lcxcontainers.c: list_active_containers
 692     my $res = {};
 693
 694     my $fh = IO::File->new ($filename, "r");
 695     return $res if !$fh;
 696
 697     while (defined(my $line = <$fh>)) {
 698         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 699             my $path = $1;
 700             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 701                 $res->{$1} = 1;
 702             }
 703         }
 704     }
 705
 706     close($fh);
 707
 708     return $res;
 709 }
 710
 711 # warning: this is slow
 712 sub check_running {
 713     my ($vmid) = @_;
 714
 715     my $active_hash = list_active_containers();
 716
 717     return 1 if defined($active_hash->{$vmid});
 718
 719     return undef;
 720 }
 721
 722 sub get_container_disk_usage {
 723     my ($vmid) = @_;
 724
 725     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 726
 727     my $res = {
 728         total => 0,
 729         used => 0,
 730         avail => 0,
 731     };
 732
 733     my $parser = sub {
 734         my $line = shift;
 735         if (my ($fsid, $total, $used, $avail) = $line =~
 736             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 737             $res = {
 738                 total => $total,
 739                 used => $used,
 740                 avail => $avail,
 741             };
 742         }
 743     };
 744     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 745     warn $@ if $@;
 746
 747     return $res;
 748 }
 749
 750 sub vmstatus {
 751     my ($opt_vmid) = @_;
 752
 753     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 754
 755     my $active_hash = list_active_containers();
 756
 757     foreach my $vmid (keys %$list) {
 758         my $d = $list->{$vmid};
 759
 760         my $running = defined($active_hash->{$vmid});
 761
 762         $d->{status} = $running ? 'running' : 'stopped';
 763
 764         my $cfspath = cfs_config_path($vmid);
 765         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 766
 767         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 768         $d->{name} =~ s/[\s]//g;
 769
 770         $d->{cpus} = $conf->{cpulimit} // 0;
 771
 772         if ($running) {
 773             my $res = get_container_disk_usage($vmid);
 774             $d->{disk} = $res->{used};
 775             $d->{maxdisk} = $res->{total};
 776         } else {
 777             $d->{disk} = 0;
 778             # use 4GB by default ??
 779             if (my $rootfs = $conf->{rootfs}) {
 780                 my $rootinfo = parse_ct_mountpoint($rootfs);
 781                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 782             } else {
 783                 $d->{maxdisk} = 4*1024*1024*1024;
 784             }
 785         }
 786
 787         $d->{mem} = 0;
 788         $d->{swap} = 0;
 789         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 790         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 791
 792         $d->{uptime} = 0;
 793         $d->{cpu} = 0;
 794
 795         $d->{netout} = 0;
 796         $d->{netin} = 0;
 797
 798         $d->{diskread} = 0;
 799         $d->{diskwrite} = 0;
 800
 801         $d->{template} = is_template($conf);
 802     }
 803
 804     foreach my $vmid (keys %$list) {
 805         my $d = $list->{$vmid};
 806         next if $d->{status} ne 'running';
 807
 808         my $pid = find_lxc_pid($vmid);
 809         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 810         $d->{uptime} = time - $ctime; # the method lxcfs uses
 811
 812         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 813         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 814
 815         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 816         my @bytes = split(/\n/, $blkio_bytes);
 817         foreach my $byte (@bytes) {
 818             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 819                 $d->{diskread} = $2 if $key eq 'Read';
 820                 $d->{diskwrite} = $2 if $key eq 'Write';
 821             }
 822         }
 823     }
 824
 825     return $list;
 826 }
 827
 828 my $parse_size = sub {
 829     my ($value) = @_;
 830
 831     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 832     my ($size, $unit) = ($1, $3);
 833     if ($unit) {
 834         if ($unit eq 'K') {
 835             $size = $size * 1024;
 836         } elsif ($unit eq 'M') {
 837             $size = $size * 1024 * 1024;
 838         } elsif ($unit eq 'G') {
 839             $size = $size * 1024 * 1024 * 1024;
 840         }
 841     }
 842     return int($size);
 843 };
 844
 845 my $format_size = sub {
 846     my ($size) = @_;
 847
 848     $size = int($size);
 849
 850     my $kb = int($size/1024);
 851     return $size if $kb*1024 != $size;
 852
 853     my $mb = int($kb/1024);
 854     return "${kb}K" if $mb*1024 != $kb;
 855
 856     my $gb = int($mb/1024);
 857     return "${mb}M" if $gb*1024 != $mb;
 858
 859     return "${gb}G";
 860 };
 861
 862 sub parse_ct_mountpoint {
 863     my ($data) = @_;
 864
 865     $data //= '';
 866
 867     my $res;
 868     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 869     if ($@) {
 870         warn $@;
 871         return undef;
 872     }
 873
 874     return undef if !defined($res->{volume});
 875
 876     if ($res->{size}) {
 877         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 878     }
 879
 880     return $res;
 881 }
 882
 883 sub print_ct_mountpoint {
 884     my ($info, $nomp) = @_;
 885     my $skip = $nomp ? ['mp'] : [];
 886     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 887 }
 888
 889 sub print_lxc_network {
 890     my $net = shift;
 891     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 892 }
 893
 894 sub parse_lxc_network {
 895     my ($data) = @_;
 896
 897     my $res = {};
 898
 899     return $res if !$data;
 900
 901     eval { $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data) };
 902     if ($@) {
 903         warn $@;
 904         return undef;
 905     }
 906
 907     $res->{type} = 'veth';
 908     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 909
 910     return $res;
 911 }
 912
 913 sub read_cgroup_value {
 914     my ($group, $vmid, $name, $full) = @_;
 915
 916     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 917
 918     return PVE::Tools::file_get_contents($path) if $full;
 919
 920     return PVE::Tools::file_read_firstline($path);
 921 }
 922
 923 sub write_cgroup_value {
 924    my ($group, $vmid, $name, $value) = @_;
 925
 926    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 927    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 928
 929 }
 930
 931 sub find_lxc_console_pids {
 932
 933     my $res = {};
 934
 935     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 936         my ($pid) = @_;
 937
 938         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 939         return if !$cmdline;
 940
 941         my @args = split(/\0/, $cmdline);
 942
 943         # serach for lxc-console -n <vmid>
 944         return if scalar(@args) != 3;
 945         return if $args[1] ne '-n';
 946         return if $args[2] !~ m/^\d+$/;
 947         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 948
 949         my $vmid = $args[2];
 950
 951         push @{$res->{$vmid}}, $pid;
 952     });
 953
 954     return $res;
 955 }
 956
 957 sub find_lxc_pid {
 958     my ($vmid) = @_;
 959
 960     my $pid = undef;
 961     my $parser = sub {
 962         my $line = shift;
 963         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 964     };
 965     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
 966
 967     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 968
 969     return $pid;
 970 }
 971
 972 my $ipv4_reverse_mask = [
 973     '0.0.0.0',
 974     '128.0.0.0',
 975     '192.0.0.0',
 976     '224.0.0.0',
 977     '240.0.0.0',
 978     '248.0.0.0',
 979     '252.0.0.0',
 980     '254.0.0.0',
 981     '255.0.0.0',
 982     '255.128.0.0',
 983     '255.192.0.0',
 984     '255.224.0.0',
 985     '255.240.0.0',
 986     '255.248.0.0',
 987     '255.252.0.0',
 988     '255.254.0.0',
 989     '255.255.0.0',
 990     '255.255.128.0',
 991     '255.255.192.0',
 992     '255.255.224.0',
 993     '255.255.240.0',
 994     '255.255.248.0',
 995     '255.255.252.0',
 996     '255.255.254.0',
 997     '255.255.255.0',
 998     '255.255.255.128',
 999     '255.255.255.192',
1000     '255.255.255.224',
1001     '255.255.255.240',
1002     '255.255.255.248',
1003     '255.255.255.252',
1004     '255.255.255.254',
1005     '255.255.255.255',
1006 ];
1007
1008 # Note: we cannot use Net:IP, because that only allows strict
1009 # CIDR networks
1010 sub parse_ipv4_cidr {
1011     my ($cidr, $noerr) = @_;
1012
1013     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
1014         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
1015     }
1016
1017     return undef if $noerr;
1018
1019     die "unable to parse ipv4 address/mask\n";
1020 }
1021
1022 sub check_lock {
1023     my ($conf) = @_;
1024
1025     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1026 }
1027
1028 sub check_protection {
1029     my ($vm_conf, $err_msg) = @_;
1030
1031     if ($vm_conf->{protection}) {
1032         die "$err_msg - protection mode enabled\n";
1033     }
1034 }
1035
1036 sub update_lxc_config {
1037     my ($storage_cfg, $vmid, $conf) = @_;
1038
1039     my $dir = "/var/lib/lxc/$vmid";
1040
1041     if ($conf->{template}) {
1042
1043         unlink "$dir/config";
1044
1045         return;
1046     }
1047
1048     my $raw = '';
1049
1050     die "missing 'arch' - internal error" if !$conf->{arch};
1051     $raw .= "lxc.arch = $conf->{arch}\n";
1052
1053     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1054     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1055         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1056     } else {
1057         die "implement me";
1058     }
1059
1060     if (!has_dev_console($conf)) {
1061         $raw .= "lxc.console = none\n";
1062         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1063     }
1064
1065     my $ttycount = get_tty_count($conf);
1066     $raw .= "lxc.tty = $ttycount\n";
1067
1068     # some init scripts expects a linux terminal (turnkey).
1069     $raw .= "lxc.environment = TERM=linux\n";
1070
1071     my $utsname = $conf->{hostname} || "CT$vmid";
1072     $raw .= "lxc.utsname = $utsname\n";
1073
1074     my $memory = $conf->{memory} || 512;
1075     my $swap = $conf->{swap} // 0;
1076
1077     my $lxcmem = int($memory*1024*1024);
1078     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1079
1080     my $lxcswap = int(($memory + $swap)*1024*1024);
1081     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1082
1083     if (my $cpulimit = $conf->{cpulimit}) {
1084         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1085         my $value = int(100000*$cpulimit);
1086         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1087     }
1088
1089     my $shares = $conf->{cpuunits} || 1024;
1090     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1091
1092     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1093     $mountpoint->{mp} = '/';
1094
1095     my ($path, $use_loopdev) = mountpoint_mount_path($mountpoint, $storage_cfg);
1096     $path = "loop:$path" if $use_loopdev;
1097
1098     $raw .= "lxc.rootfs = $path\n";
1099
1100     my $netcount = 0;
1101     foreach my $k (keys %$conf) {
1102         next if $k !~ m/^net(\d+)$/;
1103         my $ind = $1;
1104         my $d = parse_lxc_network($conf->{$k});
1105         $netcount++;
1106         $raw .= "lxc.network.type = veth\n";
1107         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1108         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1109         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1110         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1111     }
1112
1113     if (my $lxcconf = $conf->{lxc}) {
1114         foreach my $entry (@$lxcconf) {
1115             my ($k, $v) = @$entry;
1116             $netcount++ if $k eq 'lxc.network.type';
1117             $raw .= "$k = $v\n";
1118         }
1119     }
1120
1121     $raw .= "lxc.network.type = empty\n" if !$netcount;
1122
1123     File::Path::mkpath("$dir/rootfs");
1124
1125     PVE::Tools::file_set_contents("$dir/config", $raw);
1126 }
1127
1128 # verify and cleanup nameserver list (replace \0 with ' ')
1129 sub verify_nameserver_list {
1130     my ($nameserver_list) = @_;
1131
1132     my @list = ();
1133     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1134         PVE::JSONSchema::pve_verify_ip($server);
1135         push @list, $server;
1136     }
1137
1138     return join(' ', @list);
1139 }
1140
1141 sub verify_searchdomain_list {
1142     my ($searchdomain_list) = @_;
1143
1144     my @list = ();
1145     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1146         # todo: should we add checks for valid dns domains?
1147         push @list, $server;
1148     }
1149
1150     return join(' ', @list);
1151 }
1152
1153 sub update_pct_config {
1154     my ($vmid, $conf, $running, $param, $delete) = @_;
1155
1156     my @nohotplug;
1157
1158     my $new_disks = 0;
1159
1160     my $rootdir;
1161     if ($running) {
1162         my $pid = find_lxc_pid($vmid);
1163         $rootdir = "/proc/$pid/root";
1164     }
1165
1166     if (defined($delete)) {
1167         foreach my $opt (@$delete) {
1168             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1169                 die "unable to delete required option '$opt'\n";
1170             } elsif ($opt eq 'swap') {
1171                 delete $conf->{$opt};
1172                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1173             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1174                 delete $conf->{$opt};
1175             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1176                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1177                 delete $conf->{$opt};
1178                 push @nohotplug, $opt;
1179                 next if $running;
1180             } elsif ($opt =~ m/^net(\d)$/) {
1181                 delete $conf->{$opt};
1182                 next if !$running;
1183                 my $netid = $1;
1184                 PVE::Network::veth_delete("veth${vmid}i$netid");
1185             } elsif ($opt eq 'protection') {
1186                 delete $conf->{$opt};
1187             } elsif ($opt =~ m/^mp(\d+)$/) {
1188                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1189                 delete $conf->{$opt};
1190                 push @nohotplug, $opt;
1191                 next if $running;
1192             } else {
1193                 die "implement me"
1194             }
1195             write_config($vmid, $conf) if $running;
1196         }
1197     }
1198
1199     # There's no separate swap size to configure, there's memory and "total"
1200     # memory (iow. memory+swap). This means we have to change them together.
1201     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1202     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1203     if (defined($wanted_memory) || defined($wanted_swap)) {
1204
1205         $wanted_memory //= ($conf->{memory} || 512);
1206         $wanted_swap //=  ($conf->{swap} || 0);
1207
1208         my $total = $wanted_memory + $wanted_swap;
1209         if ($running) {
1210             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1211             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1212         }
1213         $conf->{memory} = $wanted_memory;
1214         $conf->{swap} = $wanted_swap;
1215
1216         write_config($vmid, $conf) if $running;
1217     }
1218
1219     foreach my $opt (keys %$param) {
1220         my $value = $param->{$opt};
1221         if ($opt eq 'hostname') {
1222             $conf->{$opt} = $value;
1223         } elsif ($opt eq 'onboot') {
1224             $conf->{$opt} = $value ? 1 : 0;
1225         } elsif ($opt eq 'startup') {
1226             $conf->{$opt} = $value;
1227         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1228             $conf->{$opt} = $value;
1229             push @nohotplug, $opt;
1230             next if $running;
1231         } elsif ($opt eq 'nameserver') {
1232             my $list = verify_nameserver_list($value);
1233             $conf->{$opt} = $list;
1234             push @nohotplug, $opt;
1235             next if $running;
1236         } elsif ($opt eq 'searchdomain') {
1237             my $list = verify_searchdomain_list($value);
1238             $conf->{$opt} = $list;
1239             push @nohotplug, $opt;
1240             next if $running;
1241         } elsif ($opt eq 'cpulimit') {
1242             $conf->{$opt} = $value;
1243             push @nohotplug, $opt; # fixme: hotplug
1244             next;
1245         } elsif ($opt eq 'cpuunits') {
1246             $conf->{$opt} = $value;
1247             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1248         } elsif ($opt eq 'description') {
1249             $conf->{$opt} = PVE::Tools::encode_text($value);
1250         } elsif ($opt =~ m/^net(\d+)$/) {
1251             my $netid = $1;
1252             my $net = parse_lxc_network($value);
1253             if (!$running) {
1254                 $conf->{$opt} = print_lxc_network($net);
1255             } else {
1256                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1257             }
1258         } elsif ($opt eq 'protection') {
1259             $conf->{$opt} = $value ? 1 : 0;
1260         } elsif ($opt =~ m/^mp(\d+)$/) {
1261             check_protection($conf, "can't update CT $vmid drive '$opt'");
1262             $conf->{$opt} = $value;
1263             $new_disks = 1;
1264             push @nohotplug, $opt;
1265             next;
1266         } elsif ($opt eq 'rootfs') {
1267             check_protection($conf, "can't update CT $vmid drive '$opt'");
1268             die "implement me: $opt";
1269         } else {
1270             die "implement me: $opt";
1271         }
1272         write_config($vmid, $conf) if $running;
1273     }
1274
1275     if ($running && scalar(@nohotplug)) {
1276         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1277     }
1278
1279     if ($new_disks) {
1280         my $storage_cfg = PVE::Storage::config();
1281         create_disks($storage_cfg, $vmid, $conf, $conf);
1282     }
1283 }
1284
1285 sub has_dev_console {
1286     my ($conf) = @_;
1287
1288     return !(defined($conf->{console}) && !$conf->{console});
1289 }
1290
1291 sub get_tty_count {
1292     my ($conf) = @_;
1293
1294     return $conf->{tty} // $confdesc->{tty}->{default};
1295 }
1296
1297 sub get_cmode {
1298     my ($conf) = @_;
1299
1300     return $conf->{cmode} // $confdesc->{cmode}->{default};
1301 }
1302
1303 sub get_console_command {
1304     my ($vmid, $conf) = @_;
1305
1306     my $cmode = get_cmode($conf);
1307
1308     if ($cmode eq 'console') {
1309         return ['lxc-console', '-n',  $vmid, '-t', 0];
1310     } elsif ($cmode eq 'tty') {
1311         return ['lxc-console', '-n',  $vmid];
1312     } elsif ($cmode eq 'shell') {
1313         return ['lxc-attach', '--clear-env', '-n', $vmid];
1314     } else {
1315         die "internal error";
1316     }
1317 }
1318
1319 sub get_primary_ips {
1320     my ($conf) = @_;
1321
1322     # return data from net0
1323
1324     return undef if !defined($conf->{net0});
1325     my $net = parse_lxc_network($conf->{net0});
1326
1327     my $ipv4 = $net->{ip};
1328     if ($ipv4) {
1329         if ($ipv4 =~ /^(dhcp|manual)$/) {
1330             $ipv4 = undef
1331         } else {
1332             $ipv4 =~ s!/\d+$!!;
1333         }
1334     }
1335     my $ipv6 = $net->{ip6};
1336     if ($ipv6) {
1337         if ($ipv6 =~ /^(dhcp|manual)$/) {
1338             $ipv6 = undef;
1339         } else {
1340             $ipv6 =~ s!/\d+$!!;
1341         }
1342     }
1343
1344     return ($ipv4, $ipv6);
1345 }
1346
1347
1348 sub destroy_lxc_container {
1349     my ($storage_cfg, $vmid, $conf) = @_;
1350
1351     foreach_mountpoint($conf, sub {
1352         my ($ms, $mountpoint) = @_;
1353
1354         # skip bind mounts and block devices
1355         if ($mountpoint->{volume} =~ m|^/|) {
1356                 return;
1357         }
1358
1359         my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $mountpoint->{volume});
1360         PVE::Storage::vdisk_free($storage_cfg, $mountpoint->{volume}) if $vmid == $owner;
1361     });
1362
1363     rmdir "/var/lib/lxc/$vmid/rootfs";
1364     unlink "/var/lib/lxc/$vmid/config";
1365     rmdir "/var/lib/lxc/$vmid";
1366     destroy_config($vmid);
1367
1368     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1369     #PVE::Tools::run_command($cmd);
1370 }
1371
1372 sub vm_stop_cleanup {
1373     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1374
1375     eval {
1376         if (!$keepActive) {
1377
1378             my $vollist = get_vm_volumes($conf);
1379             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1380         }
1381     };
1382     warn $@ if $@; # avoid errors - just warn
1383 }
1384
1385 my $safe_num_ne = sub {
1386     my ($a, $b) = @_;
1387
1388     return 0 if !defined($a) && !defined($b);
1389     return 1 if !defined($a);
1390     return 1 if !defined($b);
1391
1392     return $a != $b;
1393 };
1394
1395 my $safe_string_ne = sub {
1396     my ($a, $b) = @_;
1397
1398     return 0 if !defined($a) && !defined($b);
1399     return 1 if !defined($a);
1400     return 1 if !defined($b);
1401
1402     return $a ne $b;
1403 };
1404
1405 sub update_net {
1406     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1407
1408     if ($newnet->{type} ne 'veth') {
1409         # for when there are physical interfaces
1410         die "cannot update interface of type $newnet->{type}";
1411     }
1412
1413     my $veth = "veth${vmid}i${netid}";
1414     my $eth = $newnet->{name};
1415
1416     if (my $oldnetcfg = $conf->{$opt}) {
1417         my $oldnet = parse_lxc_network($oldnetcfg);
1418
1419         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1420             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1421
1422             PVE::Network::veth_delete($veth);
1423             delete $conf->{$opt};
1424             write_config($vmid, $conf);
1425
1426             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1427
1428         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1429                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1430                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1431
1432                 if ($oldnet->{bridge}) {
1433                     PVE::Network::tap_unplug($veth);
1434                     foreach (qw(bridge tag firewall)) {
1435                         delete $oldnet->{$_};
1436                     }
1437                     $conf->{$opt} = print_lxc_network($oldnet);
1438                     write_config($vmid, $conf);
1439                 }
1440
1441                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1442                 foreach (qw(bridge tag firewall)) {
1443                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1444                 }
1445                 $conf->{$opt} = print_lxc_network($oldnet);
1446                 write_config($vmid, $conf);
1447         }
1448     } else {
1449         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1450     }
1451
1452     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1453 }
1454
1455 sub hotplug_net {
1456     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1457
1458     my $veth = "veth${vmid}i${netid}";
1459     my $vethpeer = $veth . "p";
1460     my $eth = $newnet->{name};
1461
1462     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1463     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1464
1465     # attach peer in container
1466     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1467     PVE::Tools::run_command($cmd);
1468
1469     # link up peer in container
1470     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1471     PVE::Tools::run_command($cmd);
1472
1473     my $done = { type => 'veth' };
1474     foreach (qw(bridge tag firewall hwaddr name)) {
1475         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1476     }
1477     $conf->{$opt} = print_lxc_network($done);
1478
1479     write_config($vmid, $conf);
1480 }
1481
1482 sub update_ipconfig {
1483     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1484
1485     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1486
1487     my $optdata = parse_lxc_network($conf->{$opt});
1488     my $deleted = [];
1489     my $added = [];
1490     my $nscmd = sub {
1491         my $cmdargs = shift;
1492         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1493     };
1494     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1495
1496     my $change_ip_config = sub {
1497         my ($ipversion) = @_;
1498
1499         my $family_opt = "-$ipversion";
1500         my $suffix = $ipversion == 4 ? '' : $ipversion;
1501         my $gw= "gw$suffix";
1502         my $ip= "ip$suffix";
1503
1504         my $newip = $newnet->{$ip};
1505         my $newgw = $newnet->{$gw};
1506         my $oldip = $optdata->{$ip};
1507
1508         my $change_ip = &$safe_string_ne($oldip, $newip);
1509         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1510
1511         return if !$change_ip && !$change_gw;
1512
1513         # step 1: add new IP, if this fails we cancel
1514         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1515             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1516             if (my $err = $@) {
1517                 warn $err;
1518                 return;
1519             }
1520         }
1521
1522         # step 2: replace gateway
1523         #   If this fails we delete the added IP and cancel.
1524         #   If it succeeds we save the config and delete the old IP, ignoring
1525         #   errors. The config is then saved.
1526         # Note: 'ip route replace' can add
1527         if ($change_gw) {
1528             if ($newgw) {
1529                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1530                 if (my $err = $@) {
1531                     warn $err;
1532                     # the route was not replaced, the old IP is still available
1533                     # rollback (delete new IP) and cancel
1534                     if ($change_ip) {
1535                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1536                         warn $@ if $@; # no need to die here
1537                     }
1538                     return;
1539                 }
1540             } else {
1541                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1542                 # if the route was not deleted, the guest might have deleted it manually
1543                 # warn and continue
1544                 warn $@ if $@;
1545             }
1546         }
1547
1548         # from this point on we save the configuration
1549         # step 3: delete old IP ignoring errors
1550         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1551             # We need to enable promote_secondaries, otherwise our newly added
1552             # address will be removed along with the old one.
1553             my $promote = 0;
1554             eval {
1555                 if ($ipversion == 4) {
1556                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1557                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1558                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1559                 }
1560                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1561             };
1562             warn $@ if $@; # no need to die here
1563
1564             if ($ipversion == 4) {
1565                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1566             }
1567         }
1568
1569         foreach my $property ($ip, $gw) {
1570             if ($newnet->{$property}) {
1571                 $optdata->{$property} = $newnet->{$property};
1572             } else {
1573                 delete $optdata->{$property};
1574             }
1575         }
1576         $conf->{$opt} = print_lxc_network($optdata);
1577         write_config($vmid, $conf);
1578         $lxc_setup->setup_network($conf);
1579     };
1580
1581     &$change_ip_config(4);
1582     &$change_ip_config(6);
1583
1584 }
1585
1586 # Internal snapshots
1587
1588 # NOTE: Snapshot create/delete involves several non-atomic
1589 # action, and can take a long time.
1590 # So we try to avoid locking the file and use 'lock' variable
1591 # inside the config file instead.
1592
1593 my $snapshot_copy_config = sub {
1594     my ($source, $dest) = @_;
1595
1596     foreach my $k (keys %$source) {
1597         next if $k eq 'snapshots';
1598         next if $k eq 'snapstate';
1599         next if $k eq 'snaptime';
1600         next if $k eq 'vmstate';
1601         next if $k eq 'lock';
1602         next if $k eq 'digest';
1603         next if $k eq 'description';
1604
1605         $dest->{$k} = $source->{$k};
1606     }
1607 };
1608
1609 my $snapshot_prepare = sub {
1610     my ($vmid, $snapname, $comment) = @_;
1611
1612     my $snap;
1613
1614     my $updatefn =  sub {
1615
1616         my $conf = load_config($vmid);
1617
1618         die "you can't take a snapshot if it's a template\n"
1619             if is_template($conf);
1620
1621         check_lock($conf);
1622
1623         $conf->{lock} = 'snapshot';
1624
1625         die "snapshot name '$snapname' already used\n"
1626             if defined($conf->{snapshots}->{$snapname});
1627
1628         my $storecfg = PVE::Storage::config();
1629         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1630
1631         $snap = $conf->{snapshots}->{$snapname} = {};
1632
1633         &$snapshot_copy_config($conf, $snap);
1634
1635         $snap->{'snapstate'} = "prepare";
1636         $snap->{'snaptime'} = time();
1637         $snap->{'description'} = $comment if $comment;
1638         $conf->{snapshots}->{$snapname} = $snap;
1639
1640         write_config($vmid, $conf);
1641     };
1642
1643     lock_container($vmid, 10, $updatefn);
1644
1645     return $snap;
1646 };
1647
1648 my $snapshot_commit = sub {
1649     my ($vmid, $snapname) = @_;
1650
1651     my $updatefn = sub {
1652
1653         my $conf = load_config($vmid);
1654
1655         die "missing snapshot lock\n"
1656             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1657
1658         die "snapshot '$snapname' does not exist\n"
1659             if !defined($conf->{snapshots}->{$snapname});
1660
1661         die "wrong snapshot state\n"
1662             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1663                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1664
1665         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1666         delete $conf->{lock};
1667         $conf->{parent} = $snapname;
1668
1669         write_config($vmid, $conf);
1670     };
1671
1672     lock_container($vmid, 10 ,$updatefn);
1673 };
1674
1675 sub has_feature {
1676     my ($feature, $conf, $storecfg, $snapname) = @_;
1677
1678     my $err;
1679
1680     foreach_mountpoint($conf, sub {
1681         my ($ms, $mountpoint) = @_;
1682
1683         return if $err; # skip further test
1684
1685         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1686
1687         # TODO: implement support for mountpoints
1688         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1689             if $ms ne 'rootfs';
1690     });
1691
1692     return $err ? 0 : 1;
1693 }
1694
1695 sub snapshot_create {
1696     my ($vmid, $snapname, $comment) = @_;
1697
1698     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1699
1700     my $conf = load_config($vmid);
1701
1702     my $running = check_running($vmid);
1703     eval {
1704         if ($running) {
1705             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1706             PVE::Tools::run_command(['/bin/sync']);
1707         };
1708
1709         my $storecfg = PVE::Storage::config();
1710         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1711         my $volid = $rootinfo->{volume};
1712
1713         if ($running) {
1714             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1715         };
1716
1717         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1718         &$snapshot_commit($vmid, $snapname);
1719     };
1720     if(my $err = $@) {
1721         snapshot_delete($vmid, $snapname, 1);
1722         die "$err\n";
1723     }
1724 }
1725
1726 sub snapshot_delete {
1727     my ($vmid, $snapname, $force) = @_;
1728
1729     my $snap;
1730
1731     my $conf;
1732
1733     my $updatefn =  sub {
1734
1735         $conf = load_config($vmid);
1736
1737         die "you can't delete a snapshot if vm is a template\n"
1738             if is_template($conf);
1739
1740         $snap = $conf->{snapshots}->{$snapname};
1741
1742         check_lock($conf);
1743
1744         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1745
1746         $snap->{snapstate} = 'delete';
1747
1748         write_config($vmid, $conf);
1749     };
1750
1751     lock_container($vmid, 10, $updatefn);
1752
1753     my $storecfg = PVE::Storage::config();
1754
1755     my $del_snap =  sub {
1756
1757         check_lock($conf);
1758
1759         if ($conf->{parent} eq $snapname) {
1760             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1761                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1762             } else {
1763                 delete $conf->{parent};
1764             }
1765         }
1766
1767         delete $conf->{snapshots}->{$snapname};
1768
1769         write_config($vmid, $conf);
1770     };
1771
1772     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1773     my $rootinfo = parse_ct_mountpoint($rootfs);
1774     my $volid = $rootinfo->{volume};
1775
1776     eval {
1777         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1778     };
1779     my $err = $@;
1780
1781     if(!$err || ($err && $force)) {
1782         lock_container($vmid, 10, $del_snap);
1783         if ($err) {
1784             die "Can't delete snapshot: $vmid $snapname $err\n";
1785         }
1786     }
1787 }
1788
1789 sub snapshot_rollback {
1790     my ($vmid, $snapname) = @_;
1791
1792     my $storecfg = PVE::Storage::config();
1793
1794     my $conf = load_config($vmid);
1795
1796     die "you can't rollback if vm is a template\n" if is_template($conf);
1797
1798     my $snap = $conf->{snapshots}->{$snapname};
1799
1800     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1801
1802     my $rootfs = $snap->{rootfs};
1803     my $rootinfo = parse_ct_mountpoint($rootfs);
1804     my $volid = $rootinfo->{volume};
1805
1806     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1807
1808     my $updatefn = sub {
1809
1810         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1811             if $snap->{snapstate};
1812
1813         check_lock($conf);
1814
1815         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1816
1817         die "unable to rollback vm $vmid: vm is running\n"
1818             if check_running($vmid);
1819
1820         $conf->{lock} = 'rollback';
1821
1822         my $forcemachine;
1823
1824         # copy snapshot config to current config
1825
1826         my $tmp_conf = $conf;
1827         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1828         $conf->{snapshots} = $tmp_conf->{snapshots};
1829         delete $conf->{snaptime};
1830         delete $conf->{snapname};
1831         $conf->{parent} = $snapname;
1832
1833         write_config($vmid, $conf);
1834     };
1835
1836     my $unlockfn = sub {
1837         delete $conf->{lock};
1838         write_config($vmid, $conf);
1839     };
1840
1841     lock_container($vmid, 10, $updatefn);
1842
1843     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1844
1845     lock_container($vmid, 5, $unlockfn);
1846 }
1847
1848 sub template_create {
1849     my ($vmid, $conf) = @_;
1850
1851     my $storecfg = PVE::Storage::config();
1852
1853     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1854     my $volid = $rootinfo->{volume};
1855
1856     die "Template feature is not available for '$volid'\n"
1857         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1858
1859     PVE::Storage::activate_volumes($storecfg, [$volid]);
1860
1861     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1862     $rootinfo->{volume} = $template_volid;
1863     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1864
1865     write_config($vmid, $conf);
1866 }
1867
1868 sub is_template {
1869     my ($conf) = @_;
1870
1871     return 1 if defined $conf->{template} && $conf->{template} == 1;
1872 }
1873
1874 sub mountpoint_names {
1875     my ($reverse) = @_;
1876
1877     my @names = ('rootfs');
1878
1879     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1880         push @names, "mp$i";
1881     }
1882
1883     return $reverse ? reverse @names : @names;
1884 }
1885
1886 # The container might have *different* symlinks than the host. realpath/abs_path
1887 # use the actual filesystem to resolve links.
1888 sub sanitize_mountpoint {
1889     my ($mp) = @_;
1890     $mp = '/' . $mp; # we always start with a slash
1891     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1892     $mp =~ s@/\./@@g; # collapse /./
1893     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1894     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1895     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1896     return $mp;
1897 }
1898
1899 sub foreach_mountpoint_full {
1900     my ($conf, $reverse, $func) = @_;
1901
1902     foreach my $key (mountpoint_names($reverse)) {
1903         my $value = $conf->{$key};
1904         next if !defined($value);
1905         my $mountpoint = parse_ct_mountpoint($value);
1906
1907         # just to be sure: rootfs is /
1908         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1909         $mountpoint->{mp} = sanitize_mountpoint($path);
1910
1911         $path = $mountpoint->{volume};
1912         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1913
1914         &$func($key, $mountpoint);
1915     }
1916 }
1917
1918 sub foreach_mountpoint {
1919     my ($conf, $func) = @_;
1920
1921     foreach_mountpoint_full($conf, 0, $func);
1922 }
1923
1924 sub foreach_mountpoint_reverse {
1925     my ($conf, $func) = @_;
1926
1927     foreach_mountpoint_full($conf, 1, $func);
1928 }
1929
1930 sub check_ct_modify_config_perm {
1931     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1932
1933     return 1 if $authuser ne 'root@pam';
1934
1935     foreach my $opt (@$key_list) {
1936
1937         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1938             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1939         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1940             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1941         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1942             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1943         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1944                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1945             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1946         } else {
1947             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1948         }
1949     }
1950
1951     return 1;
1952 }
1953
1954 sub umount_all {
1955     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
1956
1957     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1958     my $volid_list = get_vm_volumes($conf);
1959
1960     foreach_mountpoint_reverse($conf, sub {
1961         my ($ms, $mountpoint) = @_;
1962
1963         my $volid = $mountpoint->{volume};
1964         my $mount = $mountpoint->{mp};
1965
1966         return if !$volid || !$mount;
1967
1968         my $mount_path = "$rootdir/$mount";
1969         $mount_path =~ s!/+!/!g;
1970
1971         return if !PVE::ProcFSTools::is_mounted($mount_path);
1972
1973         eval {
1974             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1975         };
1976         if (my $err = $@) {
1977             if ($noerr) {
1978                 warn $err;
1979             } else {
1980                 die $err;
1981             }
1982         }
1983     });
1984 }
1985
1986 sub mount_all {
1987     my ($vmid, $storage_cfg, $conf) = @_;
1988
1989     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1990     File::Path::make_path($rootdir);
1991
1992     my $volid_list = get_vm_volumes($conf);
1993     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
1994
1995     eval {
1996         foreach_mountpoint($conf, sub {
1997             my ($ms, $mountpoint) = @_;
1998
1999             my $volid = $mountpoint->{volume};
2000             my $mount = $mountpoint->{mp};
2001
2002             return if !$volid || !$mount;
2003
2004             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2005             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2006                 PVE::Storage::parse_volname($storage_cfg, $volid);
2007
2008             die "unable to mount base volume - internal error" if $isBase;
2009
2010             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2011         });
2012     };
2013     if (my $err = $@) {
2014         warn "mounting container failed - $err";
2015         umount_all($vmid, $storage_cfg, $conf, 1);
2016     }
2017
2018     return $rootdir;
2019 }
2020
2021
2022 sub mountpoint_mount_path {
2023     my ($mountpoint, $storage_cfg, $snapname) = @_;
2024
2025     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2026 }
2027
2028 my $check_mount_path = sub {
2029     my ($path) = @_;
2030     $path = File::Spec->canonpath($path);
2031     my $real = Cwd::realpath($path);
2032     if ($real ne $path) {
2033         die "mount path modified by symlink: $path != $real";
2034     }
2035 };
2036
2037 # use $rootdir = undef to just return the corresponding mount path
2038 sub mountpoint_mount {
2039     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2040
2041     my $volid = $mountpoint->{volume};
2042     my $mount = $mountpoint->{mp};
2043
2044     return if !$volid || !$mount;
2045
2046     my $mount_path;
2047
2048     if (defined($rootdir)) {
2049         $rootdir =~ s!/+$!!;
2050         $mount_path = "$rootdir/$mount";
2051         $mount_path =~ s!/+!/!g;
2052         &$check_mount_path($mount_path);
2053         File::Path::mkpath($mount_path);
2054     }
2055
2056     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2057
2058     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2059
2060     if ($storage) {
2061
2062         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2063         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2064
2065         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2066             PVE::Storage::parse_volname($storage_cfg, $volid);
2067
2068         if ($format eq 'subvol') {
2069             if ($mount_path) {
2070                 if ($snapname) {
2071                     if ($scfg->{type} eq 'zfspool') {
2072                         my $path_arg = $path;
2073                         $path_arg =~ s!^/+!!;
2074                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', '-t', 'zfs', $path_arg, $mount_path]);
2075                     } else {
2076                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2077                     }
2078                 } else {
2079                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2080                 }
2081             }
2082             return wantarray ? ($path, 0) : $path;
2083         } elsif ($format eq 'raw') {
2084             my $use_loopdev = 0;
2085             my @extra_opts;
2086             if ($scfg->{path}) {
2087                 push @extra_opts, '-o', 'loop';
2088                 $use_loopdev = 1;
2089             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2090                 # do nothing
2091             } else {
2092                 die "unsupported storage type '$scfg->{type}'\n";
2093             }
2094             if ($mount_path) {
2095                 if ($isBase || defined($snapname)) {
2096                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2097                 } else {
2098                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2099                 }
2100             }
2101             return wantarray ? ($path, $use_loopdev) : $path;
2102         } else {
2103             die "unsupported image format '$format'\n";
2104         }
2105     } elsif ($volid =~ m|^/dev/.+|) {
2106         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2107         return wantarray ? ($volid, 0) : $volid;
2108     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2109         &$check_mount_path($volid);
2110         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2111         return wantarray ? ($volid, 0) : $volid;
2112     }
2113
2114     die "unsupported storage";
2115 }
2116
2117 sub get_vm_volumes {
2118     my ($conf, $excludes) = @_;
2119
2120     my $vollist = [];
2121
2122     foreach_mountpoint($conf, sub {
2123         my ($ms, $mountpoint) = @_;
2124
2125         return if $excludes && $ms eq $excludes;
2126
2127         my $volid = $mountpoint->{volume};
2128
2129         return if !$volid || $volid =~ m|^/|;
2130
2131         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2132         return if !$sid;
2133
2134         push @$vollist, $volid;
2135     });
2136
2137     return $vollist;
2138 }
2139
2140 sub mkfs {
2141     my ($dev) = @_;
2142
2143     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2144 }
2145
2146 sub format_disk {
2147     my ($storage_cfg, $volid) = @_;
2148
2149     if ($volid =~ m!^/dev/.+!) {
2150         mkfs($volid);
2151         return;
2152     }
2153
2154     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2155
2156     die "cannot format volume '$volid' with no storage\n" if !$storage;
2157
2158     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2159
2160     my $path = PVE::Storage::path($storage_cfg, $volid);
2161
2162     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2163         PVE::Storage::parse_volname($storage_cfg, $volid);
2164
2165     die "cannot format volume '$volid' (format == $format)\n"
2166         if $format ne 'raw';
2167
2168     mkfs($path);
2169 }
2170
2171 sub destroy_disks {
2172     my ($storecfg, $vollist) = @_;
2173
2174     foreach my $volid (@$vollist) {
2175         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2176         warn $@ if $@;
2177     }
2178 }
2179
2180 sub create_disks {
2181     my ($storecfg, $vmid, $settings, $conf) = @_;
2182
2183     my $vollist = [];
2184
2185     eval {
2186         foreach_mountpoint($settings, sub {
2187             my ($ms, $mountpoint) = @_;
2188
2189             my $volid = $mountpoint->{volume};
2190             my $mp = $mountpoint->{mp};
2191
2192             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2193
2194             return if !$storage;
2195
2196             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2197                 my ($storeid, $size_gb) = ($1, $2);
2198
2199                 my $size_kb = int(${size_gb}*1024) * 1024;
2200
2201                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2202                 # fixme: use better naming ct-$vmid-disk-X.raw?
2203
2204                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2205                     if ($size_kb > 0) {
2206                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2207                                                            undef, $size_kb);
2208                         format_disk($storecfg, $volid);
2209                     } else {
2210                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2211                                                            undef, 0);
2212                     }
2213                 } elsif ($scfg->{type} eq 'zfspool') {
2214
2215                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2216                                                        undef, $size_kb);
2217                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2218
2219                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2220                     format_disk($storecfg, $volid);
2221
2222                 } elsif ($scfg->{type} eq 'rbd') {
2223
2224                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2225                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2226                     format_disk($storecfg, $volid);
2227                 } else {
2228                     die "unable to create containers on storage type '$scfg->{type}'\n";
2229                 }
2230                 push @$vollist, $volid;
2231                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2232                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2233             } else {
2234                 # use specified/existing volid
2235             }
2236         });
2237     };
2238     # free allocated images on error
2239     if (my $err = $@) {
2240         destroy_disks($storecfg, $vollist);
2241         die $err;
2242     }
2243     return $vollist;
2244 }
2245
2246 # bash completion helper
2247
2248 sub complete_os_templates {
2249     my ($cmdname, $pname, $cvalue) = @_;
2250
2251     my $cfg = PVE::Storage::config();
2252
2253     my $storeid;
2254
2255     if ($cvalue =~ m/^([^:]+):/) {
2256         $storeid = $1;
2257     }
2258
2259     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2260     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2261
2262     my $res = [];
2263     foreach my $id (keys %$data) {
2264         foreach my $item (@{$data->{$id}}) {
2265             push @$res, $item->{volid} if defined($item->{volid});
2266         }
2267     }
2268
2269     return $res;
2270 }
2271
2272 my $complete_ctid_full = sub {
2273     my ($running) = @_;
2274
2275     my $idlist = vmstatus();
2276
2277     my $active_hash = list_active_containers();
2278
2279     my $res = [];
2280
2281     foreach my $id (keys %$idlist) {
2282         my $d = $idlist->{$id};
2283         if (defined($running)) {
2284             next if $d->{template};
2285             next if $running && !$active_hash->{$id};
2286             next if !$running && $active_hash->{$id};
2287         }
2288         push @$res, $id;
2289
2290     }
2291     return $res;
2292 };
2293
2294 sub complete_ctid {
2295     return &$complete_ctid_full();
2296 }
2297
2298 sub complete_ctid_stopped {
2299     return &$complete_ctid_full(0);
2300 }
2301
2302 sub complete_ctid_running {
2303     return &$complete_ctid_full(1);
2304 }
2305
2306 1;