src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21
  22 use Data::Dumper;
  23
  24 my $nodename = PVE::INotify::nodename();
  25
  26 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  27
  28 my $rootfs_desc = {
  29     volume => {
  30         type => 'string',
  31         default_key => 1,
  32         format_description => 'volume',
  33         description => 'Volume, device or directory to mount into the container.',
  34     },
  35     backup => {
  36         type => 'boolean',
  37         format_description => '[1|0]',
  38         description => 'Whether to include the mountpoint in backups.',
  39         optional => 1,
  40     },
  41     size => {
  42         type => 'string',
  43         format_description => 'DiskSize',
  44         pattern => '\d+[TGMK]?',
  45         description => 'Volume size (read only value).',
  46         optional => 1,
  47     },
  48 };
  49
  50 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  51     type => 'string', format => $rootfs_desc,
  52     description => "Use volume as container root.",
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  57     description => "The name of the snapshot.",
  58     type => 'string', format => 'pve-configid',
  59     maxLength => 40,
  60 });
  61
  62 my $confdesc = {
  63     lock => {
  64         optional => 1,
  65         type => 'string',
  66         description => "Lock/unlock the VM.",
  67         enum => [qw(migrate backup snapshot rollback)],
  68     },
  69     onboot => {
  70         optional => 1,
  71         type => 'boolean',
  72         description => "Specifies whether a VM will be started during system bootup.",
  73         default => 0,
  74     },
  75     startup => get_standard_option('pve-startup-order'),
  76     template => {
  77         optional => 1,
  78         type => 'boolean',
  79         description => "Enable/disable Template.",
  80         default => 0,
  81     },
  82     arch => {
  83         optional => 1,
  84         type => 'string',
  85         enum => ['amd64', 'i386'],
  86         description => "OS architecture type.",
  87         default => 'amd64',
  88     },
  89     ostype => {
  90         optional => 1,
  91         type => 'string',
  92         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  93         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  94     },
  95     console => {
  96         optional => 1,
  97         type => 'boolean',
  98         description => "Attach a console device (/dev/console) to the container.",
  99         default => 1,
 100     },
 101     tty => {
 102         optional => 1,
 103         type => 'integer',
 104         description => "Specify the number of tty available to the container",
 105         minimum => 0,
 106         maximum => 6,
 107         default => 2,
 108     },
 109     cpulimit => {
 110         optional => 1,
 111         type => 'number',
 112         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 113         minimum => 0,
 114         maximum => 128,
 115         default => 0,
 116     },
 117     cpuunits => {
 118         optional => 1,
 119         type => 'integer',
 120         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 121         minimum => 0,
 122         maximum => 500000,
 123         default => 1024,
 124     },
 125     memory => {
 126         optional => 1,
 127         type => 'integer',
 128         description => "Amount of RAM for the VM in MB.",
 129         minimum => 16,
 130         default => 512,
 131     },
 132     swap => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "Amount of SWAP for the VM in MB.",
 136         minimum => 0,
 137         default => 512,
 138     },
 139     hostname => {
 140         optional => 1,
 141         description => "Set a host name for the container.",
 142         type => 'string', format => 'dns-name',
 143         maxLength => 255,
 144     },
 145     description => {
 146         optional => 1,
 147         type => 'string',
 148         description => "Container description. Only used on the configuration web interface.",
 149     },
 150     searchdomain => {
 151         optional => 1,
 152         type => 'string', format => 'dns-name-list',
 153         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 154     },
 155     nameserver => {
 156         optional => 1,
 157         type => 'string', format => 'address-list',
 158         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 159     },
 160     rootfs => get_standard_option('pve-ct-rootfs'),
 161     parent => {
 162         optional => 1,
 163         type => 'string', format => 'pve-configid',
 164         maxLength => 40,
 165         description => "Parent snapshot name. This is used internally, and should not be modified.",
 166     },
 167     snaptime => {
 168         optional => 1,
 169         description => "Timestamp for snapshots.",
 170         type => 'integer',
 171         minimum => 0,
 172     },
 173     cmode => {
 174         optional => 1,
 175         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 176         type => 'string',
 177         enum => ['shell', 'console', 'tty'],
 178         default => 'tty',
 179     },
 180     protection => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 184         default => 0,
 185     },
 186 };
 187
 188 my $valid_lxc_conf_keys = {
 189     'lxc.include' => 1,
 190     'lxc.arch' => 1,
 191     'lxc.utsname' => 1,
 192     'lxc.haltsignal' => 1,
 193     'lxc.rebootsignal' => 1,
 194     'lxc.stopsignal' => 1,
 195     'lxc.init_cmd' => 1,
 196     'lxc.network.type' => 1,
 197     'lxc.network.flags' => 1,
 198     'lxc.network.link' => 1,
 199     'lxc.network.mtu' => 1,
 200     'lxc.network.name' => 1,
 201     'lxc.network.hwaddr' => 1,
 202     'lxc.network.ipv4' => 1,
 203     'lxc.network.ipv4.gateway' => 1,
 204     'lxc.network.ipv6' => 1,
 205     'lxc.network.ipv6.gateway' => 1,
 206     'lxc.network.script.up' => 1,
 207     'lxc.network.script.down' => 1,
 208     'lxc.pts' => 1,
 209     'lxc.console.logfile' => 1,
 210     'lxc.console' => 1,
 211     'lxc.tty' => 1,
 212     'lxc.devttydir' => 1,
 213     'lxc.hook.autodev' => 1,
 214     'lxc.autodev' => 1,
 215     'lxc.kmsg' => 1,
 216     'lxc.mount' => 1,
 217     'lxc.mount.entry' => 1,
 218     'lxc.mount.auto' => 1,
 219     'lxc.rootfs' => 1,
 220     'lxc.rootfs.mount' => 1,
 221     'lxc.rootfs.options' => 1,
 222     # lxc.cgroup.*
 223     'lxc.cap.drop' => 1,
 224     'lxc.cap.keep' => 1,
 225     'lxc.aa_profile' => 1,
 226     'lxc.aa_allow_incomplete' => 1,
 227     'lxc.se_context' => 1,
 228     'lxc.seccomp' => 1,
 229     'lxc.id_map' => 1,
 230     'lxc.hook.pre-start' => 1,
 231     'lxc.hook.pre-mount' => 1,
 232     'lxc.hook.mount' => 1,
 233     'lxc.hook.start' => 1,
 234     'lxc.hook.post-stop' => 1,
 235     'lxc.hook.clone' => 1,
 236     'lxc.hook.destroy' => 1,
 237     'lxc.loglevel' => 1,
 238     'lxc.logfile' => 1,
 239     'lxc.start.auto' => 1,
 240     'lxc.start.delay' => 1,
 241     'lxc.start.order' => 1,
 242     'lxc.group' => 1,
 243     'lxc.environment' => 1,
 244     'lxc.' => 1,
 245     'lxc.' => 1,
 246     'lxc.' => 1,
 247     'lxc.' => 1,
 248 };
 249
 250 my $netconf_desc = {
 251     type => {
 252         type => 'string',
 253         optional => 1,
 254         description => "Network interface type.",
 255         enum => [qw(veth)],
 256     },
 257     name => {
 258         type => 'string',
 259         format_description => 'String',
 260         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 261         pattern => '[-_.\w\d]+',
 262     },
 263     bridge => {
 264         type => 'string',
 265         format_description => 'vmbr<Number>',
 266         description => 'Bridge to attach the network device to.',
 267         pattern => '[-_.\w\d]+',
 268     },
 269     hwaddr => {
 270         type => 'string',
 271         format_description => 'MAC',
 272         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 273         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 274         optional => 1,
 275     },
 276     mtu => {
 277         type => 'integer',
 278         format_description => 'Number',
 279         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 280         optional => 1,
 281     },
 282     ip => {
 283         type => 'string',
 284         format => 'pve-ipv4-config',
 285         format_description => 'IPv4Format/CIDR',
 286         description => 'IPv4 address in CIDR format.',
 287         optional => 1,
 288     },
 289     gw => {
 290         type => 'string',
 291         format => 'ipv4',
 292         format_description => 'GatewayIPv4',
 293         description => 'Default gateway for IPv4 traffic.',
 294         optional => 1,
 295     },
 296     ip6 => {
 297         type => 'string',
 298         format => 'pve-ipv6-config',
 299         format_description => 'IPv6Format/CIDR',
 300         description => 'IPv6 address in CIDR format.',
 301         optional => 1,
 302     },
 303     gw6 => {
 304         type => 'string',
 305         format => 'ipv6',
 306         format_description => 'GatewayIPv6',
 307         description => 'Default gateway for IPv6 traffic.',
 308         optional => 1,
 309     },
 310     firewall => {
 311         type => 'boolean',
 312         format_description => '[1|0]',
 313         description => "Controls whether this interface's firewall rules should be used.",
 314         optional => 1,
 315     },
 316     tag => {
 317         type => 'integer',
 318         format_description => 'VlanNo',
 319         minimum => '2',
 320         maximum => '4094',
 321         description => "VLAN tag foro this interface.",
 322         optional => 1,
 323     },
 324 };
 325 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 326
 327 my $MAX_LXC_NETWORKS = 10;
 328 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 329     $confdesc->{"net$i"} = {
 330         optional => 1,
 331         type => 'string', format => $netconf_desc,
 332         description => "Specifies network interfaces for the container.",
 333     };
 334 }
 335
 336 my $mp_desc = {
 337     %$rootfs_desc,
 338     mp => {
 339         type => 'string',
 340         format_description => 'Path',
 341         description => 'Path to the mountpoint as seen from inside the container.',
 342         optional => 1,
 343     },
 344 };
 345 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 346
 347 my $MAX_MOUNT_POINTS = 10;
 348 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 349     $confdesc->{"mp$i"} = {
 350         optional => 1,
 351         type => 'string', format => $mp_desc,
 352         description => "Use volume as container mount point (experimental feature).",
 353         optional => 1,
 354     };
 355 }
 356
 357 sub write_pct_config {
 358     my ($filename, $conf) = @_;
 359
 360     delete $conf->{snapstate}; # just to be sure
 361
 362     my $generate_raw_config = sub {
 363         my ($conf) = @_;
 364
 365         my $raw = '';
 366
 367         # add description as comment to top of file
 368         my $descr = $conf->{description} || '';
 369         foreach my $cl (split(/\n/, $descr)) {
 370             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 371         }
 372
 373         foreach my $key (sort keys %$conf) {
 374             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 375                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 376             my $value = $conf->{$key};
 377             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 378             $raw .= "$key: $value\n";
 379         }
 380
 381         if (my $lxcconf = $conf->{lxc}) {
 382             foreach my $entry (@$lxcconf) {
 383                 my ($k, $v) = @$entry;
 384                 $raw .= "$k: $v\n";
 385             }
 386         }
 387
 388         return $raw;
 389     };
 390
 391     my $raw = &$generate_raw_config($conf);
 392
 393     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 394         $raw .= "\n[$snapname]\n";
 395         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 396     }
 397
 398     return $raw;
 399 }
 400
 401 sub check_type {
 402     my ($key, $value) = @_;
 403
 404     die "unknown setting '$key'\n" if !$confdesc->{$key};
 405
 406     my $type = $confdesc->{$key}->{type};
 407
 408     if (!defined($value)) {
 409         die "got undefined value\n";
 410     }
 411
 412     if ($value =~ m/[\n\r]/) {
 413         die "property contains a line feed\n";
 414     }
 415
 416     if ($type eq 'boolean') {
 417         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 418         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 419         die "type check ('boolean') failed - got '$value'\n";
 420     } elsif ($type eq 'integer') {
 421         return int($1) if $value =~ m/^(\d+)$/;
 422         die "type check ('integer') failed - got '$value'\n";
 423     } elsif ($type eq 'number') {
 424         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 425         die "type check ('number') failed - got '$value'\n";
 426     } elsif ($type eq 'string') {
 427         if (my $fmt = $confdesc->{$key}->{format}) {
 428             PVE::JSONSchema::check_format($fmt, $value);
 429             return $value;
 430         }
 431         return $value;
 432     } else {
 433         die "internal error"
 434     }
 435 }
 436
 437 sub parse_pct_config {
 438     my ($filename, $raw) = @_;
 439
 440     return undef if !defined($raw);
 441
 442     my $res = {
 443         digest => Digest::SHA::sha1_hex($raw),
 444         snapshots => {},
 445     };
 446
 447     $filename =~ m|/lxc/(\d+).conf$|
 448         || die "got strange filename '$filename'";
 449
 450     my $vmid = $1;
 451
 452     my $conf = $res;
 453     my $descr = '';
 454     my $section = '';
 455
 456     my @lines = split(/\n/, $raw);
 457     foreach my $line (@lines) {
 458         next if $line =~ m/^\s*$/;
 459
 460         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 461             $section = $1;
 462             $conf->{description} = $descr if $descr;
 463             $descr = '';
 464             $conf = $res->{snapshots}->{$section} = {};
 465             next;
 466         }
 467
 468         if ($line =~ m/^\#(.*)\s*$/) {
 469             $descr .= PVE::Tools::decode_text($1) . "\n";
 470             next;
 471         }
 472
 473         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 474             my $key = $1;
 475             my $value = $3;
 476             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 477                 push @{$conf->{lxc}}, [$key, $value];
 478             } else {
 479                 warn "vm $vmid - unable to parse config: $line\n";
 480             }
 481         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 482             $descr .= PVE::Tools::decode_text($2);
 483         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 484             $conf->{snapstate} = $1;
 485         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 486             my $key = $1;
 487             my $value = $2;
 488             eval { $value = check_type($key, $value); };
 489             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 490             $conf->{$key} = $value;
 491         } else {
 492             warn "vm $vmid - unable to parse config: $line\n";
 493         }
 494     }
 495
 496     $conf->{description} = $descr if $descr;
 497
 498     delete $res->{snapstate}; # just to be sure
 499
 500     return $res;
 501 }
 502
 503 sub config_list {
 504     my $vmlist = PVE::Cluster::get_vmlist();
 505     my $res = {};
 506     return $res if !$vmlist || !$vmlist->{ids};
 507     my $ids = $vmlist->{ids};
 508
 509     foreach my $vmid (keys %$ids) {
 510         next if !$vmid; # skip CT0
 511         my $d = $ids->{$vmid};
 512         next if !$d->{node} || $d->{node} ne $nodename;
 513         next if !$d->{type} || $d->{type} ne 'lxc';
 514         $res->{$vmid}->{type} = 'lxc';
 515     }
 516     return $res;
 517 }
 518
 519 sub cfs_config_path {
 520     my ($vmid, $node) = @_;
 521
 522     $node = $nodename if !$node;
 523     return "nodes/$node/lxc/$vmid.conf";
 524 }
 525
 526 sub config_file {
 527     my ($vmid, $node) = @_;
 528
 529     my $cfspath = cfs_config_path($vmid, $node);
 530     return "/etc/pve/$cfspath";
 531 }
 532
 533 sub load_config {
 534     my ($vmid, $node) = @_;
 535
 536     $node = $nodename if !$node;
 537     my $cfspath = cfs_config_path($vmid, $node);
 538
 539     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 540     die "container $vmid does not exists\n" if !defined($conf);
 541
 542     return $conf;
 543 }
 544
 545 sub create_config {
 546     my ($vmid, $conf) = @_;
 547
 548     my $dir = "/etc/pve/nodes/$nodename/lxc";
 549     mkdir $dir;
 550
 551     write_config($vmid, $conf);
 552 }
 553
 554 sub destroy_config {
 555     my ($vmid) = @_;
 556
 557     unlink config_file($vmid, $nodename);
 558 }
 559
 560 sub write_config {
 561     my ($vmid, $conf) = @_;
 562
 563     my $cfspath = cfs_config_path($vmid);
 564
 565     PVE::Cluster::cfs_write_file($cfspath, $conf);
 566 }
 567
 568 # flock: we use one file handle per process, so lock file
 569 # can be called multiple times and succeeds for the same process.
 570
 571 my $lock_handles =  {};
 572 my $lockdir = "/run/lock/lxc";
 573
 574 sub lock_filename {
 575     my ($vmid) = @_;
 576
 577     return "$lockdir/pve-config-${vmid}.lock";
 578 }
 579
 580 sub lock_aquire {
 581     my ($vmid, $timeout) = @_;
 582
 583     $timeout = 10 if !$timeout;
 584     my $mode = LOCK_EX;
 585
 586     my $filename = lock_filename($vmid);
 587
 588     mkdir $lockdir if !-d $lockdir;
 589
 590     my $lock_func = sub {
 591         if (!$lock_handles->{$$}->{$filename}) {
 592             my $fh = new IO::File(">>$filename") ||
 593                 die "can't open file - $!\n";
 594             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 595         }
 596
 597         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 598             print STDERR "trying to aquire lock...";
 599             my $success;
 600             while(1) {
 601                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 602                 # try again on EINTR (see bug #273)
 603                 if ($success || ($! != EINTR)) {
 604                     last;
 605                 }
 606             }
 607             if (!$success) {
 608                 print STDERR " failed\n";
 609                 die "can't aquire lock - $!\n";
 610             }
 611
 612             print STDERR " OK\n";
 613         }
 614
 615         $lock_handles->{$$}->{$filename}->{refcount}++;
 616     };
 617
 618     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 619     my $err = $@;
 620     if ($err) {
 621         die "can't lock file '$filename' - $err";
 622     }
 623 }
 624
 625 sub lock_release {
 626     my ($vmid) = @_;
 627
 628     my $filename = lock_filename($vmid);
 629
 630     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 631         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 632         if ($refcount <= 0) {
 633             $lock_handles->{$$}->{$filename} = undef;
 634             close ($fh);
 635         }
 636     }
 637 }
 638
 639 sub lock_container {
 640     my ($vmid, $timeout, $code, @param) = @_;
 641
 642     my $res;
 643
 644     lock_aquire($vmid, $timeout);
 645     eval { $res = &$code(@param) };
 646     my $err = $@;
 647     lock_release($vmid);
 648
 649     die $err if $err;
 650
 651     return $res;
 652 }
 653
 654 sub option_exists {
 655     my ($name) = @_;
 656
 657     return defined($confdesc->{$name});
 658 }
 659
 660 # add JSON properties for create and set function
 661 sub json_config_properties {
 662     my $prop = shift;
 663
 664     foreach my $opt (keys %$confdesc) {
 665         next if $opt eq 'parent' || $opt eq 'snaptime';
 666         next if $prop->{$opt};
 667         $prop->{$opt} = $confdesc->{$opt};
 668     }
 669
 670     return $prop;
 671 }
 672
 673 sub json_config_properties_no_rootfs {
 674     my $prop = shift;
 675
 676     foreach my $opt (keys %$confdesc) {
 677         next if $prop->{$opt};
 678         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 679         $prop->{$opt} = $confdesc->{$opt};
 680     }
 681
 682     return $prop;
 683 }
 684
 685 # container status helpers
 686
 687 sub list_active_containers {
 688
 689     my $filename = "/proc/net/unix";
 690
 691     # similar test is used by lcxcontainers.c: list_active_containers
 692     my $res = {};
 693
 694     my $fh = IO::File->new ($filename, "r");
 695     return $res if !$fh;
 696
 697     while (defined(my $line = <$fh>)) {
 698         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 699             my $path = $1;
 700             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 701                 $res->{$1} = 1;
 702             }
 703         }
 704     }
 705
 706     close($fh);
 707
 708     return $res;
 709 }
 710
 711 # warning: this is slow
 712 sub check_running {
 713     my ($vmid) = @_;
 714
 715     my $active_hash = list_active_containers();
 716
 717     return 1 if defined($active_hash->{$vmid});
 718
 719     return undef;
 720 }
 721
 722 sub get_container_disk_usage {
 723     my ($vmid) = @_;
 724
 725     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 726
 727     my $res = {
 728         total => 0,
 729         used => 0,
 730         avail => 0,
 731     };
 732
 733     my $parser = sub {
 734         my $line = shift;
 735         if (my ($fsid, $total, $used, $avail) = $line =~
 736             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 737             $res = {
 738                 total => $total,
 739                 used => $used,
 740                 avail => $avail,
 741             };
 742         }
 743     };
 744     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 745     warn $@ if $@;
 746
 747     return $res;
 748 }
 749
 750 sub vmstatus {
 751     my ($opt_vmid) = @_;
 752
 753     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 754
 755     my $active_hash = list_active_containers();
 756
 757     foreach my $vmid (keys %$list) {
 758         my $d = $list->{$vmid};
 759
 760         my $running = defined($active_hash->{$vmid});
 761
 762         $d->{status} = $running ? 'running' : 'stopped';
 763
 764         my $cfspath = cfs_config_path($vmid);
 765         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 766
 767         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 768         $d->{name} =~ s/[\s]//g;
 769
 770         $d->{cpus} = $conf->{cpulimit} // 0;
 771
 772         if ($running) {
 773             my $res = get_container_disk_usage($vmid);
 774             $d->{disk} = $res->{used};
 775             $d->{maxdisk} = $res->{total};
 776         } else {
 777             $d->{disk} = 0;
 778             # use 4GB by default ??
 779             if (my $rootfs = $conf->{rootfs}) {
 780                 my $rootinfo = parse_ct_mountpoint($rootfs);
 781                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 782             } else {
 783                 $d->{maxdisk} = 4*1024*1024*1024;
 784             }
 785         }
 786
 787         $d->{mem} = 0;
 788         $d->{swap} = 0;
 789         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 790         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 791
 792         $d->{uptime} = 0;
 793         $d->{cpu} = 0;
 794
 795         $d->{netout} = 0;
 796         $d->{netin} = 0;
 797
 798         $d->{diskread} = 0;
 799         $d->{diskwrite} = 0;
 800
 801         $d->{template} = is_template($conf);
 802     }
 803
 804     foreach my $vmid (keys %$list) {
 805         my $d = $list->{$vmid};
 806         next if $d->{status} ne 'running';
 807
 808         my $pid = find_lxc_pid($vmid);
 809         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 810         $d->{uptime} = time - $ctime; # the method lxcfs uses
 811
 812         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 813         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 814
 815         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 816         my @bytes = split(/\n/, $blkio_bytes);
 817         foreach my $byte (@bytes) {
 818             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 819                 $d->{diskread} = $2 if $key eq 'Read';
 820                 $d->{diskwrite} = $2 if $key eq 'Write';
 821             }
 822         }
 823     }
 824
 825     return $list;
 826 }
 827
 828 my $parse_size = sub {
 829     my ($value) = @_;
 830
 831     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 832     my ($size, $unit) = ($1, $3);
 833     if ($unit) {
 834         if ($unit eq 'K') {
 835             $size = $size * 1024;
 836         } elsif ($unit eq 'M') {
 837             $size = $size * 1024 * 1024;
 838         } elsif ($unit eq 'G') {
 839             $size = $size * 1024 * 1024 * 1024;
 840         }
 841     }
 842     return int($size);
 843 };
 844
 845 my $format_size = sub {
 846     my ($size) = @_;
 847
 848     $size = int($size);
 849
 850     my $kb = int($size/1024);
 851     return $size if $kb*1024 != $size;
 852
 853     my $mb = int($kb/1024);
 854     return "${kb}K" if $mb*1024 != $kb;
 855
 856     my $gb = int($mb/1024);
 857     return "${mb}M" if $gb*1024 != $mb;
 858
 859     return "${gb}G";
 860 };
 861
 862 sub parse_ct_mountpoint {
 863     my ($data) = @_;
 864
 865     $data //= '';
 866
 867     my $res;
 868     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 869     if ($@) {
 870         warn $@;
 871         return undef;
 872     }
 873
 874     return undef if !defined($res->{volume});
 875
 876     if ($res->{size}) {
 877         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 878     }
 879
 880     return $res;
 881 }
 882
 883 sub print_ct_mountpoint {
 884     my ($info, $nomp) = @_;
 885
 886     my $opts = '';
 887
 888     die "missing volume\n" if !$info->{volume};
 889
 890     foreach my $o (qw(backup)) {
 891         $opts .= ",$o=$info->{$o}" if defined($info->{$o});
 892     }
 893
 894     if ($info->{size}) {
 895         $opts .= ",size=" . &$format_size($info->{size});
 896     }
 897
 898     $opts .= ",mp=$info->{mp}" if !$nomp;
 899
 900     return "$info->{volume}$opts";
 901 }
 902
 903 sub print_lxc_network {
 904     my $net = shift;
 905
 906     die "no network name defined\n" if !$net->{name};
 907
 908     my $res = "name=$net->{name}";
 909
 910     foreach my $k (qw(hwaddr mtu bridge ip gw ip6 gw6 firewall tag)) {
 911         next if !defined($net->{$k});
 912         $res .= ",$k=$net->{$k}";
 913     }
 914
 915     return $res;
 916 }
 917
 918 sub parse_lxc_network {
 919     my ($data) = @_;
 920
 921     my $res = {};
 922
 923     return $res if !$data;
 924
 925     eval { $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data) };
 926     if ($@) {
 927         warn $@;
 928         return undef;
 929     }
 930
 931     $res->{type} = 'veth';
 932     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 933
 934     return $res;
 935 }
 936
 937 sub read_cgroup_value {
 938     my ($group, $vmid, $name, $full) = @_;
 939
 940     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 941
 942     return PVE::Tools::file_get_contents($path) if $full;
 943
 944     return PVE::Tools::file_read_firstline($path);
 945 }
 946
 947 sub write_cgroup_value {
 948    my ($group, $vmid, $name, $value) = @_;
 949
 950    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 951    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 952
 953 }
 954
 955 sub find_lxc_console_pids {
 956
 957     my $res = {};
 958
 959     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 960         my ($pid) = @_;
 961
 962         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 963         return if !$cmdline;
 964
 965         my @args = split(/\0/, $cmdline);
 966
 967         # serach for lxc-console -n <vmid>
 968         return if scalar(@args) != 3;
 969         return if $args[1] ne '-n';
 970         return if $args[2] !~ m/^\d+$/;
 971         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 972
 973         my $vmid = $args[2];
 974
 975         push @{$res->{$vmid}}, $pid;
 976     });
 977
 978     return $res;
 979 }
 980
 981 sub find_lxc_pid {
 982     my ($vmid) = @_;
 983
 984     my $pid = undef;
 985     my $parser = sub {
 986         my $line = shift;
 987         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 988     };
 989     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
 990
 991     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 992
 993     return $pid;
 994 }
 995
 996 my $ipv4_reverse_mask = [
 997     '0.0.0.0',
 998     '128.0.0.0',
 999     '192.0.0.0',
1000     '224.0.0.0',
1001     '240.0.0.0',
1002     '248.0.0.0',
1003     '252.0.0.0',
1004     '254.0.0.0',
1005     '255.0.0.0',
1006     '255.128.0.0',
1007     '255.192.0.0',
1008     '255.224.0.0',
1009     '255.240.0.0',
1010     '255.248.0.0',
1011     '255.252.0.0',
1012     '255.254.0.0',
1013     '255.255.0.0',
1014     '255.255.128.0',
1015     '255.255.192.0',
1016     '255.255.224.0',
1017     '255.255.240.0',
1018     '255.255.248.0',
1019     '255.255.252.0',
1020     '255.255.254.0',
1021     '255.255.255.0',
1022     '255.255.255.128',
1023     '255.255.255.192',
1024     '255.255.255.224',
1025     '255.255.255.240',
1026     '255.255.255.248',
1027     '255.255.255.252',
1028     '255.255.255.254',
1029     '255.255.255.255',
1030 ];
1031
1032 # Note: we cannot use Net:IP, because that only allows strict
1033 # CIDR networks
1034 sub parse_ipv4_cidr {
1035     my ($cidr, $noerr) = @_;
1036
1037     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
1038         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
1039     }
1040
1041     return undef if $noerr;
1042
1043     die "unable to parse ipv4 address/mask\n";
1044 }
1045
1046 sub check_lock {
1047     my ($conf) = @_;
1048
1049     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1050 }
1051
1052 sub check_protection {
1053     my ($vm_conf, $err_msg) = @_;
1054
1055     if ($vm_conf->{protection}) {
1056         die "$err_msg - protection mode enabled\n";
1057     }
1058 }
1059
1060 sub update_lxc_config {
1061     my ($storage_cfg, $vmid, $conf) = @_;
1062
1063     my $dir = "/var/lib/lxc/$vmid";
1064
1065     if ($conf->{template}) {
1066
1067         unlink "$dir/config";
1068
1069         return;
1070     }
1071
1072     my $raw = '';
1073
1074     die "missing 'arch' - internal error" if !$conf->{arch};
1075     $raw .= "lxc.arch = $conf->{arch}\n";
1076
1077     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1078     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1079         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1080     } else {
1081         die "implement me";
1082     }
1083
1084     if (!has_dev_console($conf)) {
1085         $raw .= "lxc.console = none\n";
1086         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1087     }
1088
1089     my $ttycount = get_tty_count($conf);
1090     $raw .= "lxc.tty = $ttycount\n";
1091
1092     # some init scripts expects a linux terminal (turnkey).
1093     $raw .= "lxc.environment = TERM=linux\n";
1094
1095     my $utsname = $conf->{hostname} || "CT$vmid";
1096     $raw .= "lxc.utsname = $utsname\n";
1097
1098     my $memory = $conf->{memory} || 512;
1099     my $swap = $conf->{swap} // 0;
1100
1101     my $lxcmem = int($memory*1024*1024);
1102     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1103
1104     my $lxcswap = int(($memory + $swap)*1024*1024);
1105     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1106
1107     if (my $cpulimit = $conf->{cpulimit}) {
1108         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1109         my $value = int(100000*$cpulimit);
1110         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1111     }
1112
1113     my $shares = $conf->{cpuunits} || 1024;
1114     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1115
1116     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1117     $mountpoint->{mp} = '/';
1118
1119     my ($path, $use_loopdev) = mountpoint_mount_path($mountpoint, $storage_cfg);
1120     $path = "loop:$path" if $use_loopdev;
1121
1122     $raw .= "lxc.rootfs = $path\n";
1123
1124     my $netcount = 0;
1125     foreach my $k (keys %$conf) {
1126         next if $k !~ m/^net(\d+)$/;
1127         my $ind = $1;
1128         my $d = parse_lxc_network($conf->{$k});
1129         $netcount++;
1130         $raw .= "lxc.network.type = veth\n";
1131         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1132         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1133         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1134         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1135     }
1136
1137     if (my $lxcconf = $conf->{lxc}) {
1138         foreach my $entry (@$lxcconf) {
1139             my ($k, $v) = @$entry;
1140             $netcount++ if $k eq 'lxc.network.type';
1141             $raw .= "$k = $v\n";
1142         }
1143     }
1144
1145     $raw .= "lxc.network.type = empty\n" if !$netcount;
1146
1147     File::Path::mkpath("$dir/rootfs");
1148
1149     PVE::Tools::file_set_contents("$dir/config", $raw);
1150 }
1151
1152 # verify and cleanup nameserver list (replace \0 with ' ')
1153 sub verify_nameserver_list {
1154     my ($nameserver_list) = @_;
1155
1156     my @list = ();
1157     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1158         PVE::JSONSchema::pve_verify_ip($server);
1159         push @list, $server;
1160     }
1161
1162     return join(' ', @list);
1163 }
1164
1165 sub verify_searchdomain_list {
1166     my ($searchdomain_list) = @_;
1167
1168     my @list = ();
1169     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1170         # todo: should we add checks for valid dns domains?
1171         push @list, $server;
1172     }
1173
1174     return join(' ', @list);
1175 }
1176
1177 sub update_pct_config {
1178     my ($vmid, $conf, $running, $param, $delete) = @_;
1179
1180     my @nohotplug;
1181
1182     my $new_disks = 0;
1183
1184     my $rootdir;
1185     if ($running) {
1186         my $pid = find_lxc_pid($vmid);
1187         $rootdir = "/proc/$pid/root";
1188     }
1189
1190     if (defined($delete)) {
1191         foreach my $opt (@$delete) {
1192             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1193                 die "unable to delete required option '$opt'\n";
1194             } elsif ($opt eq 'swap') {
1195                 delete $conf->{$opt};
1196                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1197             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1198                 delete $conf->{$opt};
1199             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1200                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1201                 delete $conf->{$opt};
1202                 push @nohotplug, $opt;
1203                 next if $running;
1204             } elsif ($opt =~ m/^net(\d)$/) {
1205                 delete $conf->{$opt};
1206                 next if !$running;
1207                 my $netid = $1;
1208                 PVE::Network::veth_delete("veth${vmid}i$netid");
1209             } elsif ($opt eq 'protection') {
1210                 delete $conf->{$opt};
1211             } elsif ($opt =~ m/^mp(\d+)$/) {
1212                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1213                 delete $conf->{$opt};
1214                 push @nohotplug, $opt;
1215                 next if $running;
1216             } else {
1217                 die "implement me"
1218             }
1219             write_config($vmid, $conf) if $running;
1220         }
1221     }
1222
1223     # There's no separate swap size to configure, there's memory and "total"
1224     # memory (iow. memory+swap). This means we have to change them together.
1225     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1226     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1227     if (defined($wanted_memory) || defined($wanted_swap)) {
1228
1229         $wanted_memory //= ($conf->{memory} || 512);
1230         $wanted_swap //=  ($conf->{swap} || 0);
1231
1232         my $total = $wanted_memory + $wanted_swap;
1233         if ($running) {
1234             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1235             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1236         }
1237         $conf->{memory} = $wanted_memory;
1238         $conf->{swap} = $wanted_swap;
1239
1240         write_config($vmid, $conf) if $running;
1241     }
1242
1243     foreach my $opt (keys %$param) {
1244         my $value = $param->{$opt};
1245         if ($opt eq 'hostname') {
1246             $conf->{$opt} = $value;
1247         } elsif ($opt eq 'onboot') {
1248             $conf->{$opt} = $value ? 1 : 0;
1249         } elsif ($opt eq 'startup') {
1250             $conf->{$opt} = $value;
1251         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1252             $conf->{$opt} = $value;
1253             push @nohotplug, $opt;
1254             next if $running;
1255         } elsif ($opt eq 'nameserver') {
1256             my $list = verify_nameserver_list($value);
1257             $conf->{$opt} = $list;
1258             push @nohotplug, $opt;
1259             next if $running;
1260         } elsif ($opt eq 'searchdomain') {
1261             my $list = verify_searchdomain_list($value);
1262             $conf->{$opt} = $list;
1263             push @nohotplug, $opt;
1264             next if $running;
1265         } elsif ($opt eq 'cpulimit') {
1266             $conf->{$opt} = $value;
1267             push @nohotplug, $opt; # fixme: hotplug
1268             next;
1269         } elsif ($opt eq 'cpuunits') {
1270             $conf->{$opt} = $value;
1271             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1272         } elsif ($opt eq 'description') {
1273             $conf->{$opt} = PVE::Tools::encode_text($value);
1274         } elsif ($opt =~ m/^net(\d+)$/) {
1275             my $netid = $1;
1276             my $net = parse_lxc_network($value);
1277             if (!$running) {
1278                 $conf->{$opt} = print_lxc_network($net);
1279             } else {
1280                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1281             }
1282         } elsif ($opt eq 'protection') {
1283             $conf->{$opt} = $value ? 1 : 0;
1284         } elsif ($opt =~ m/^mp(\d+)$/) {
1285             check_protection($conf, "can't update CT $vmid drive '$opt'");
1286             $conf->{$opt} = $value;
1287             $new_disks = 1;
1288             push @nohotplug, $opt;
1289             next;
1290         } elsif ($opt eq 'rootfs') {
1291             check_protection($conf, "can't update CT $vmid drive '$opt'");
1292             die "implement me: $opt";
1293         } else {
1294             die "implement me: $opt";
1295         }
1296         write_config($vmid, $conf) if $running;
1297     }
1298
1299     if ($running && scalar(@nohotplug)) {
1300         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1301     }
1302
1303     if ($new_disks) {
1304         my $storage_cfg = PVE::Storage::config();
1305         create_disks($storage_cfg, $vmid, $conf, $conf);
1306     }
1307 }
1308
1309 sub has_dev_console {
1310     my ($conf) = @_;
1311
1312     return !(defined($conf->{console}) && !$conf->{console});
1313 }
1314
1315 sub get_tty_count {
1316     my ($conf) = @_;
1317
1318     return $conf->{tty} // $confdesc->{tty}->{default};
1319 }
1320
1321 sub get_cmode {
1322     my ($conf) = @_;
1323
1324     return $conf->{cmode} // $confdesc->{cmode}->{default};
1325 }
1326
1327 sub get_console_command {
1328     my ($vmid, $conf) = @_;
1329
1330     my $cmode = get_cmode($conf);
1331
1332     if ($cmode eq 'console') {
1333         return ['lxc-console', '-n',  $vmid, '-t', 0];
1334     } elsif ($cmode eq 'tty') {
1335         return ['lxc-console', '-n',  $vmid];
1336     } elsif ($cmode eq 'shell') {
1337         return ['lxc-attach', '--clear-env', '-n', $vmid];
1338     } else {
1339         die "internal error";
1340     }
1341 }
1342
1343 sub get_primary_ips {
1344     my ($conf) = @_;
1345
1346     # return data from net0
1347
1348     return undef if !defined($conf->{net0});
1349     my $net = parse_lxc_network($conf->{net0});
1350
1351     my $ipv4 = $net->{ip};
1352     if ($ipv4) {
1353         if ($ipv4 =~ /^(dhcp|manual)$/) {
1354             $ipv4 = undef
1355         } else {
1356             $ipv4 =~ s!/\d+$!!;
1357         }
1358     }
1359     my $ipv6 = $net->{ip6};
1360     if ($ipv6) {
1361         if ($ipv6 =~ /^(dhcp|manual)$/) {
1362             $ipv6 = undef;
1363         } else {
1364             $ipv6 =~ s!/\d+$!!;
1365         }
1366     }
1367
1368     return ($ipv4, $ipv6);
1369 }
1370
1371
1372 sub destroy_lxc_container {
1373     my ($storage_cfg, $vmid, $conf) = @_;
1374
1375     foreach_mountpoint($conf, sub {
1376         my ($ms, $mountpoint) = @_;
1377
1378         # skip bind mounts and block devices
1379         if ($mountpoint->{volume} =~ m|^/|) {
1380                 return;
1381         }
1382
1383         my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $mountpoint->{volume});
1384         PVE::Storage::vdisk_free($storage_cfg, $mountpoint->{volume}) if $vmid == $owner;
1385     });
1386
1387     rmdir "/var/lib/lxc/$vmid/rootfs";
1388     unlink "/var/lib/lxc/$vmid/config";
1389     rmdir "/var/lib/lxc/$vmid";
1390     destroy_config($vmid);
1391
1392     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1393     #PVE::Tools::run_command($cmd);
1394 }
1395
1396 sub vm_stop_cleanup {
1397     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1398
1399     eval {
1400         if (!$keepActive) {
1401
1402             my $vollist = get_vm_volumes($conf);
1403             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1404         }
1405     };
1406     warn $@ if $@; # avoid errors - just warn
1407 }
1408
1409 my $safe_num_ne = sub {
1410     my ($a, $b) = @_;
1411
1412     return 0 if !defined($a) && !defined($b);
1413     return 1 if !defined($a);
1414     return 1 if !defined($b);
1415
1416     return $a != $b;
1417 };
1418
1419 my $safe_string_ne = sub {
1420     my ($a, $b) = @_;
1421
1422     return 0 if !defined($a) && !defined($b);
1423     return 1 if !defined($a);
1424     return 1 if !defined($b);
1425
1426     return $a ne $b;
1427 };
1428
1429 sub update_net {
1430     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1431
1432     if ($newnet->{type} ne 'veth') {
1433         # for when there are physical interfaces
1434         die "cannot update interface of type $newnet->{type}";
1435     }
1436
1437     my $veth = "veth${vmid}i${netid}";
1438     my $eth = $newnet->{name};
1439
1440     if (my $oldnetcfg = $conf->{$opt}) {
1441         my $oldnet = parse_lxc_network($oldnetcfg);
1442
1443         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1444             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1445
1446             PVE::Network::veth_delete($veth);
1447             delete $conf->{$opt};
1448             write_config($vmid, $conf);
1449
1450             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1451
1452         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1453                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1454                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1455
1456                 if ($oldnet->{bridge}) {
1457                     PVE::Network::tap_unplug($veth);
1458                     foreach (qw(bridge tag firewall)) {
1459                         delete $oldnet->{$_};
1460                     }
1461                     $conf->{$opt} = print_lxc_network($oldnet);
1462                     write_config($vmid, $conf);
1463                 }
1464
1465                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1466                 foreach (qw(bridge tag firewall)) {
1467                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1468                 }
1469                 $conf->{$opt} = print_lxc_network($oldnet);
1470                 write_config($vmid, $conf);
1471         }
1472     } else {
1473         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1474     }
1475
1476     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1477 }
1478
1479 sub hotplug_net {
1480     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1481
1482     my $veth = "veth${vmid}i${netid}";
1483     my $vethpeer = $veth . "p";
1484     my $eth = $newnet->{name};
1485
1486     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1487     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1488
1489     # attach peer in container
1490     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1491     PVE::Tools::run_command($cmd);
1492
1493     # link up peer in container
1494     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1495     PVE::Tools::run_command($cmd);
1496
1497     my $done = { type => 'veth' };
1498     foreach (qw(bridge tag firewall hwaddr name)) {
1499         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1500     }
1501     $conf->{$opt} = print_lxc_network($done);
1502
1503     write_config($vmid, $conf);
1504 }
1505
1506 sub update_ipconfig {
1507     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1508
1509     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1510
1511     my $optdata = parse_lxc_network($conf->{$opt});
1512     my $deleted = [];
1513     my $added = [];
1514     my $nscmd = sub {
1515         my $cmdargs = shift;
1516         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1517     };
1518     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1519
1520     my $change_ip_config = sub {
1521         my ($ipversion) = @_;
1522
1523         my $family_opt = "-$ipversion";
1524         my $suffix = $ipversion == 4 ? '' : $ipversion;
1525         my $gw= "gw$suffix";
1526         my $ip= "ip$suffix";
1527
1528         my $newip = $newnet->{$ip};
1529         my $newgw = $newnet->{$gw};
1530         my $oldip = $optdata->{$ip};
1531
1532         my $change_ip = &$safe_string_ne($oldip, $newip);
1533         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1534
1535         return if !$change_ip && !$change_gw;
1536
1537         # step 1: add new IP, if this fails we cancel
1538         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1539             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1540             if (my $err = $@) {
1541                 warn $err;
1542                 return;
1543             }
1544         }
1545
1546         # step 2: replace gateway
1547         #   If this fails we delete the added IP and cancel.
1548         #   If it succeeds we save the config and delete the old IP, ignoring
1549         #   errors. The config is then saved.
1550         # Note: 'ip route replace' can add
1551         if ($change_gw) {
1552             if ($newgw) {
1553                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1554                 if (my $err = $@) {
1555                     warn $err;
1556                     # the route was not replaced, the old IP is still available
1557                     # rollback (delete new IP) and cancel
1558                     if ($change_ip) {
1559                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1560                         warn $@ if $@; # no need to die here
1561                     }
1562                     return;
1563                 }
1564             } else {
1565                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1566                 # if the route was not deleted, the guest might have deleted it manually
1567                 # warn and continue
1568                 warn $@ if $@;
1569             }
1570         }
1571
1572         # from this point on we save the configuration
1573         # step 3: delete old IP ignoring errors
1574         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1575             # We need to enable promote_secondaries, otherwise our newly added
1576             # address will be removed along with the old one.
1577             my $promote = 0;
1578             eval {
1579                 if ($ipversion == 4) {
1580                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1581                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1582                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1583                 }
1584                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1585             };
1586             warn $@ if $@; # no need to die here
1587
1588             if ($ipversion == 4) {
1589                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1590             }
1591         }
1592
1593         foreach my $property ($ip, $gw) {
1594             if ($newnet->{$property}) {
1595                 $optdata->{$property} = $newnet->{$property};
1596             } else {
1597                 delete $optdata->{$property};
1598             }
1599         }
1600         $conf->{$opt} = print_lxc_network($optdata);
1601         write_config($vmid, $conf);
1602         $lxc_setup->setup_network($conf);
1603     };
1604
1605     &$change_ip_config(4);
1606     &$change_ip_config(6);
1607
1608 }
1609
1610 # Internal snapshots
1611
1612 # NOTE: Snapshot create/delete involves several non-atomic
1613 # action, and can take a long time.
1614 # So we try to avoid locking the file and use 'lock' variable
1615 # inside the config file instead.
1616
1617 my $snapshot_copy_config = sub {
1618     my ($source, $dest) = @_;
1619
1620     foreach my $k (keys %$source) {
1621         next if $k eq 'snapshots';
1622         next if $k eq 'snapstate';
1623         next if $k eq 'snaptime';
1624         next if $k eq 'vmstate';
1625         next if $k eq 'lock';
1626         next if $k eq 'digest';
1627         next if $k eq 'description';
1628
1629         $dest->{$k} = $source->{$k};
1630     }
1631 };
1632
1633 my $snapshot_prepare = sub {
1634     my ($vmid, $snapname, $comment) = @_;
1635
1636     my $snap;
1637
1638     my $updatefn =  sub {
1639
1640         my $conf = load_config($vmid);
1641
1642         die "you can't take a snapshot if it's a template\n"
1643             if is_template($conf);
1644
1645         check_lock($conf);
1646
1647         $conf->{lock} = 'snapshot';
1648
1649         die "snapshot name '$snapname' already used\n"
1650             if defined($conf->{snapshots}->{$snapname});
1651
1652         my $storecfg = PVE::Storage::config();
1653         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1654
1655         $snap = $conf->{snapshots}->{$snapname} = {};
1656
1657         &$snapshot_copy_config($conf, $snap);
1658
1659         $snap->{'snapstate'} = "prepare";
1660         $snap->{'snaptime'} = time();
1661         $snap->{'description'} = $comment if $comment;
1662         $conf->{snapshots}->{$snapname} = $snap;
1663
1664         write_config($vmid, $conf);
1665     };
1666
1667     lock_container($vmid, 10, $updatefn);
1668
1669     return $snap;
1670 };
1671
1672 my $snapshot_commit = sub {
1673     my ($vmid, $snapname) = @_;
1674
1675     my $updatefn = sub {
1676
1677         my $conf = load_config($vmid);
1678
1679         die "missing snapshot lock\n"
1680             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1681
1682         die "snapshot '$snapname' does not exist\n"
1683             if !defined($conf->{snapshots}->{$snapname});
1684
1685         die "wrong snapshot state\n"
1686             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1687                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1688
1689         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1690         delete $conf->{lock};
1691         $conf->{parent} = $snapname;
1692
1693         write_config($vmid, $conf);
1694     };
1695
1696     lock_container($vmid, 10 ,$updatefn);
1697 };
1698
1699 sub has_feature {
1700     my ($feature, $conf, $storecfg, $snapname) = @_;
1701
1702     my $err;
1703
1704     foreach_mountpoint($conf, sub {
1705         my ($ms, $mountpoint) = @_;
1706
1707         return if $err; # skip further test
1708
1709         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1710
1711         # TODO: implement support for mountpoints
1712         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1713             if $ms ne 'rootfs';
1714     });
1715
1716     return $err ? 0 : 1;
1717 }
1718
1719 sub snapshot_create {
1720     my ($vmid, $snapname, $comment) = @_;
1721
1722     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1723
1724     my $conf = load_config($vmid);
1725
1726     my $running = check_running($vmid);
1727     eval {
1728         if ($running) {
1729             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1730             PVE::Tools::run_command(['/bin/sync']);
1731         };
1732
1733         my $storecfg = PVE::Storage::config();
1734         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1735         my $volid = $rootinfo->{volume};
1736
1737         if ($running) {
1738             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1739         };
1740
1741         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1742         &$snapshot_commit($vmid, $snapname);
1743     };
1744     if(my $err = $@) {
1745         snapshot_delete($vmid, $snapname, 1);
1746         die "$err\n";
1747     }
1748 }
1749
1750 sub snapshot_delete {
1751     my ($vmid, $snapname, $force) = @_;
1752
1753     my $snap;
1754
1755     my $conf;
1756
1757     my $updatefn =  sub {
1758
1759         $conf = load_config($vmid);
1760
1761         die "you can't delete a snapshot if vm is a template\n"
1762             if is_template($conf);
1763
1764         $snap = $conf->{snapshots}->{$snapname};
1765
1766         check_lock($conf);
1767
1768         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1769
1770         $snap->{snapstate} = 'delete';
1771
1772         write_config($vmid, $conf);
1773     };
1774
1775     lock_container($vmid, 10, $updatefn);
1776
1777     my $storecfg = PVE::Storage::config();
1778
1779     my $del_snap =  sub {
1780
1781         check_lock($conf);
1782
1783         if ($conf->{parent} eq $snapname) {
1784             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1785                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1786             } else {
1787                 delete $conf->{parent};
1788             }
1789         }
1790
1791         delete $conf->{snapshots}->{$snapname};
1792
1793         write_config($vmid, $conf);
1794     };
1795
1796     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1797     my $rootinfo = parse_ct_mountpoint($rootfs);
1798     my $volid = $rootinfo->{volume};
1799
1800     eval {
1801         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1802     };
1803     my $err = $@;
1804
1805     if(!$err || ($err && $force)) {
1806         lock_container($vmid, 10, $del_snap);
1807         if ($err) {
1808             die "Can't delete snapshot: $vmid $snapname $err\n";
1809         }
1810     }
1811 }
1812
1813 sub snapshot_rollback {
1814     my ($vmid, $snapname) = @_;
1815
1816     my $storecfg = PVE::Storage::config();
1817
1818     my $conf = load_config($vmid);
1819
1820     die "you can't rollback if vm is a template\n" if is_template($conf);
1821
1822     my $snap = $conf->{snapshots}->{$snapname};
1823
1824     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1825
1826     my $rootfs = $snap->{rootfs};
1827     my $rootinfo = parse_ct_mountpoint($rootfs);
1828     my $volid = $rootinfo->{volume};
1829
1830     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1831
1832     my $updatefn = sub {
1833
1834         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1835             if $snap->{snapstate};
1836
1837         check_lock($conf);
1838
1839         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1840
1841         die "unable to rollback vm $vmid: vm is running\n"
1842             if check_running($vmid);
1843
1844         $conf->{lock} = 'rollback';
1845
1846         my $forcemachine;
1847
1848         # copy snapshot config to current config
1849
1850         my $tmp_conf = $conf;
1851         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1852         $conf->{snapshots} = $tmp_conf->{snapshots};
1853         delete $conf->{snaptime};
1854         delete $conf->{snapname};
1855         $conf->{parent} = $snapname;
1856
1857         write_config($vmid, $conf);
1858     };
1859
1860     my $unlockfn = sub {
1861         delete $conf->{lock};
1862         write_config($vmid, $conf);
1863     };
1864
1865     lock_container($vmid, 10, $updatefn);
1866
1867     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1868
1869     lock_container($vmid, 5, $unlockfn);
1870 }
1871
1872 sub template_create {
1873     my ($vmid, $conf) = @_;
1874
1875     my $storecfg = PVE::Storage::config();
1876
1877     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1878     my $volid = $rootinfo->{volume};
1879
1880     die "Template feature is not available for '$volid'\n"
1881         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1882
1883     PVE::Storage::activate_volumes($storecfg, [$volid]);
1884
1885     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1886     $rootinfo->{volume} = $template_volid;
1887     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1888
1889     write_config($vmid, $conf);
1890 }
1891
1892 sub is_template {
1893     my ($conf) = @_;
1894
1895     return 1 if defined $conf->{template} && $conf->{template} == 1;
1896 }
1897
1898 sub mountpoint_names {
1899     my ($reverse) = @_;
1900
1901     my @names = ('rootfs');
1902
1903     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1904         push @names, "mp$i";
1905     }
1906
1907     return $reverse ? reverse @names : @names;
1908 }
1909
1910 # The container might have *different* symlinks than the host. realpath/abs_path
1911 # use the actual filesystem to resolve links.
1912 sub sanitize_mountpoint {
1913     my ($mp) = @_;
1914     $mp = '/' . $mp; # we always start with a slash
1915     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1916     $mp =~ s@/\./@@g; # collapse /./
1917     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1918     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1919     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1920     return $mp;
1921 }
1922
1923 sub foreach_mountpoint_full {
1924     my ($conf, $reverse, $func) = @_;
1925
1926     foreach my $key (mountpoint_names($reverse)) {
1927         my $value = $conf->{$key};
1928         next if !defined($value);
1929         my $mountpoint = parse_ct_mountpoint($value);
1930
1931         # just to be sure: rootfs is /
1932         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1933         $mountpoint->{mp} = sanitize_mountpoint($path);
1934
1935         $path = $mountpoint->{volume};
1936         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1937
1938         &$func($key, $mountpoint);
1939     }
1940 }
1941
1942 sub foreach_mountpoint {
1943     my ($conf, $func) = @_;
1944
1945     foreach_mountpoint_full($conf, 0, $func);
1946 }
1947
1948 sub foreach_mountpoint_reverse {
1949     my ($conf, $func) = @_;
1950
1951     foreach_mountpoint_full($conf, 1, $func);
1952 }
1953
1954 sub check_ct_modify_config_perm {
1955     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1956
1957     return 1 if $authuser ne 'root@pam';
1958
1959     foreach my $opt (@$key_list) {
1960
1961         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1962             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1963         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1964             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1965         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1966             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1967         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1968                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1969             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1970         } else {
1971             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1972         }
1973     }
1974
1975     return 1;
1976 }
1977
1978 sub umount_all {
1979     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
1980
1981     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1982     my $volid_list = get_vm_volumes($conf);
1983
1984     foreach_mountpoint_reverse($conf, sub {
1985         my ($ms, $mountpoint) = @_;
1986
1987         my $volid = $mountpoint->{volume};
1988         my $mount = $mountpoint->{mp};
1989
1990         return if !$volid || !$mount;
1991
1992         my $mount_path = "$rootdir/$mount";
1993         $mount_path =~ s!/+!/!g;
1994
1995         return if !PVE::ProcFSTools::is_mounted($mount_path);
1996
1997         eval {
1998             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1999         };
2000         if (my $err = $@) {
2001             if ($noerr) {
2002                 warn $err;
2003             } else {
2004                 die $err;
2005             }
2006         }
2007     });
2008 }
2009
2010 sub mount_all {
2011     my ($vmid, $storage_cfg, $conf) = @_;
2012
2013     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2014     File::Path::make_path($rootdir);
2015
2016     my $volid_list = get_vm_volumes($conf);
2017     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2018
2019     eval {
2020         foreach_mountpoint($conf, sub {
2021             my ($ms, $mountpoint) = @_;
2022
2023             my $volid = $mountpoint->{volume};
2024             my $mount = $mountpoint->{mp};
2025
2026             return if !$volid || !$mount;
2027
2028             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2029             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2030                 PVE::Storage::parse_volname($storage_cfg, $volid);
2031
2032             die "unable to mount base volume - internal error" if $isBase;
2033
2034             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2035         });
2036     };
2037     if (my $err = $@) {
2038         warn "mounting container failed - $err";
2039         umount_all($vmid, $storage_cfg, $conf, 1);
2040     }
2041
2042     return $rootdir;
2043 }
2044
2045
2046 sub mountpoint_mount_path {
2047     my ($mountpoint, $storage_cfg, $snapname) = @_;
2048
2049     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2050 }
2051
2052 my $check_mount_path = sub {
2053     my ($path) = @_;
2054     $path = File::Spec->canonpath($path);
2055     my $real = Cwd::realpath($path);
2056     if ($real ne $path) {
2057         die "mount path modified by symlink: $path != $real";
2058     }
2059 };
2060
2061 # use $rootdir = undef to just return the corresponding mount path
2062 sub mountpoint_mount {
2063     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2064
2065     my $volid = $mountpoint->{volume};
2066     my $mount = $mountpoint->{mp};
2067
2068     return if !$volid || !$mount;
2069
2070     my $mount_path;
2071
2072     if (defined($rootdir)) {
2073         $rootdir =~ s!/+$!!;
2074         $mount_path = "$rootdir/$mount";
2075         $mount_path =~ s!/+!/!g;
2076         &$check_mount_path($mount_path);
2077         File::Path::mkpath($mount_path);
2078     }
2079
2080     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2081
2082     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2083
2084     if ($storage) {
2085
2086         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2087         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2088
2089         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2090             PVE::Storage::parse_volname($storage_cfg, $volid);
2091
2092         if ($format eq 'subvol') {
2093             if ($mount_path) {
2094                 if ($snapname) {
2095                     if ($scfg->{type} eq 'zfspool') {
2096                         my $path_arg = $path;
2097                         $path_arg =~ s!^/+!!;
2098                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', '-t', 'zfs', $path_arg, $mount_path]);
2099                     } else {
2100                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2101                     }
2102                 } else {
2103                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2104                 }
2105             }
2106             return wantarray ? ($path, 0) : $path;
2107         } elsif ($format eq 'raw') {
2108             my $use_loopdev = 0;
2109             my @extra_opts;
2110             if ($scfg->{path}) {
2111                 push @extra_opts, '-o', 'loop';
2112                 $use_loopdev = 1;
2113             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2114                 # do nothing
2115             } else {
2116                 die "unsupported storage type '$scfg->{type}'\n";
2117             }
2118             if ($mount_path) {
2119                 if ($isBase || defined($snapname)) {
2120                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2121                 } else {
2122                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2123                 }
2124             }
2125             return wantarray ? ($path, $use_loopdev) : $path;
2126         } else {
2127             die "unsupported image format '$format'\n";
2128         }
2129     } elsif ($volid =~ m|^/dev/.+|) {
2130         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2131         return wantarray ? ($volid, 0) : $volid;
2132     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2133         &$check_mount_path($volid);
2134         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2135         return wantarray ? ($volid, 0) : $volid;
2136     }
2137
2138     die "unsupported storage";
2139 }
2140
2141 sub get_vm_volumes {
2142     my ($conf, $excludes) = @_;
2143
2144     my $vollist = [];
2145
2146     foreach_mountpoint($conf, sub {
2147         my ($ms, $mountpoint) = @_;
2148
2149         return if $excludes && $ms eq $excludes;
2150
2151         my $volid = $mountpoint->{volume};
2152
2153         return if !$volid || $volid =~ m|^/|;
2154
2155         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2156         return if !$sid;
2157
2158         push @$vollist, $volid;
2159     });
2160
2161     return $vollist;
2162 }
2163
2164 sub mkfs {
2165     my ($dev) = @_;
2166
2167     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2168 }
2169
2170 sub format_disk {
2171     my ($storage_cfg, $volid) = @_;
2172
2173     if ($volid =~ m!^/dev/.+!) {
2174         mkfs($volid);
2175         return;
2176     }
2177
2178     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2179
2180     die "cannot format volume '$volid' with no storage\n" if !$storage;
2181
2182     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2183
2184     my $path = PVE::Storage::path($storage_cfg, $volid);
2185
2186     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2187         PVE::Storage::parse_volname($storage_cfg, $volid);
2188
2189     die "cannot format volume '$volid' (format == $format)\n"
2190         if $format ne 'raw';
2191
2192     mkfs($path);
2193 }
2194
2195 sub destroy_disks {
2196     my ($storecfg, $vollist) = @_;
2197
2198     foreach my $volid (@$vollist) {
2199         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2200         warn $@ if $@;
2201     }
2202 }
2203
2204 sub create_disks {
2205     my ($storecfg, $vmid, $settings, $conf) = @_;
2206
2207     my $vollist = [];
2208
2209     eval {
2210         foreach_mountpoint($settings, sub {
2211             my ($ms, $mountpoint) = @_;
2212
2213             my $volid = $mountpoint->{volume};
2214             my $mp = $mountpoint->{mp};
2215
2216             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2217
2218             return if !$storage;
2219
2220             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2221                 my ($storeid, $size_gb) = ($1, $2);
2222
2223                 my $size_kb = int(${size_gb}*1024) * 1024;
2224
2225                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2226                 # fixme: use better naming ct-$vmid-disk-X.raw?
2227
2228                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2229                     if ($size_kb > 0) {
2230                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2231                                                            undef, $size_kb);
2232                         format_disk($storecfg, $volid);
2233                     } else {
2234                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2235                                                            undef, 0);
2236                     }
2237                 } elsif ($scfg->{type} eq 'zfspool') {
2238
2239                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2240                                                        undef, $size_kb);
2241                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2242
2243                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2244                     format_disk($storecfg, $volid);
2245
2246                 } elsif ($scfg->{type} eq 'rbd') {
2247
2248                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2249                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2250                     format_disk($storecfg, $volid);
2251                 } else {
2252                     die "unable to create containers on storage type '$scfg->{type}'\n";
2253                 }
2254                 push @$vollist, $volid;
2255                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2256                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2257             } else {
2258                 # use specified/existing volid
2259             }
2260         });
2261     };
2262     # free allocated images on error
2263     if (my $err = $@) {
2264         destroy_disks($storecfg, $vollist);
2265         die $err;
2266     }
2267     return $vollist;
2268 }
2269
2270 # bash completion helper
2271
2272 sub complete_os_templates {
2273     my ($cmdname, $pname, $cvalue) = @_;
2274
2275     my $cfg = PVE::Storage::config();
2276
2277     my $storeid;
2278
2279     if ($cvalue =~ m/^([^:]+):/) {
2280         $storeid = $1;
2281     }
2282
2283     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2284     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2285
2286     my $res = [];
2287     foreach my $id (keys %$data) {
2288         foreach my $item (@{$data->{$id}}) {
2289             push @$res, $item->{volid} if defined($item->{volid});
2290         }
2291     }
2292
2293     return $res;
2294 }
2295
2296 my $complete_ctid_full = sub {
2297     my ($running) = @_;
2298
2299     my $idlist = vmstatus();
2300
2301     my $active_hash = list_active_containers();
2302
2303     my $res = [];
2304
2305     foreach my $id (keys %$idlist) {
2306         my $d = $idlist->{$id};
2307         if (defined($running)) {
2308             next if $d->{template};
2309             next if $running && !$active_hash->{$id};
2310             next if !$running && $active_hash->{$id};
2311         }
2312         push @$res, $id;
2313
2314     }
2315     return $res;
2316 };
2317
2318 sub complete_ctid {
2319     return &$complete_ctid_full();
2320 }
2321
2322 sub complete_ctid_stopped {
2323     return &$complete_ctid_full(0);
2324 }
2325
2326 sub complete_ctid_running {
2327     return &$complete_ctid_full(1);
2328 }
2329
2330 1;