src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21
  22 use Data::Dumper;
  23
  24 my $nodename = PVE::INotify::nodename();
  25
  26 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  27
  28 my $rootfs_desc = {
  29     volume => {
  30         type => 'string',
  31         default_key => 1,
  32         format_description => 'volume',
  33         description => 'Volume, device or directory to mount into the container.',
  34     },
  35     backup => {
  36         type => 'boolean',
  37         format_description => '[1|0]',
  38         description => 'Whether to include the mountpoint in backups.',
  39         optional => 1,
  40     },
  41     size => {
  42         type => 'string',
  43         format => 'disk-size',
  44         format_description => 'DiskSize',
  45         description => 'Volume size (read only value).',
  46         optional => 1,
  47     },
  48 };
  49
  50 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  51     type => 'string', format => $rootfs_desc,
  52     description => "Use volume as container root.",
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  57     description => "The name of the snapshot.",
  58     type => 'string', format => 'pve-configid',
  59     maxLength => 40,
  60 });
  61
  62 my $confdesc = {
  63     lock => {
  64         optional => 1,
  65         type => 'string',
  66         description => "Lock/unlock the VM.",
  67         enum => [qw(migrate backup snapshot rollback)],
  68     },
  69     onboot => {
  70         optional => 1,
  71         type => 'boolean',
  72         description => "Specifies whether a VM will be started during system bootup.",
  73         default => 0,
  74     },
  75     startup => get_standard_option('pve-startup-order'),
  76     template => {
  77         optional => 1,
  78         type => 'boolean',
  79         description => "Enable/disable Template.",
  80         default => 0,
  81     },
  82     arch => {
  83         optional => 1,
  84         type => 'string',
  85         enum => ['amd64', 'i386'],
  86         description => "OS architecture type.",
  87         default => 'amd64',
  88     },
  89     ostype => {
  90         optional => 1,
  91         type => 'string',
  92         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  93         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  94     },
  95     console => {
  96         optional => 1,
  97         type => 'boolean',
  98         description => "Attach a console device (/dev/console) to the container.",
  99         default => 1,
 100     },
 101     tty => {
 102         optional => 1,
 103         type => 'integer',
 104         description => "Specify the number of tty available to the container",
 105         minimum => 0,
 106         maximum => 6,
 107         default => 2,
 108     },
 109     cpulimit => {
 110         optional => 1,
 111         type => 'number',
 112         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 113         minimum => 0,
 114         maximum => 128,
 115         default => 0,
 116     },
 117     cpuunits => {
 118         optional => 1,
 119         type => 'integer',
 120         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 121         minimum => 0,
 122         maximum => 500000,
 123         default => 1024,
 124     },
 125     memory => {
 126         optional => 1,
 127         type => 'integer',
 128         description => "Amount of RAM for the VM in MB.",
 129         minimum => 16,
 130         default => 512,
 131     },
 132     swap => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "Amount of SWAP for the VM in MB.",
 136         minimum => 0,
 137         default => 512,
 138     },
 139     hostname => {
 140         optional => 1,
 141         description => "Set a host name for the container.",
 142         type => 'string', format => 'dns-name',
 143         maxLength => 255,
 144     },
 145     description => {
 146         optional => 1,
 147         type => 'string',
 148         description => "Container description. Only used on the configuration web interface.",
 149     },
 150     searchdomain => {
 151         optional => 1,
 152         type => 'string', format => 'dns-name-list',
 153         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 154     },
 155     nameserver => {
 156         optional => 1,
 157         type => 'string', format => 'address-list',
 158         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 159     },
 160     rootfs => get_standard_option('pve-ct-rootfs'),
 161     parent => {
 162         optional => 1,
 163         type => 'string', format => 'pve-configid',
 164         maxLength => 40,
 165         description => "Parent snapshot name. This is used internally, and should not be modified.",
 166     },
 167     snaptime => {
 168         optional => 1,
 169         description => "Timestamp for snapshots.",
 170         type => 'integer',
 171         minimum => 0,
 172     },
 173     cmode => {
 174         optional => 1,
 175         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 176         type => 'string',
 177         enum => ['shell', 'console', 'tty'],
 178         default => 'tty',
 179     },
 180     protection => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 184         default => 0,
 185     },
 186 };
 187
 188 my $valid_lxc_conf_keys = {
 189     'lxc.include' => 1,
 190     'lxc.arch' => 1,
 191     'lxc.utsname' => 1,
 192     'lxc.haltsignal' => 1,
 193     'lxc.rebootsignal' => 1,
 194     'lxc.stopsignal' => 1,
 195     'lxc.init_cmd' => 1,
 196     'lxc.network.type' => 1,
 197     'lxc.network.flags' => 1,
 198     'lxc.network.link' => 1,
 199     'lxc.network.mtu' => 1,
 200     'lxc.network.name' => 1,
 201     'lxc.network.hwaddr' => 1,
 202     'lxc.network.ipv4' => 1,
 203     'lxc.network.ipv4.gateway' => 1,
 204     'lxc.network.ipv6' => 1,
 205     'lxc.network.ipv6.gateway' => 1,
 206     'lxc.network.script.up' => 1,
 207     'lxc.network.script.down' => 1,
 208     'lxc.pts' => 1,
 209     'lxc.console.logfile' => 1,
 210     'lxc.console' => 1,
 211     'lxc.tty' => 1,
 212     'lxc.devttydir' => 1,
 213     'lxc.hook.autodev' => 1,
 214     'lxc.autodev' => 1,
 215     'lxc.kmsg' => 1,
 216     'lxc.mount' => 1,
 217     'lxc.mount.entry' => 1,
 218     'lxc.mount.auto' => 1,
 219     'lxc.rootfs' => 1,
 220     'lxc.rootfs.mount' => 1,
 221     'lxc.rootfs.options' => 1,
 222     # lxc.cgroup.*
 223     'lxc.cap.drop' => 1,
 224     'lxc.cap.keep' => 1,
 225     'lxc.aa_profile' => 1,
 226     'lxc.aa_allow_incomplete' => 1,
 227     'lxc.se_context' => 1,
 228     'lxc.seccomp' => 1,
 229     'lxc.id_map' => 1,
 230     'lxc.hook.pre-start' => 1,
 231     'lxc.hook.pre-mount' => 1,
 232     'lxc.hook.mount' => 1,
 233     'lxc.hook.start' => 1,
 234     'lxc.hook.stop' => 1,
 235     'lxc.hook.post-stop' => 1,
 236     'lxc.hook.clone' => 1,
 237     'lxc.hook.destroy' => 1,
 238     'lxc.loglevel' => 1,
 239     'lxc.logfile' => 1,
 240     'lxc.start.auto' => 1,
 241     'lxc.start.delay' => 1,
 242     'lxc.start.order' => 1,
 243     'lxc.group' => 1,
 244     'lxc.environment' => 1,
 245     'lxc.' => 1,
 246     'lxc.' => 1,
 247     'lxc.' => 1,
 248     'lxc.' => 1,
 249 };
 250
 251 my $netconf_desc = {
 252     type => {
 253         type => 'string',
 254         optional => 1,
 255         description => "Network interface type.",
 256         enum => [qw(veth)],
 257     },
 258     name => {
 259         type => 'string',
 260         format_description => 'String',
 261         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 262         pattern => '[-_.\w\d]+',
 263     },
 264     bridge => {
 265         type => 'string',
 266         format_description => 'vmbr<Number>',
 267         description => 'Bridge to attach the network device to.',
 268         pattern => '[-_.\w\d]+',
 269         optional => 1,
 270     },
 271     hwaddr => {
 272         type => 'string',
 273         format_description => 'MAC',
 274         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 275         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 276         optional => 1,
 277     },
 278     mtu => {
 279         type => 'integer',
 280         format_description => 'Number',
 281         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 282         minimum => 64, # minimum ethernet frame is 64 bytes
 283         optional => 1,
 284     },
 285     ip => {
 286         type => 'string',
 287         format => 'pve-ipv4-config',
 288         format_description => 'IPv4Format/CIDR',
 289         description => 'IPv4 address in CIDR format.',
 290         optional => 1,
 291     },
 292     gw => {
 293         type => 'string',
 294         format => 'ipv4',
 295         format_description => 'GatewayIPv4',
 296         description => 'Default gateway for IPv4 traffic.',
 297         optional => 1,
 298     },
 299     ip6 => {
 300         type => 'string',
 301         format => 'pve-ipv6-config',
 302         format_description => 'IPv6Format/CIDR',
 303         description => 'IPv6 address in CIDR format.',
 304         optional => 1,
 305     },
 306     gw6 => {
 307         type => 'string',
 308         format => 'ipv6',
 309         format_description => 'GatewayIPv6',
 310         description => 'Default gateway for IPv6 traffic.',
 311         optional => 1,
 312     },
 313     firewall => {
 314         type => 'boolean',
 315         format_description => '[1|0]',
 316         description => "Controls whether this interface's firewall rules should be used.",
 317         optional => 1,
 318     },
 319     tag => {
 320         type => 'integer',
 321         format_description => 'VlanNo',
 322         minimum => '2',
 323         maximum => '4094',
 324         description => "VLAN tag foro this interface.",
 325         optional => 1,
 326     },
 327 };
 328 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 329
 330 my $MAX_LXC_NETWORKS = 10;
 331 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 332     $confdesc->{"net$i"} = {
 333         optional => 1,
 334         type => 'string', format => $netconf_desc,
 335         description => "Specifies network interfaces for the container.",
 336     };
 337 }
 338
 339 my $mp_desc = {
 340     %$rootfs_desc,
 341     mp => {
 342         type => 'string',
 343         format_description => 'Path',
 344         description => 'Path to the mountpoint as seen from inside the container.',
 345         optional => 1,
 346     },
 347 };
 348 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 349
 350 my $unuseddesc = {
 351     optional => 1,
 352     type => 'string', format => 'pve-volume-id',
 353     description => "Reference to unused volumes.",
 354 };
 355
 356 my $MAX_MOUNT_POINTS = 10;
 357 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 358     $confdesc->{"mp$i"} = {
 359         optional => 1,
 360         type => 'string', format => $mp_desc,
 361         description => "Use volume as container mount point (experimental feature).",
 362         optional => 1,
 363     };
 364 }
 365
 366 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 367 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 368     $confdesc->{"unused$i"} = $unuseddesc;
 369 }
 370
 371 sub write_pct_config {
 372     my ($filename, $conf) = @_;
 373
 374     delete $conf->{snapstate}; # just to be sure
 375
 376     my $generate_raw_config = sub {
 377         my ($conf) = @_;
 378
 379         my $raw = '';
 380
 381         # add description as comment to top of file
 382         my $descr = $conf->{description} || '';
 383         foreach my $cl (split(/\n/, $descr)) {
 384             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 385         }
 386
 387         foreach my $key (sort keys %$conf) {
 388             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 389                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 390             my $value = $conf->{$key};
 391             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 392             $raw .= "$key: $value\n";
 393         }
 394
 395         if (my $lxcconf = $conf->{lxc}) {
 396             foreach my $entry (@$lxcconf) {
 397                 my ($k, $v) = @$entry;
 398                 $raw .= "$k: $v\n";
 399             }
 400         }
 401
 402         return $raw;
 403     };
 404
 405     my $raw = &$generate_raw_config($conf);
 406
 407     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 408         $raw .= "\n[$snapname]\n";
 409         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 410     }
 411
 412     return $raw;
 413 }
 414
 415 sub check_type {
 416     my ($key, $value) = @_;
 417
 418     die "unknown setting '$key'\n" if !$confdesc->{$key};
 419
 420     my $type = $confdesc->{$key}->{type};
 421
 422     if (!defined($value)) {
 423         die "got undefined value\n";
 424     }
 425
 426     if ($value =~ m/[\n\r]/) {
 427         die "property contains a line feed\n";
 428     }
 429
 430     if ($type eq 'boolean') {
 431         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 432         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 433         die "type check ('boolean') failed - got '$value'\n";
 434     } elsif ($type eq 'integer') {
 435         return int($1) if $value =~ m/^(\d+)$/;
 436         die "type check ('integer') failed - got '$value'\n";
 437     } elsif ($type eq 'number') {
 438         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 439         die "type check ('number') failed - got '$value'\n";
 440     } elsif ($type eq 'string') {
 441         if (my $fmt = $confdesc->{$key}->{format}) {
 442             PVE::JSONSchema::check_format($fmt, $value);
 443             return $value;
 444         }
 445         return $value;
 446     } else {
 447         die "internal error"
 448     }
 449 }
 450
 451 sub parse_pct_config {
 452     my ($filename, $raw) = @_;
 453
 454     return undef if !defined($raw);
 455
 456     my $res = {
 457         digest => Digest::SHA::sha1_hex($raw),
 458         snapshots => {},
 459     };
 460
 461     $filename =~ m|/lxc/(\d+).conf$|
 462         || die "got strange filename '$filename'";
 463
 464     my $vmid = $1;
 465
 466     my $conf = $res;
 467     my $descr = '';
 468     my $section = '';
 469
 470     my @lines = split(/\n/, $raw);
 471     foreach my $line (@lines) {
 472         next if $line =~ m/^\s*$/;
 473
 474         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 475             $section = $1;
 476             $conf->{description} = $descr if $descr;
 477             $descr = '';
 478             $conf = $res->{snapshots}->{$section} = {};
 479             next;
 480         }
 481
 482         if ($line =~ m/^\#(.*)\s*$/) {
 483             $descr .= PVE::Tools::decode_text($1) . "\n";
 484             next;
 485         }
 486
 487         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 488             my $key = $1;
 489             my $value = $3;
 490             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 491                 push @{$conf->{lxc}}, [$key, $value];
 492             } else {
 493                 warn "vm $vmid - unable to parse config: $line\n";
 494             }
 495         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 496             $descr .= PVE::Tools::decode_text($2);
 497         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 498             $conf->{snapstate} = $1;
 499         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 500             my $key = $1;
 501             my $value = $2;
 502             eval { $value = check_type($key, $value); };
 503             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 504             $conf->{$key} = $value;
 505         } else {
 506             warn "vm $vmid - unable to parse config: $line\n";
 507         }
 508     }
 509
 510     $conf->{description} = $descr if $descr;
 511
 512     delete $res->{snapstate}; # just to be sure
 513
 514     return $res;
 515 }
 516
 517 sub config_list {
 518     my $vmlist = PVE::Cluster::get_vmlist();
 519     my $res = {};
 520     return $res if !$vmlist || !$vmlist->{ids};
 521     my $ids = $vmlist->{ids};
 522
 523     foreach my $vmid (keys %$ids) {
 524         next if !$vmid; # skip CT0
 525         my $d = $ids->{$vmid};
 526         next if !$d->{node} || $d->{node} ne $nodename;
 527         next if !$d->{type} || $d->{type} ne 'lxc';
 528         $res->{$vmid}->{type} = 'lxc';
 529     }
 530     return $res;
 531 }
 532
 533 sub cfs_config_path {
 534     my ($vmid, $node) = @_;
 535
 536     $node = $nodename if !$node;
 537     return "nodes/$node/lxc/$vmid.conf";
 538 }
 539
 540 sub config_file {
 541     my ($vmid, $node) = @_;
 542
 543     my $cfspath = cfs_config_path($vmid, $node);
 544     return "/etc/pve/$cfspath";
 545 }
 546
 547 sub load_config {
 548     my ($vmid, $node) = @_;
 549
 550     $node = $nodename if !$node;
 551     my $cfspath = cfs_config_path($vmid, $node);
 552
 553     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 554     die "container $vmid does not exists\n" if !defined($conf);
 555
 556     return $conf;
 557 }
 558
 559 sub create_config {
 560     my ($vmid, $conf) = @_;
 561
 562     my $dir = "/etc/pve/nodes/$nodename/lxc";
 563     mkdir $dir;
 564
 565     write_config($vmid, $conf);
 566 }
 567
 568 sub destroy_config {
 569     my ($vmid) = @_;
 570
 571     unlink config_file($vmid, $nodename);
 572 }
 573
 574 sub write_config {
 575     my ($vmid, $conf) = @_;
 576
 577     my $cfspath = cfs_config_path($vmid);
 578
 579     PVE::Cluster::cfs_write_file($cfspath, $conf);
 580 }
 581
 582 # flock: we use one file handle per process, so lock file
 583 # can be called multiple times and succeeds for the same process.
 584
 585 my $lock_handles =  {};
 586 my $lockdir = "/run/lock/lxc";
 587
 588 sub lock_filename {
 589     my ($vmid) = @_;
 590
 591     return "$lockdir/pve-config-${vmid}.lock";
 592 }
 593
 594 sub lock_aquire {
 595     my ($vmid, $timeout) = @_;
 596
 597     $timeout = 10 if !$timeout;
 598     my $mode = LOCK_EX;
 599
 600     my $filename = lock_filename($vmid);
 601
 602     mkdir $lockdir if !-d $lockdir;
 603
 604     my $lock_func = sub {
 605         if (!$lock_handles->{$$}->{$filename}) {
 606             my $fh = new IO::File(">>$filename") ||
 607                 die "can't open file - $!\n";
 608             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 609         }
 610
 611         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 612             print STDERR "trying to aquire lock...";
 613             my $success;
 614             while(1) {
 615                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 616                 # try again on EINTR (see bug #273)
 617                 if ($success || ($! != EINTR)) {
 618                     last;
 619                 }
 620             }
 621             if (!$success) {
 622                 print STDERR " failed\n";
 623                 die "can't aquire lock - $!\n";
 624             }
 625
 626             print STDERR " OK\n";
 627         }
 628
 629         $lock_handles->{$$}->{$filename}->{refcount}++;
 630     };
 631
 632     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 633     my $err = $@;
 634     if ($err) {
 635         die "can't lock file '$filename' - $err";
 636     }
 637 }
 638
 639 sub lock_release {
 640     my ($vmid) = @_;
 641
 642     my $filename = lock_filename($vmid);
 643
 644     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 645         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 646         if ($refcount <= 0) {
 647             $lock_handles->{$$}->{$filename} = undef;
 648             close ($fh);
 649         }
 650     }
 651 }
 652
 653 sub lock_container {
 654     my ($vmid, $timeout, $code, @param) = @_;
 655
 656     my $res;
 657
 658     lock_aquire($vmid, $timeout);
 659     eval { $res = &$code(@param) };
 660     my $err = $@;
 661     lock_release($vmid);
 662
 663     die $err if $err;
 664
 665     return $res;
 666 }
 667
 668 sub option_exists {
 669     my ($name) = @_;
 670
 671     return defined($confdesc->{$name});
 672 }
 673
 674 # add JSON properties for create and set function
 675 sub json_config_properties {
 676     my $prop = shift;
 677
 678     foreach my $opt (keys %$confdesc) {
 679         next if $opt eq 'parent' || $opt eq 'snaptime';
 680         next if $prop->{$opt};
 681         $prop->{$opt} = $confdesc->{$opt};
 682     }
 683
 684     return $prop;
 685 }
 686
 687 sub json_config_properties_no_rootfs {
 688     my $prop = shift;
 689
 690     foreach my $opt (keys %$confdesc) {
 691         next if $prop->{$opt};
 692         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 693         $prop->{$opt} = $confdesc->{$opt};
 694     }
 695
 696     return $prop;
 697 }
 698
 699 # container status helpers
 700
 701 sub list_active_containers {
 702
 703     my $filename = "/proc/net/unix";
 704
 705     # similar test is used by lcxcontainers.c: list_active_containers
 706     my $res = {};
 707
 708     my $fh = IO::File->new ($filename, "r");
 709     return $res if !$fh;
 710
 711     while (defined(my $line = <$fh>)) {
 712         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 713             my $path = $1;
 714             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 715                 $res->{$1} = 1;
 716             }
 717         }
 718     }
 719
 720     close($fh);
 721
 722     return $res;
 723 }
 724
 725 # warning: this is slow
 726 sub check_running {
 727     my ($vmid) = @_;
 728
 729     my $active_hash = list_active_containers();
 730
 731     return 1 if defined($active_hash->{$vmid});
 732
 733     return undef;
 734 }
 735
 736 sub get_container_disk_usage {
 737     my ($vmid) = @_;
 738
 739     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 740
 741     my $res = {
 742         total => 0,
 743         used => 0,
 744         avail => 0,
 745     };
 746
 747     my $parser = sub {
 748         my $line = shift;
 749         if (my ($fsid, $total, $used, $avail) = $line =~
 750             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 751             $res = {
 752                 total => $total,
 753                 used => $used,
 754                 avail => $avail,
 755             };
 756         }
 757     };
 758     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 759     warn $@ if $@;
 760
 761     return $res;
 762 }
 763
 764 sub vmstatus {
 765     my ($opt_vmid) = @_;
 766
 767     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 768
 769     my $active_hash = list_active_containers();
 770
 771     foreach my $vmid (keys %$list) {
 772         my $d = $list->{$vmid};
 773
 774         my $running = defined($active_hash->{$vmid});
 775
 776         $d->{status} = $running ? 'running' : 'stopped';
 777
 778         my $cfspath = cfs_config_path($vmid);
 779         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 780
 781         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 782         $d->{name} =~ s/[\s]//g;
 783
 784         $d->{cpus} = $conf->{cpulimit} // 0;
 785
 786         if ($running) {
 787             my $res = get_container_disk_usage($vmid);
 788             $d->{disk} = $res->{used};
 789             $d->{maxdisk} = $res->{total};
 790         } else {
 791             $d->{disk} = 0;
 792             # use 4GB by default ??
 793             if (my $rootfs = $conf->{rootfs}) {
 794                 my $rootinfo = parse_ct_mountpoint($rootfs);
 795                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 796             } else {
 797                 $d->{maxdisk} = 4*1024*1024*1024;
 798             }
 799         }
 800
 801         $d->{mem} = 0;
 802         $d->{swap} = 0;
 803         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 804         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 805
 806         $d->{uptime} = 0;
 807         $d->{cpu} = 0;
 808
 809         $d->{netout} = 0;
 810         $d->{netin} = 0;
 811
 812         $d->{diskread} = 0;
 813         $d->{diskwrite} = 0;
 814
 815         $d->{template} = is_template($conf);
 816     }
 817
 818     foreach my $vmid (keys %$list) {
 819         my $d = $list->{$vmid};
 820         next if $d->{status} ne 'running';
 821
 822         my $pid = find_lxc_pid($vmid);
 823         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 824         $d->{uptime} = time - $ctime; # the method lxcfs uses
 825
 826         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 827         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 828
 829         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 830         my @bytes = split(/\n/, $blkio_bytes);
 831         foreach my $byte (@bytes) {
 832             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 833                 $d->{diskread} = $2 if $key eq 'Read';
 834                 $d->{diskwrite} = $2 if $key eq 'Write';
 835             }
 836         }
 837     }
 838
 839     return $list;
 840 }
 841
 842 sub parse_ct_mountpoint {
 843     my ($data, $noerr) = @_;
 844
 845     $data //= '';
 846
 847     my $res;
 848     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 849     if ($@) {
 850         return undef if $noerr;
 851         die $@;
 852     }
 853
 854     if (defined(my $size = $res->{size})) {
 855         $size = PVE::JSONSchema::parse_size($size);
 856         if (!defined($size)) {
 857             return undef if $noerr;
 858             die "invalid size: $size\n";
 859         }
 860         $res->{size} = $size;
 861     }
 862
 863     return $res;
 864 }
 865
 866 sub print_ct_mountpoint {
 867     my ($info, $nomp) = @_;
 868     my $skip = $nomp ? ['mp'] : [];
 869     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 870 }
 871
 872 sub print_lxc_network {
 873     my $net = shift;
 874     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 875 }
 876
 877 sub parse_lxc_network {
 878     my ($data) = @_;
 879
 880     my $res = {};
 881
 882     return $res if !$data;
 883
 884     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 885
 886     $res->{type} = 'veth';
 887     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 888
 889     return $res;
 890 }
 891
 892 sub read_cgroup_value {
 893     my ($group, $vmid, $name, $full) = @_;
 894
 895     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 896
 897     return PVE::Tools::file_get_contents($path) if $full;
 898
 899     return PVE::Tools::file_read_firstline($path);
 900 }
 901
 902 sub write_cgroup_value {
 903    my ($group, $vmid, $name, $value) = @_;
 904
 905    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 906    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 907
 908 }
 909
 910 sub find_lxc_console_pids {
 911
 912     my $res = {};
 913
 914     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 915         my ($pid) = @_;
 916
 917         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 918         return if !$cmdline;
 919
 920         my @args = split(/\0/, $cmdline);
 921
 922         # serach for lxc-console -n <vmid>
 923         return if scalar(@args) != 3;
 924         return if $args[1] ne '-n';
 925         return if $args[2] !~ m/^\d+$/;
 926         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 927
 928         my $vmid = $args[2];
 929
 930         push @{$res->{$vmid}}, $pid;
 931     });
 932
 933     return $res;
 934 }
 935
 936 sub find_lxc_pid {
 937     my ($vmid) = @_;
 938
 939     my $pid = undef;
 940     my $parser = sub {
 941         my $line = shift;
 942         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 943     };
 944     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
 945
 946     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 947
 948     return $pid;
 949 }
 950
 951 # Note: we cannot use Net:IP, because that only allows strict
 952 # CIDR networks
 953 sub parse_ipv4_cidr {
 954     my ($cidr, $noerr) = @_;
 955
 956     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
 957         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
 958     }
 959
 960     return undef if $noerr;
 961
 962     die "unable to parse ipv4 address/mask\n";
 963 }
 964
 965 sub check_lock {
 966     my ($conf) = @_;
 967
 968     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
 969 }
 970
 971 sub check_protection {
 972     my ($vm_conf, $err_msg) = @_;
 973
 974     if ($vm_conf->{protection}) {
 975         die "$err_msg - protection mode enabled\n";
 976     }
 977 }
 978
 979 sub update_lxc_config {
 980     my ($storage_cfg, $vmid, $conf) = @_;
 981
 982     my $dir = "/var/lib/lxc/$vmid";
 983
 984     if ($conf->{template}) {
 985
 986         unlink "$dir/config";
 987
 988         return;
 989     }
 990
 991     my $raw = '';
 992
 993     die "missing 'arch' - internal error" if !$conf->{arch};
 994     $raw .= "lxc.arch = $conf->{arch}\n";
 995
 996     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
 997     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
 998         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
 999     } else {
1000         die "implement me";
1001     }
1002
1003     if (!has_dev_console($conf)) {
1004         $raw .= "lxc.console = none\n";
1005         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1006     }
1007
1008     my $ttycount = get_tty_count($conf);
1009     $raw .= "lxc.tty = $ttycount\n";
1010
1011     # some init scripts expects a linux terminal (turnkey).
1012     $raw .= "lxc.environment = TERM=linux\n";
1013
1014     my $utsname = $conf->{hostname} || "CT$vmid";
1015     $raw .= "lxc.utsname = $utsname\n";
1016
1017     my $memory = $conf->{memory} || 512;
1018     my $swap = $conf->{swap} // 0;
1019
1020     my $lxcmem = int($memory*1024*1024);
1021     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1022
1023     my $lxcswap = int(($memory + $swap)*1024*1024);
1024     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1025
1026     if (my $cpulimit = $conf->{cpulimit}) {
1027         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1028         my $value = int(100000*$cpulimit);
1029         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1030     }
1031
1032     my $shares = $conf->{cpuunits} || 1024;
1033     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1034
1035     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1036     $mountpoint->{mp} = '/';
1037
1038     my ($path, $use_loopdev) = mountpoint_mount_path($mountpoint, $storage_cfg);
1039     $path = "loop:$path" if $use_loopdev;
1040
1041     $raw .= "lxc.rootfs = $path\n";
1042
1043     my $netcount = 0;
1044     foreach my $k (keys %$conf) {
1045         next if $k !~ m/^net(\d+)$/;
1046         my $ind = $1;
1047         my $d = parse_lxc_network($conf->{$k});
1048         $netcount++;
1049         $raw .= "lxc.network.type = veth\n";
1050         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1051         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1052         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1053         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1054     }
1055
1056     if (my $lxcconf = $conf->{lxc}) {
1057         foreach my $entry (@$lxcconf) {
1058             my ($k, $v) = @$entry;
1059             $netcount++ if $k eq 'lxc.network.type';
1060             $raw .= "$k = $v\n";
1061         }
1062     }
1063
1064     $raw .= "lxc.network.type = empty\n" if !$netcount;
1065
1066     File::Path::mkpath("$dir/rootfs");
1067
1068     PVE::Tools::file_set_contents("$dir/config", $raw);
1069 }
1070
1071 # verify and cleanup nameserver list (replace \0 with ' ')
1072 sub verify_nameserver_list {
1073     my ($nameserver_list) = @_;
1074
1075     my @list = ();
1076     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1077         PVE::JSONSchema::pve_verify_ip($server);
1078         push @list, $server;
1079     }
1080
1081     return join(' ', @list);
1082 }
1083
1084 sub verify_searchdomain_list {
1085     my ($searchdomain_list) = @_;
1086
1087     my @list = ();
1088     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1089         # todo: should we add checks for valid dns domains?
1090         push @list, $server;
1091     }
1092
1093     return join(' ', @list);
1094 }
1095
1096 sub add_unused_volume {
1097     my ($config, $volid) = @_;
1098
1099     # skip bind mounts and block devices
1100     return if $volid =~ m|^/|;
1101
1102     my $key;
1103     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1104         my $test = "unused$ind";
1105         if (my $vid = $config->{$test}) {
1106             return if $vid eq $volid; # do not add duplicates
1107         } else {
1108             $key = $test;
1109         }
1110     }
1111
1112     die "To many unused volume - please delete them first.\n" if !$key;
1113
1114     $config->{$key} = $volid;
1115
1116     return $key;
1117 }
1118
1119 sub update_pct_config {
1120     my ($vmid, $conf, $running, $param, $delete) = @_;
1121
1122     my @nohotplug;
1123
1124     my $new_disks = 0;
1125     my @deleted_volumes;
1126
1127     my $rootdir;
1128     if ($running) {
1129         my $pid = find_lxc_pid($vmid);
1130         $rootdir = "/proc/$pid/root";
1131     }
1132
1133     my $hotplug_error = sub {
1134         if ($running) {
1135             push @nohotplug, @_;
1136             return 1;
1137         } else {
1138             return 0;
1139         }
1140     };
1141
1142     if (defined($delete)) {
1143         foreach my $opt (@$delete) {
1144             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1145                 die "unable to delete required option '$opt'\n";
1146             } elsif ($opt eq 'swap') {
1147                 delete $conf->{$opt};
1148                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1149             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1150                 delete $conf->{$opt};
1151             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1152                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1153                 next if $hotplug_error->($opt);
1154                 delete $conf->{$opt};
1155             } elsif ($opt =~ m/^net(\d)$/) {
1156                 delete $conf->{$opt};
1157                 next if !$running;
1158                 my $netid = $1;
1159                 PVE::Network::veth_delete("veth${vmid}i$netid");
1160             } elsif ($opt eq 'protection') {
1161                 delete $conf->{$opt};
1162             } elsif ($opt =~ m/^unused(\d+)$/) {
1163                 next if $hotplug_error->($opt);
1164                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1165                 push @deleted_volumes, $conf->{$opt};
1166                 delete $conf->{$opt};
1167             } elsif ($opt =~ m/^mp(\d+)$/) {
1168                 next if $hotplug_error->($opt);
1169                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1170                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1171                 add_unused_volume($conf, $mountpoint->{volume});
1172                 delete $conf->{$opt};
1173             } else {
1174                 die "implement me"
1175             }
1176             write_config($vmid, $conf) if $running;
1177         }
1178     }
1179
1180     # There's no separate swap size to configure, there's memory and "total"
1181     # memory (iow. memory+swap). This means we have to change them together.
1182     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1183     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1184     if (defined($wanted_memory) || defined($wanted_swap)) {
1185
1186         $wanted_memory //= ($conf->{memory} || 512);
1187         $wanted_swap //=  ($conf->{swap} || 0);
1188
1189         my $total = $wanted_memory + $wanted_swap;
1190         if ($running) {
1191             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1192             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1193         }
1194         $conf->{memory} = $wanted_memory;
1195         $conf->{swap} = $wanted_swap;
1196
1197         write_config($vmid, $conf) if $running;
1198     }
1199
1200     foreach my $opt (keys %$param) {
1201         my $value = $param->{$opt};
1202         if ($opt eq 'hostname') {
1203             $conf->{$opt} = $value;
1204         } elsif ($opt eq 'onboot') {
1205             $conf->{$opt} = $value ? 1 : 0;
1206         } elsif ($opt eq 'startup') {
1207             $conf->{$opt} = $value;
1208         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1209             next if $hotplug_error->($opt);
1210             $conf->{$opt} = $value;
1211         } elsif ($opt eq 'nameserver') {
1212             next if $hotplug_error->($opt);
1213             my $list = verify_nameserver_list($value);
1214             $conf->{$opt} = $list;
1215         } elsif ($opt eq 'searchdomain') {
1216             next if $hotplug_error->($opt);
1217             my $list = verify_searchdomain_list($value);
1218             $conf->{$opt} = $list;
1219         } elsif ($opt eq 'cpulimit') {
1220             next if $hotplug_error->($opt); # FIXME: hotplug
1221             $conf->{$opt} = $value;
1222         } elsif ($opt eq 'cpuunits') {
1223             $conf->{$opt} = $value;
1224             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1225         } elsif ($opt eq 'description') {
1226             $conf->{$opt} = PVE::Tools::encode_text($value);
1227         } elsif ($opt =~ m/^net(\d+)$/) {
1228             my $netid = $1;
1229             my $net = parse_lxc_network($value);
1230             if (!$running) {
1231                 $conf->{$opt} = print_lxc_network($net);
1232             } else {
1233                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1234             }
1235         } elsif ($opt eq 'protection') {
1236             $conf->{$opt} = $value ? 1 : 0;
1237         } elsif ($opt =~ m/^mp(\d+)$/) {
1238             next if $hotplug_error->($opt);
1239             check_protection($conf, "can't update CT $vmid drive '$opt'");
1240             $conf->{$opt} = $value;
1241             $new_disks = 1;
1242         } elsif ($opt eq 'rootfs') {
1243             check_protection($conf, "can't update CT $vmid drive '$opt'");
1244             die "implement me: $opt";
1245         } else {
1246             die "implement me: $opt";
1247         }
1248         write_config($vmid, $conf) if $running;
1249     }
1250
1251     if (@deleted_volumes) {
1252         my $storage_cfg = PVE::Storage::config();
1253         foreach my $volume (@deleted_volumes) {
1254             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1255         }
1256     }
1257
1258     if ($new_disks) {
1259         my $storage_cfg = PVE::Storage::config();
1260         create_disks($storage_cfg, $vmid, $conf, $conf);
1261     }
1262
1263     # This should be the last thing we do here
1264     if ($running && scalar(@nohotplug)) {
1265         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1266     }
1267 }
1268
1269 sub has_dev_console {
1270     my ($conf) = @_;
1271
1272     return !(defined($conf->{console}) && !$conf->{console});
1273 }
1274
1275 sub get_tty_count {
1276     my ($conf) = @_;
1277
1278     return $conf->{tty} // $confdesc->{tty}->{default};
1279 }
1280
1281 sub get_cmode {
1282     my ($conf) = @_;
1283
1284     return $conf->{cmode} // $confdesc->{cmode}->{default};
1285 }
1286
1287 sub get_console_command {
1288     my ($vmid, $conf) = @_;
1289
1290     my $cmode = get_cmode($conf);
1291
1292     if ($cmode eq 'console') {
1293         return ['lxc-console', '-n',  $vmid, '-t', 0];
1294     } elsif ($cmode eq 'tty') {
1295         return ['lxc-console', '-n',  $vmid];
1296     } elsif ($cmode eq 'shell') {
1297         return ['lxc-attach', '--clear-env', '-n', $vmid];
1298     } else {
1299         die "internal error";
1300     }
1301 }
1302
1303 sub get_primary_ips {
1304     my ($conf) = @_;
1305
1306     # return data from net0
1307
1308     return undef if !defined($conf->{net0});
1309     my $net = parse_lxc_network($conf->{net0});
1310
1311     my $ipv4 = $net->{ip};
1312     if ($ipv4) {
1313         if ($ipv4 =~ /^(dhcp|manual)$/) {
1314             $ipv4 = undef
1315         } else {
1316             $ipv4 =~ s!/\d+$!!;
1317         }
1318     }
1319     my $ipv6 = $net->{ip6};
1320     if ($ipv6) {
1321         if ($ipv6 =~ /^(dhcp|manual)$/) {
1322             $ipv6 = undef;
1323         } else {
1324             $ipv6 =~ s!/\d+$!!;
1325         }
1326     }
1327
1328     return ($ipv4, $ipv6);
1329 }
1330
1331 sub delete_mountpoint_volume {
1332     my ($storage_cfg, $vmid, $volume) = @_;
1333
1334     # skip bind mounts and block devices
1335     if ($volume =~ m|^/|) {
1336             return;
1337     }
1338
1339     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1340     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1341 }
1342
1343 sub destroy_lxc_container {
1344     my ($storage_cfg, $vmid, $conf) = @_;
1345
1346     foreach_mountpoint($conf, sub {
1347         my ($ms, $mountpoint) = @_;
1348         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1349     });
1350
1351     rmdir "/var/lib/lxc/$vmid/rootfs";
1352     unlink "/var/lib/lxc/$vmid/config";
1353     rmdir "/var/lib/lxc/$vmid";
1354     destroy_config($vmid);
1355
1356     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1357     #PVE::Tools::run_command($cmd);
1358 }
1359
1360 sub vm_stop_cleanup {
1361     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1362
1363     eval {
1364         if (!$keepActive) {
1365
1366             my $vollist = get_vm_volumes($conf);
1367             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1368         }
1369     };
1370     warn $@ if $@; # avoid errors - just warn
1371 }
1372
1373 my $safe_num_ne = sub {
1374     my ($a, $b) = @_;
1375
1376     return 0 if !defined($a) && !defined($b);
1377     return 1 if !defined($a);
1378     return 1 if !defined($b);
1379
1380     return $a != $b;
1381 };
1382
1383 my $safe_string_ne = sub {
1384     my ($a, $b) = @_;
1385
1386     return 0 if !defined($a) && !defined($b);
1387     return 1 if !defined($a);
1388     return 1 if !defined($b);
1389
1390     return $a ne $b;
1391 };
1392
1393 sub update_net {
1394     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1395
1396     if ($newnet->{type} ne 'veth') {
1397         # for when there are physical interfaces
1398         die "cannot update interface of type $newnet->{type}";
1399     }
1400
1401     my $veth = "veth${vmid}i${netid}";
1402     my $eth = $newnet->{name};
1403
1404     if (my $oldnetcfg = $conf->{$opt}) {
1405         my $oldnet = parse_lxc_network($oldnetcfg);
1406
1407         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1408             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1409
1410             PVE::Network::veth_delete($veth);
1411             delete $conf->{$opt};
1412             write_config($vmid, $conf);
1413
1414             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1415
1416         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1417                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1418                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1419
1420                 if ($oldnet->{bridge}) {
1421                     PVE::Network::tap_unplug($veth);
1422                     foreach (qw(bridge tag firewall)) {
1423                         delete $oldnet->{$_};
1424                     }
1425                     $conf->{$opt} = print_lxc_network($oldnet);
1426                     write_config($vmid, $conf);
1427                 }
1428
1429                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1430                 foreach (qw(bridge tag firewall)) {
1431                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1432                 }
1433                 $conf->{$opt} = print_lxc_network($oldnet);
1434                 write_config($vmid, $conf);
1435         }
1436     } else {
1437         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1438     }
1439
1440     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1441 }
1442
1443 sub hotplug_net {
1444     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1445
1446     my $veth = "veth${vmid}i${netid}";
1447     my $vethpeer = $veth . "p";
1448     my $eth = $newnet->{name};
1449
1450     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1451     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1452
1453     # attach peer in container
1454     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1455     PVE::Tools::run_command($cmd);
1456
1457     # link up peer in container
1458     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1459     PVE::Tools::run_command($cmd);
1460
1461     my $done = { type => 'veth' };
1462     foreach (qw(bridge tag firewall hwaddr name)) {
1463         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1464     }
1465     $conf->{$opt} = print_lxc_network($done);
1466
1467     write_config($vmid, $conf);
1468 }
1469
1470 sub update_ipconfig {
1471     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1472
1473     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1474
1475     my $optdata = parse_lxc_network($conf->{$opt});
1476     my $deleted = [];
1477     my $added = [];
1478     my $nscmd = sub {
1479         my $cmdargs = shift;
1480         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1481     };
1482     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1483
1484     my $change_ip_config = sub {
1485         my ($ipversion) = @_;
1486
1487         my $family_opt = "-$ipversion";
1488         my $suffix = $ipversion == 4 ? '' : $ipversion;
1489         my $gw= "gw$suffix";
1490         my $ip= "ip$suffix";
1491
1492         my $newip = $newnet->{$ip};
1493         my $newgw = $newnet->{$gw};
1494         my $oldip = $optdata->{$ip};
1495
1496         my $change_ip = &$safe_string_ne($oldip, $newip);
1497         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1498
1499         return if !$change_ip && !$change_gw;
1500
1501         # step 1: add new IP, if this fails we cancel
1502         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1503         if ($change_ip && $is_real_ip) {
1504             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1505             if (my $err = $@) {
1506                 warn $err;
1507                 return;
1508             }
1509         }
1510
1511         # step 2: replace gateway
1512         #   If this fails we delete the added IP and cancel.
1513         #   If it succeeds we save the config and delete the old IP, ignoring
1514         #   errors. The config is then saved.
1515         # Note: 'ip route replace' can add
1516         if ($change_gw) {
1517             if ($newgw) {
1518                 eval {
1519                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1520                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1521                     }
1522                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1523                 };
1524                 if (my $err = $@) {
1525                     warn $err;
1526                     # the route was not replaced, the old IP is still available
1527                     # rollback (delete new IP) and cancel
1528                     if ($change_ip) {
1529                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1530                         warn $@ if $@; # no need to die here
1531                     }
1532                     return;
1533                 }
1534             } else {
1535                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1536                 # if the route was not deleted, the guest might have deleted it manually
1537                 # warn and continue
1538                 warn $@ if $@;
1539             }
1540         }
1541
1542         # from this point on we save the configuration
1543         # step 3: delete old IP ignoring errors
1544         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1545             # We need to enable promote_secondaries, otherwise our newly added
1546             # address will be removed along with the old one.
1547             my $promote = 0;
1548             eval {
1549                 if ($ipversion == 4) {
1550                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1551                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1552                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1553                 }
1554                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1555             };
1556             warn $@ if $@; # no need to die here
1557
1558             if ($ipversion == 4) {
1559                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1560             }
1561         }
1562
1563         foreach my $property ($ip, $gw) {
1564             if ($newnet->{$property}) {
1565                 $optdata->{$property} = $newnet->{$property};
1566             } else {
1567                 delete $optdata->{$property};
1568             }
1569         }
1570         $conf->{$opt} = print_lxc_network($optdata);
1571         write_config($vmid, $conf);
1572         $lxc_setup->setup_network($conf);
1573     };
1574
1575     &$change_ip_config(4);
1576     &$change_ip_config(6);
1577
1578 }
1579
1580 # Internal snapshots
1581
1582 # NOTE: Snapshot create/delete involves several non-atomic
1583 # action, and can take a long time.
1584 # So we try to avoid locking the file and use 'lock' variable
1585 # inside the config file instead.
1586
1587 my $snapshot_copy_config = sub {
1588     my ($source, $dest) = @_;
1589
1590     foreach my $k (keys %$source) {
1591         next if $k eq 'snapshots';
1592         next if $k eq 'snapstate';
1593         next if $k eq 'snaptime';
1594         next if $k eq 'vmstate';
1595         next if $k eq 'lock';
1596         next if $k eq 'digest';
1597         next if $k eq 'description';
1598
1599         $dest->{$k} = $source->{$k};
1600     }
1601 };
1602
1603 my $snapshot_prepare = sub {
1604     my ($vmid, $snapname, $comment) = @_;
1605
1606     my $snap;
1607
1608     my $updatefn =  sub {
1609
1610         my $conf = load_config($vmid);
1611
1612         die "you can't take a snapshot if it's a template\n"
1613             if is_template($conf);
1614
1615         check_lock($conf);
1616
1617         $conf->{lock} = 'snapshot';
1618
1619         die "snapshot name '$snapname' already used\n"
1620             if defined($conf->{snapshots}->{$snapname});
1621
1622         my $storecfg = PVE::Storage::config();
1623         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1624
1625         $snap = $conf->{snapshots}->{$snapname} = {};
1626
1627         &$snapshot_copy_config($conf, $snap);
1628
1629         $snap->{'snapstate'} = "prepare";
1630         $snap->{'snaptime'} = time();
1631         $snap->{'description'} = $comment if $comment;
1632         $conf->{snapshots}->{$snapname} = $snap;
1633
1634         write_config($vmid, $conf);
1635     };
1636
1637     lock_container($vmid, 10, $updatefn);
1638
1639     return $snap;
1640 };
1641
1642 my $snapshot_commit = sub {
1643     my ($vmid, $snapname) = @_;
1644
1645     my $updatefn = sub {
1646
1647         my $conf = load_config($vmid);
1648
1649         die "missing snapshot lock\n"
1650             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1651
1652         die "snapshot '$snapname' does not exist\n"
1653             if !defined($conf->{snapshots}->{$snapname});
1654
1655         die "wrong snapshot state\n"
1656             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1657                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1658
1659         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1660         delete $conf->{lock};
1661         $conf->{parent} = $snapname;
1662
1663         write_config($vmid, $conf);
1664     };
1665
1666     lock_container($vmid, 10 ,$updatefn);
1667 };
1668
1669 sub has_feature {
1670     my ($feature, $conf, $storecfg, $snapname) = @_;
1671
1672     my $err;
1673
1674     foreach_mountpoint($conf, sub {
1675         my ($ms, $mountpoint) = @_;
1676
1677         return if $err; # skip further test
1678
1679         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1680
1681         # TODO: implement support for mountpoints
1682         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1683             if $ms ne 'rootfs';
1684     });
1685
1686     return $err ? 0 : 1;
1687 }
1688
1689 sub snapshot_create {
1690     my ($vmid, $snapname, $comment) = @_;
1691
1692     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1693
1694     my $conf = load_config($vmid);
1695
1696     my $running = check_running($vmid);
1697     eval {
1698         if ($running) {
1699             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1700             PVE::Tools::run_command(['/bin/sync']);
1701         };
1702
1703         my $storecfg = PVE::Storage::config();
1704         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1705         my $volid = $rootinfo->{volume};
1706
1707         if ($running) {
1708             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1709         };
1710
1711         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1712         &$snapshot_commit($vmid, $snapname);
1713     };
1714     if(my $err = $@) {
1715         snapshot_delete($vmid, $snapname, 1);
1716         die "$err\n";
1717     }
1718 }
1719
1720 sub snapshot_delete {
1721     my ($vmid, $snapname, $force) = @_;
1722
1723     my $snap;
1724
1725     my $conf;
1726
1727     my $updatefn =  sub {
1728
1729         $conf = load_config($vmid);
1730
1731         die "you can't delete a snapshot if vm is a template\n"
1732             if is_template($conf);
1733
1734         $snap = $conf->{snapshots}->{$snapname};
1735
1736         check_lock($conf);
1737
1738         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1739
1740         $snap->{snapstate} = 'delete';
1741
1742         write_config($vmid, $conf);
1743     };
1744
1745     lock_container($vmid, 10, $updatefn);
1746
1747     my $storecfg = PVE::Storage::config();
1748
1749     my $del_snap =  sub {
1750
1751         check_lock($conf);
1752
1753         if ($conf->{parent} eq $snapname) {
1754             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1755                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1756             } else {
1757                 delete $conf->{parent};
1758             }
1759         }
1760
1761         delete $conf->{snapshots}->{$snapname};
1762
1763         write_config($vmid, $conf);
1764     };
1765
1766     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1767     my $rootinfo = parse_ct_mountpoint($rootfs);
1768     my $volid = $rootinfo->{volume};
1769
1770     eval {
1771         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1772     };
1773     my $err = $@;
1774
1775     if(!$err || ($err && $force)) {
1776         lock_container($vmid, 10, $del_snap);
1777         if ($err) {
1778             die "Can't delete snapshot: $vmid $snapname $err\n";
1779         }
1780     }
1781 }
1782
1783 sub snapshot_rollback {
1784     my ($vmid, $snapname) = @_;
1785
1786     my $storecfg = PVE::Storage::config();
1787
1788     my $conf = load_config($vmid);
1789
1790     die "you can't rollback if vm is a template\n" if is_template($conf);
1791
1792     my $snap = $conf->{snapshots}->{$snapname};
1793
1794     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1795
1796     my $rootfs = $snap->{rootfs};
1797     my $rootinfo = parse_ct_mountpoint($rootfs);
1798     my $volid = $rootinfo->{volume};
1799
1800     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1801
1802     my $updatefn = sub {
1803
1804         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1805             if $snap->{snapstate};
1806
1807         check_lock($conf);
1808
1809         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1810
1811         die "unable to rollback vm $vmid: vm is running\n"
1812             if check_running($vmid);
1813
1814         $conf->{lock} = 'rollback';
1815
1816         my $forcemachine;
1817
1818         # copy snapshot config to current config
1819
1820         my $tmp_conf = $conf;
1821         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1822         $conf->{snapshots} = $tmp_conf->{snapshots};
1823         delete $conf->{snaptime};
1824         delete $conf->{snapname};
1825         $conf->{parent} = $snapname;
1826
1827         write_config($vmid, $conf);
1828     };
1829
1830     my $unlockfn = sub {
1831         delete $conf->{lock};
1832         write_config($vmid, $conf);
1833     };
1834
1835     lock_container($vmid, 10, $updatefn);
1836
1837     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1838
1839     lock_container($vmid, 5, $unlockfn);
1840 }
1841
1842 sub template_create {
1843     my ($vmid, $conf) = @_;
1844
1845     my $storecfg = PVE::Storage::config();
1846
1847     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1848     my $volid = $rootinfo->{volume};
1849
1850     die "Template feature is not available for '$volid'\n"
1851         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1852
1853     PVE::Storage::activate_volumes($storecfg, [$volid]);
1854
1855     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1856     $rootinfo->{volume} = $template_volid;
1857     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1858
1859     write_config($vmid, $conf);
1860 }
1861
1862 sub is_template {
1863     my ($conf) = @_;
1864
1865     return 1 if defined $conf->{template} && $conf->{template} == 1;
1866 }
1867
1868 sub mountpoint_names {
1869     my ($reverse) = @_;
1870
1871     my @names = ('rootfs');
1872
1873     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1874         push @names, "mp$i";
1875     }
1876
1877     return $reverse ? reverse @names : @names;
1878 }
1879
1880 # The container might have *different* symlinks than the host. realpath/abs_path
1881 # use the actual filesystem to resolve links.
1882 sub sanitize_mountpoint {
1883     my ($mp) = @_;
1884     $mp = '/' . $mp; # we always start with a slash
1885     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1886     $mp =~ s@/\./@@g; # collapse /./
1887     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1888     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1889     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1890     return $mp;
1891 }
1892
1893 sub foreach_mountpoint_full {
1894     my ($conf, $reverse, $func) = @_;
1895
1896     foreach my $key (mountpoint_names($reverse)) {
1897         my $value = $conf->{$key};
1898         next if !defined($value);
1899         my $mountpoint = parse_ct_mountpoint($value, 1);
1900         next if !defined($mountpoint);
1901
1902         # just to be sure: rootfs is /
1903         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1904         $mountpoint->{mp} = sanitize_mountpoint($path);
1905
1906         $path = $mountpoint->{volume};
1907         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1908
1909         &$func($key, $mountpoint);
1910     }
1911 }
1912
1913 sub foreach_mountpoint {
1914     my ($conf, $func) = @_;
1915
1916     foreach_mountpoint_full($conf, 0, $func);
1917 }
1918
1919 sub foreach_mountpoint_reverse {
1920     my ($conf, $func) = @_;
1921
1922     foreach_mountpoint_full($conf, 1, $func);
1923 }
1924
1925 sub check_ct_modify_config_perm {
1926     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1927
1928     return 1 if $authuser ne 'root@pam';
1929
1930     foreach my $opt (@$key_list) {
1931
1932         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1933             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1934         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1935             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1936         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1937             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1938         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1939                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1940             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1941         } else {
1942             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1943         }
1944     }
1945
1946     return 1;
1947 }
1948
1949 sub umount_all {
1950     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
1951
1952     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1953     my $volid_list = get_vm_volumes($conf);
1954
1955     foreach_mountpoint_reverse($conf, sub {
1956         my ($ms, $mountpoint) = @_;
1957
1958         my $volid = $mountpoint->{volume};
1959         my $mount = $mountpoint->{mp};
1960
1961         return if !$volid || !$mount;
1962
1963         my $mount_path = "$rootdir/$mount";
1964         $mount_path =~ s!/+!/!g;
1965
1966         return if !PVE::ProcFSTools::is_mounted($mount_path);
1967
1968         eval {
1969             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1970         };
1971         if (my $err = $@) {
1972             if ($noerr) {
1973                 warn $err;
1974             } else {
1975                 die $err;
1976             }
1977         }
1978     });
1979 }
1980
1981 sub mount_all {
1982     my ($vmid, $storage_cfg, $conf) = @_;
1983
1984     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1985     File::Path::make_path($rootdir);
1986
1987     my $volid_list = get_vm_volumes($conf);
1988     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
1989
1990     eval {
1991         foreach_mountpoint($conf, sub {
1992             my ($ms, $mountpoint) = @_;
1993
1994             my $volid = $mountpoint->{volume};
1995             my $mount = $mountpoint->{mp};
1996
1997             return if !$volid || !$mount;
1998
1999             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2000             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2001                 PVE::Storage::parse_volname($storage_cfg, $volid);
2002
2003             die "unable to mount base volume - internal error" if $isBase;
2004
2005             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2006         });
2007     };
2008     if (my $err = $@) {
2009         warn "mounting container failed - $err";
2010         umount_all($vmid, $storage_cfg, $conf, 1);
2011     }
2012
2013     return $rootdir;
2014 }
2015
2016
2017 sub mountpoint_mount_path {
2018     my ($mountpoint, $storage_cfg, $snapname) = @_;
2019
2020     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2021 }
2022
2023 my $check_mount_path = sub {
2024     my ($path) = @_;
2025     $path = File::Spec->canonpath($path);
2026     my $real = Cwd::realpath($path);
2027     if ($real ne $path) {
2028         die "mount path modified by symlink: $path != $real";
2029     }
2030 };
2031
2032 # use $rootdir = undef to just return the corresponding mount path
2033 sub mountpoint_mount {
2034     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2035
2036     my $volid = $mountpoint->{volume};
2037     my $mount = $mountpoint->{mp};
2038
2039     return if !$volid || !$mount;
2040
2041     my $mount_path;
2042
2043     if (defined($rootdir)) {
2044         $rootdir =~ s!/+$!!;
2045         $mount_path = "$rootdir/$mount";
2046         $mount_path =~ s!/+!/!g;
2047         &$check_mount_path($mount_path);
2048         File::Path::mkpath($mount_path);
2049     }
2050
2051     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2052
2053     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2054
2055     if ($storage) {
2056
2057         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2058         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2059
2060         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2061             PVE::Storage::parse_volname($storage_cfg, $volid);
2062
2063         if ($format eq 'subvol') {
2064             if ($mount_path) {
2065                 if ($snapname) {
2066                     if ($scfg->{type} eq 'zfspool') {
2067                         my $path_arg = $path;
2068                         $path_arg =~ s!^/+!!;
2069                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2070                     } else {
2071                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2072                     }
2073                 } else {
2074                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2075                 }
2076             }
2077             return wantarray ? ($path, 0) : $path;
2078         } elsif ($format eq 'raw') {
2079             my $use_loopdev = 0;
2080             my @extra_opts;
2081             if ($scfg->{path}) {
2082                 push @extra_opts, '-o', 'loop';
2083                 $use_loopdev = 1;
2084             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2085                 # do nothing
2086             } else {
2087                 die "unsupported storage type '$scfg->{type}'\n";
2088             }
2089             if ($mount_path) {
2090                 if ($isBase || defined($snapname)) {
2091                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2092                 } else {
2093                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2094                 }
2095             }
2096             return wantarray ? ($path, $use_loopdev) : $path;
2097         } else {
2098             die "unsupported image format '$format'\n";
2099         }
2100     } elsif ($volid =~ m|^/dev/.+|) {
2101         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2102         return wantarray ? ($volid, 0) : $volid;
2103     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2104         &$check_mount_path($volid);
2105         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2106         return wantarray ? ($volid, 0) : $volid;
2107     }
2108
2109     die "unsupported storage";
2110 }
2111
2112 sub get_vm_volumes {
2113     my ($conf, $excludes) = @_;
2114
2115     my $vollist = [];
2116
2117     foreach_mountpoint($conf, sub {
2118         my ($ms, $mountpoint) = @_;
2119
2120         return if $excludes && $ms eq $excludes;
2121
2122         my $volid = $mountpoint->{volume};
2123
2124         return if !$volid || $volid =~ m|^/|;
2125
2126         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2127         return if !$sid;
2128
2129         push @$vollist, $volid;
2130     });
2131
2132     return $vollist;
2133 }
2134
2135 sub mkfs {
2136     my ($dev) = @_;
2137
2138     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2139 }
2140
2141 sub format_disk {
2142     my ($storage_cfg, $volid) = @_;
2143
2144     if ($volid =~ m!^/dev/.+!) {
2145         mkfs($volid);
2146         return;
2147     }
2148
2149     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2150
2151     die "cannot format volume '$volid' with no storage\n" if !$storage;
2152
2153     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2154
2155     my $path = PVE::Storage::path($storage_cfg, $volid);
2156
2157     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2158         PVE::Storage::parse_volname($storage_cfg, $volid);
2159
2160     die "cannot format volume '$volid' (format == $format)\n"
2161         if $format ne 'raw';
2162
2163     mkfs($path);
2164 }
2165
2166 sub destroy_disks {
2167     my ($storecfg, $vollist) = @_;
2168
2169     foreach my $volid (@$vollist) {
2170         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2171         warn $@ if $@;
2172     }
2173 }
2174
2175 sub create_disks {
2176     my ($storecfg, $vmid, $settings, $conf) = @_;
2177
2178     my $vollist = [];
2179
2180     eval {
2181         foreach_mountpoint($settings, sub {
2182             my ($ms, $mountpoint) = @_;
2183
2184             my $volid = $mountpoint->{volume};
2185             my $mp = $mountpoint->{mp};
2186
2187             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2188
2189             return if !$storage;
2190
2191             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2192                 my ($storeid, $size_gb) = ($1, $2);
2193
2194                 my $size_kb = int(${size_gb}*1024) * 1024;
2195
2196                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2197                 # fixme: use better naming ct-$vmid-disk-X.raw?
2198
2199                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2200                     if ($size_kb > 0) {
2201                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2202                                                            undef, $size_kb);
2203                         format_disk($storecfg, $volid);
2204                     } else {
2205                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2206                                                            undef, 0);
2207                     }
2208                 } elsif ($scfg->{type} eq 'zfspool') {
2209
2210                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2211                                                        undef, $size_kb);
2212                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2213
2214                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2215                     format_disk($storecfg, $volid);
2216
2217                 } elsif ($scfg->{type} eq 'rbd') {
2218
2219                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2220                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2221                     format_disk($storecfg, $volid);
2222                 } else {
2223                     die "unable to create containers on storage type '$scfg->{type}'\n";
2224                 }
2225                 push @$vollist, $volid;
2226                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2227                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2228             } else {
2229                 # use specified/existing volid
2230             }
2231         });
2232     };
2233     # free allocated images on error
2234     if (my $err = $@) {
2235         destroy_disks($storecfg, $vollist);
2236         die $err;
2237     }
2238     return $vollist;
2239 }
2240
2241 # bash completion helper
2242
2243 sub complete_os_templates {
2244     my ($cmdname, $pname, $cvalue) = @_;
2245
2246     my $cfg = PVE::Storage::config();
2247
2248     my $storeid;
2249
2250     if ($cvalue =~ m/^([^:]+):/) {
2251         $storeid = $1;
2252     }
2253
2254     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2255     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2256
2257     my $res = [];
2258     foreach my $id (keys %$data) {
2259         foreach my $item (@{$data->{$id}}) {
2260             push @$res, $item->{volid} if defined($item->{volid});
2261         }
2262     }
2263
2264     return $res;
2265 }
2266
2267 my $complete_ctid_full = sub {
2268     my ($running) = @_;
2269
2270     my $idlist = vmstatus();
2271
2272     my $active_hash = list_active_containers();
2273
2274     my $res = [];
2275
2276     foreach my $id (keys %$idlist) {
2277         my $d = $idlist->{$id};
2278         if (defined($running)) {
2279             next if $d->{template};
2280             next if $running && !$active_hash->{$id};
2281             next if !$running && $active_hash->{$id};
2282         }
2283         push @$res, $id;
2284
2285     }
2286     return $res;
2287 };
2288
2289 sub complete_ctid {
2290     return &$complete_ctid_full();
2291 }
2292
2293 sub complete_ctid_stopped {
2294     return &$complete_ctid_full(0);
2295 }
2296
2297 sub complete_ctid_running {
2298     return &$complete_ctid_full(1);
2299 }
2300
2301 1;