src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21
  22 use Data::Dumper;
  23
  24 my $nodename = PVE::INotify::nodename();
  25
  26 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  27
  28 my $rootfs_desc = {
  29     volume => {
  30         type => 'string',
  31         default_key => 1,
  32         format_description => 'volume',
  33         description => 'Volume, device or directory to mount into the container.',
  34     },
  35     backup => {
  36         type => 'boolean',
  37         format_description => '[1|0]',
  38         description => 'Whether to include the mountpoint in backups.',
  39         optional => 1,
  40     },
  41     size => {
  42         type => 'disk-size',
  43         format_description => 'DiskSize',
  44         description => 'Volume size (read only value).',
  45         optional => 1,
  46     },
  47 };
  48
  49 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  50     type => 'string', format => $rootfs_desc,
  51     description => "Use volume as container root.",
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  56     description => "The name of the snapshot.",
  57     type => 'string', format => 'pve-configid',
  58     maxLength => 40,
  59 });
  60
  61 my $confdesc = {
  62     lock => {
  63         optional => 1,
  64         type => 'string',
  65         description => "Lock/unlock the VM.",
  66         enum => [qw(migrate backup snapshot rollback)],
  67     },
  68     onboot => {
  69         optional => 1,
  70         type => 'boolean',
  71         description => "Specifies whether a VM will be started during system bootup.",
  72         default => 0,
  73     },
  74     startup => get_standard_option('pve-startup-order'),
  75     template => {
  76         optional => 1,
  77         type => 'boolean',
  78         description => "Enable/disable Template.",
  79         default => 0,
  80     },
  81     arch => {
  82         optional => 1,
  83         type => 'string',
  84         enum => ['amd64', 'i386'],
  85         description => "OS architecture type.",
  86         default => 'amd64',
  87     },
  88     ostype => {
  89         optional => 1,
  90         type => 'string',
  91         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  92         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  93     },
  94     console => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Attach a console device (/dev/console) to the container.",
  98         default => 1,
  99     },
 100     tty => {
 101         optional => 1,
 102         type => 'integer',
 103         description => "Specify the number of tty available to the container",
 104         minimum => 0,
 105         maximum => 6,
 106         default => 2,
 107     },
 108     cpulimit => {
 109         optional => 1,
 110         type => 'number',
 111         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 112         minimum => 0,
 113         maximum => 128,
 114         default => 0,
 115     },
 116     cpuunits => {
 117         optional => 1,
 118         type => 'integer',
 119         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 120         minimum => 0,
 121         maximum => 500000,
 122         default => 1024,
 123     },
 124     memory => {
 125         optional => 1,
 126         type => 'integer',
 127         description => "Amount of RAM for the VM in MB.",
 128         minimum => 16,
 129         default => 512,
 130     },
 131     swap => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "Amount of SWAP for the VM in MB.",
 135         minimum => 0,
 136         default => 512,
 137     },
 138     hostname => {
 139         optional => 1,
 140         description => "Set a host name for the container.",
 141         type => 'string', format => 'dns-name',
 142         maxLength => 255,
 143     },
 144     description => {
 145         optional => 1,
 146         type => 'string',
 147         description => "Container description. Only used on the configuration web interface.",
 148     },
 149     searchdomain => {
 150         optional => 1,
 151         type => 'string', format => 'dns-name-list',
 152         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 153     },
 154     nameserver => {
 155         optional => 1,
 156         type => 'string', format => 'address-list',
 157         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 158     },
 159     rootfs => get_standard_option('pve-ct-rootfs'),
 160     parent => {
 161         optional => 1,
 162         type => 'string', format => 'pve-configid',
 163         maxLength => 40,
 164         description => "Parent snapshot name. This is used internally, and should not be modified.",
 165     },
 166     snaptime => {
 167         optional => 1,
 168         description => "Timestamp for snapshots.",
 169         type => 'integer',
 170         minimum => 0,
 171     },
 172     cmode => {
 173         optional => 1,
 174         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 175         type => 'string',
 176         enum => ['shell', 'console', 'tty'],
 177         default => 'tty',
 178     },
 179     protection => {
 180         optional => 1,
 181         type => 'boolean',
 182         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 183         default => 0,
 184     },
 185 };
 186
 187 my $valid_lxc_conf_keys = {
 188     'lxc.include' => 1,
 189     'lxc.arch' => 1,
 190     'lxc.utsname' => 1,
 191     'lxc.haltsignal' => 1,
 192     'lxc.rebootsignal' => 1,
 193     'lxc.stopsignal' => 1,
 194     'lxc.init_cmd' => 1,
 195     'lxc.network.type' => 1,
 196     'lxc.network.flags' => 1,
 197     'lxc.network.link' => 1,
 198     'lxc.network.mtu' => 1,
 199     'lxc.network.name' => 1,
 200     'lxc.network.hwaddr' => 1,
 201     'lxc.network.ipv4' => 1,
 202     'lxc.network.ipv4.gateway' => 1,
 203     'lxc.network.ipv6' => 1,
 204     'lxc.network.ipv6.gateway' => 1,
 205     'lxc.network.script.up' => 1,
 206     'lxc.network.script.down' => 1,
 207     'lxc.pts' => 1,
 208     'lxc.console.logfile' => 1,
 209     'lxc.console' => 1,
 210     'lxc.tty' => 1,
 211     'lxc.devttydir' => 1,
 212     'lxc.hook.autodev' => 1,
 213     'lxc.autodev' => 1,
 214     'lxc.kmsg' => 1,
 215     'lxc.mount' => 1,
 216     'lxc.mount.entry' => 1,
 217     'lxc.mount.auto' => 1,
 218     'lxc.rootfs' => 1,
 219     'lxc.rootfs.mount' => 1,
 220     'lxc.rootfs.options' => 1,
 221     # lxc.cgroup.*
 222     'lxc.cap.drop' => 1,
 223     'lxc.cap.keep' => 1,
 224     'lxc.aa_profile' => 1,
 225     'lxc.aa_allow_incomplete' => 1,
 226     'lxc.se_context' => 1,
 227     'lxc.seccomp' => 1,
 228     'lxc.id_map' => 1,
 229     'lxc.hook.pre-start' => 1,
 230     'lxc.hook.pre-mount' => 1,
 231     'lxc.hook.mount' => 1,
 232     'lxc.hook.start' => 1,
 233     'lxc.hook.stop' => 1,
 234     'lxc.hook.post-stop' => 1,
 235     'lxc.hook.clone' => 1,
 236     'lxc.hook.destroy' => 1,
 237     'lxc.loglevel' => 1,
 238     'lxc.logfile' => 1,
 239     'lxc.start.auto' => 1,
 240     'lxc.start.delay' => 1,
 241     'lxc.start.order' => 1,
 242     'lxc.group' => 1,
 243     'lxc.environment' => 1,
 244     'lxc.' => 1,
 245     'lxc.' => 1,
 246     'lxc.' => 1,
 247     'lxc.' => 1,
 248 };
 249
 250 my $netconf_desc = {
 251     type => {
 252         type => 'string',
 253         optional => 1,
 254         description => "Network interface type.",
 255         enum => [qw(veth)],
 256     },
 257     name => {
 258         type => 'string',
 259         format_description => 'String',
 260         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 261         pattern => '[-_.\w\d]+',
 262     },
 263     bridge => {
 264         type => 'string',
 265         format_description => 'vmbr<Number>',
 266         description => 'Bridge to attach the network device to.',
 267         pattern => '[-_.\w\d]+',
 268         optional => 1,
 269     },
 270     hwaddr => {
 271         type => 'string',
 272         format_description => 'MAC',
 273         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 274         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 275         optional => 1,
 276     },
 277     mtu => {
 278         type => 'integer',
 279         format_description => 'Number',
 280         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 281         minimum => 64, # minimum ethernet frame is 64 bytes
 282         optional => 1,
 283     },
 284     ip => {
 285         type => 'string',
 286         format => 'pve-ipv4-config',
 287         format_description => 'IPv4Format/CIDR',
 288         description => 'IPv4 address in CIDR format.',
 289         optional => 1,
 290     },
 291     gw => {
 292         type => 'string',
 293         format => 'ipv4',
 294         format_description => 'GatewayIPv4',
 295         description => 'Default gateway for IPv4 traffic.',
 296         optional => 1,
 297     },
 298     ip6 => {
 299         type => 'string',
 300         format => 'pve-ipv6-config',
 301         format_description => 'IPv6Format/CIDR',
 302         description => 'IPv6 address in CIDR format.',
 303         optional => 1,
 304     },
 305     gw6 => {
 306         type => 'string',
 307         format => 'ipv6',
 308         format_description => 'GatewayIPv6',
 309         description => 'Default gateway for IPv6 traffic.',
 310         optional => 1,
 311     },
 312     firewall => {
 313         type => 'boolean',
 314         format_description => '[1|0]',
 315         description => "Controls whether this interface's firewall rules should be used.",
 316         optional => 1,
 317     },
 318     tag => {
 319         type => 'integer',
 320         format_description => 'VlanNo',
 321         minimum => '2',
 322         maximum => '4094',
 323         description => "VLAN tag foro this interface.",
 324         optional => 1,
 325     },
 326 };
 327 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 328
 329 my $MAX_LXC_NETWORKS = 10;
 330 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 331     $confdesc->{"net$i"} = {
 332         optional => 1,
 333         type => 'string', format => $netconf_desc,
 334         description => "Specifies network interfaces for the container.",
 335     };
 336 }
 337
 338 my $mp_desc = {
 339     %$rootfs_desc,
 340     mp => {
 341         type => 'string',
 342         format_description => 'Path',
 343         description => 'Path to the mountpoint as seen from inside the container.',
 344         optional => 1,
 345     },
 346 };
 347 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 348
 349 my $unuseddesc = {
 350     optional => 1,
 351     type => 'string', format => 'pve-volume-id',
 352     description => "Reference to unused volumes.",
 353 };
 354
 355 my $MAX_MOUNT_POINTS = 10;
 356 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 357     $confdesc->{"mp$i"} = {
 358         optional => 1,
 359         type => 'string', format => $mp_desc,
 360         description => "Use volume as container mount point (experimental feature).",
 361         optional => 1,
 362     };
 363 }
 364
 365 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 366 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 367     $confdesc->{"unused$i"} = $unuseddesc;
 368 }
 369
 370 sub write_pct_config {
 371     my ($filename, $conf) = @_;
 372
 373     delete $conf->{snapstate}; # just to be sure
 374
 375     my $generate_raw_config = sub {
 376         my ($conf) = @_;
 377
 378         my $raw = '';
 379
 380         # add description as comment to top of file
 381         my $descr = $conf->{description} || '';
 382         foreach my $cl (split(/\n/, $descr)) {
 383             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 384         }
 385
 386         foreach my $key (sort keys %$conf) {
 387             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 388                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 389             my $value = $conf->{$key};
 390             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 391             $raw .= "$key: $value\n";
 392         }
 393
 394         if (my $lxcconf = $conf->{lxc}) {
 395             foreach my $entry (@$lxcconf) {
 396                 my ($k, $v) = @$entry;
 397                 $raw .= "$k: $v\n";
 398             }
 399         }
 400
 401         return $raw;
 402     };
 403
 404     my $raw = &$generate_raw_config($conf);
 405
 406     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 407         $raw .= "\n[$snapname]\n";
 408         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 409     }
 410
 411     return $raw;
 412 }
 413
 414 sub check_type {
 415     my ($key, $value) = @_;
 416
 417     die "unknown setting '$key'\n" if !$confdesc->{$key};
 418
 419     my $type = $confdesc->{$key}->{type};
 420
 421     if (!defined($value)) {
 422         die "got undefined value\n";
 423     }
 424
 425     if ($value =~ m/[\n\r]/) {
 426         die "property contains a line feed\n";
 427     }
 428
 429     if ($type eq 'boolean') {
 430         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 431         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 432         die "type check ('boolean') failed - got '$value'\n";
 433     } elsif ($type eq 'integer') {
 434         return int($1) if $value =~ m/^(\d+)$/;
 435         die "type check ('integer') failed - got '$value'\n";
 436     } elsif ($type eq 'number') {
 437         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 438         die "type check ('number') failed - got '$value'\n";
 439     } elsif ($type eq 'string') {
 440         if (my $fmt = $confdesc->{$key}->{format}) {
 441             PVE::JSONSchema::check_format($fmt, $value);
 442             return $value;
 443         }
 444         return $value;
 445     } else {
 446         die "internal error"
 447     }
 448 }
 449
 450 sub parse_pct_config {
 451     my ($filename, $raw) = @_;
 452
 453     return undef if !defined($raw);
 454
 455     my $res = {
 456         digest => Digest::SHA::sha1_hex($raw),
 457         snapshots => {},
 458     };
 459
 460     $filename =~ m|/lxc/(\d+).conf$|
 461         || die "got strange filename '$filename'";
 462
 463     my $vmid = $1;
 464
 465     my $conf = $res;
 466     my $descr = '';
 467     my $section = '';
 468
 469     my @lines = split(/\n/, $raw);
 470     foreach my $line (@lines) {
 471         next if $line =~ m/^\s*$/;
 472
 473         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 474             $section = $1;
 475             $conf->{description} = $descr if $descr;
 476             $descr = '';
 477             $conf = $res->{snapshots}->{$section} = {};
 478             next;
 479         }
 480
 481         if ($line =~ m/^\#(.*)\s*$/) {
 482             $descr .= PVE::Tools::decode_text($1) . "\n";
 483             next;
 484         }
 485
 486         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 487             my $key = $1;
 488             my $value = $3;
 489             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 490                 push @{$conf->{lxc}}, [$key, $value];
 491             } else {
 492                 warn "vm $vmid - unable to parse config: $line\n";
 493             }
 494         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 495             $descr .= PVE::Tools::decode_text($2);
 496         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 497             $conf->{snapstate} = $1;
 498         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 499             my $key = $1;
 500             my $value = $2;
 501             eval { $value = check_type($key, $value); };
 502             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 503             $conf->{$key} = $value;
 504         } else {
 505             warn "vm $vmid - unable to parse config: $line\n";
 506         }
 507     }
 508
 509     $conf->{description} = $descr if $descr;
 510
 511     delete $res->{snapstate}; # just to be sure
 512
 513     return $res;
 514 }
 515
 516 sub config_list {
 517     my $vmlist = PVE::Cluster::get_vmlist();
 518     my $res = {};
 519     return $res if !$vmlist || !$vmlist->{ids};
 520     my $ids = $vmlist->{ids};
 521
 522     foreach my $vmid (keys %$ids) {
 523         next if !$vmid; # skip CT0
 524         my $d = $ids->{$vmid};
 525         next if !$d->{node} || $d->{node} ne $nodename;
 526         next if !$d->{type} || $d->{type} ne 'lxc';
 527         $res->{$vmid}->{type} = 'lxc';
 528     }
 529     return $res;
 530 }
 531
 532 sub cfs_config_path {
 533     my ($vmid, $node) = @_;
 534
 535     $node = $nodename if !$node;
 536     return "nodes/$node/lxc/$vmid.conf";
 537 }
 538
 539 sub config_file {
 540     my ($vmid, $node) = @_;
 541
 542     my $cfspath = cfs_config_path($vmid, $node);
 543     return "/etc/pve/$cfspath";
 544 }
 545
 546 sub load_config {
 547     my ($vmid, $node) = @_;
 548
 549     $node = $nodename if !$node;
 550     my $cfspath = cfs_config_path($vmid, $node);
 551
 552     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 553     die "container $vmid does not exists\n" if !defined($conf);
 554
 555     return $conf;
 556 }
 557
 558 sub create_config {
 559     my ($vmid, $conf) = @_;
 560
 561     my $dir = "/etc/pve/nodes/$nodename/lxc";
 562     mkdir $dir;
 563
 564     write_config($vmid, $conf);
 565 }
 566
 567 sub destroy_config {
 568     my ($vmid) = @_;
 569
 570     unlink config_file($vmid, $nodename);
 571 }
 572
 573 sub write_config {
 574     my ($vmid, $conf) = @_;
 575
 576     my $cfspath = cfs_config_path($vmid);
 577
 578     PVE::Cluster::cfs_write_file($cfspath, $conf);
 579 }
 580
 581 # flock: we use one file handle per process, so lock file
 582 # can be called multiple times and succeeds for the same process.
 583
 584 my $lock_handles =  {};
 585 my $lockdir = "/run/lock/lxc";
 586
 587 sub lock_filename {
 588     my ($vmid) = @_;
 589
 590     return "$lockdir/pve-config-${vmid}.lock";
 591 }
 592
 593 sub lock_aquire {
 594     my ($vmid, $timeout) = @_;
 595
 596     $timeout = 10 if !$timeout;
 597     my $mode = LOCK_EX;
 598
 599     my $filename = lock_filename($vmid);
 600
 601     mkdir $lockdir if !-d $lockdir;
 602
 603     my $lock_func = sub {
 604         if (!$lock_handles->{$$}->{$filename}) {
 605             my $fh = new IO::File(">>$filename") ||
 606                 die "can't open file - $!\n";
 607             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 608         }
 609
 610         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 611             print STDERR "trying to aquire lock...";
 612             my $success;
 613             while(1) {
 614                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 615                 # try again on EINTR (see bug #273)
 616                 if ($success || ($! != EINTR)) {
 617                     last;
 618                 }
 619             }
 620             if (!$success) {
 621                 print STDERR " failed\n";
 622                 die "can't aquire lock - $!\n";
 623             }
 624
 625             print STDERR " OK\n";
 626         }
 627
 628         $lock_handles->{$$}->{$filename}->{refcount}++;
 629     };
 630
 631     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 632     my $err = $@;
 633     if ($err) {
 634         die "can't lock file '$filename' - $err";
 635     }
 636 }
 637
 638 sub lock_release {
 639     my ($vmid) = @_;
 640
 641     my $filename = lock_filename($vmid);
 642
 643     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 644         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 645         if ($refcount <= 0) {
 646             $lock_handles->{$$}->{$filename} = undef;
 647             close ($fh);
 648         }
 649     }
 650 }
 651
 652 sub lock_container {
 653     my ($vmid, $timeout, $code, @param) = @_;
 654
 655     my $res;
 656
 657     lock_aquire($vmid, $timeout);
 658     eval { $res = &$code(@param) };
 659     my $err = $@;
 660     lock_release($vmid);
 661
 662     die $err if $err;
 663
 664     return $res;
 665 }
 666
 667 sub option_exists {
 668     my ($name) = @_;
 669
 670     return defined($confdesc->{$name});
 671 }
 672
 673 # add JSON properties for create and set function
 674 sub json_config_properties {
 675     my $prop = shift;
 676
 677     foreach my $opt (keys %$confdesc) {
 678         next if $opt eq 'parent' || $opt eq 'snaptime';
 679         next if $prop->{$opt};
 680         $prop->{$opt} = $confdesc->{$opt};
 681     }
 682
 683     return $prop;
 684 }
 685
 686 sub json_config_properties_no_rootfs {
 687     my $prop = shift;
 688
 689     foreach my $opt (keys %$confdesc) {
 690         next if $prop->{$opt};
 691         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 692         $prop->{$opt} = $confdesc->{$opt};
 693     }
 694
 695     return $prop;
 696 }
 697
 698 # container status helpers
 699
 700 sub list_active_containers {
 701
 702     my $filename = "/proc/net/unix";
 703
 704     # similar test is used by lcxcontainers.c: list_active_containers
 705     my $res = {};
 706
 707     my $fh = IO::File->new ($filename, "r");
 708     return $res if !$fh;
 709
 710     while (defined(my $line = <$fh>)) {
 711         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 712             my $path = $1;
 713             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 714                 $res->{$1} = 1;
 715             }
 716         }
 717     }
 718
 719     close($fh);
 720
 721     return $res;
 722 }
 723
 724 # warning: this is slow
 725 sub check_running {
 726     my ($vmid) = @_;
 727
 728     my $active_hash = list_active_containers();
 729
 730     return 1 if defined($active_hash->{$vmid});
 731
 732     return undef;
 733 }
 734
 735 sub get_container_disk_usage {
 736     my ($vmid) = @_;
 737
 738     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 739
 740     my $res = {
 741         total => 0,
 742         used => 0,
 743         avail => 0,
 744     };
 745
 746     my $parser = sub {
 747         my $line = shift;
 748         if (my ($fsid, $total, $used, $avail) = $line =~
 749             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 750             $res = {
 751                 total => $total,
 752                 used => $used,
 753                 avail => $avail,
 754             };
 755         }
 756     };
 757     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 758     warn $@ if $@;
 759
 760     return $res;
 761 }
 762
 763 sub vmstatus {
 764     my ($opt_vmid) = @_;
 765
 766     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 767
 768     my $active_hash = list_active_containers();
 769
 770     foreach my $vmid (keys %$list) {
 771         my $d = $list->{$vmid};
 772
 773         my $running = defined($active_hash->{$vmid});
 774
 775         $d->{status} = $running ? 'running' : 'stopped';
 776
 777         my $cfspath = cfs_config_path($vmid);
 778         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 779
 780         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 781         $d->{name} =~ s/[\s]//g;
 782
 783         $d->{cpus} = $conf->{cpulimit} // 0;
 784
 785         if ($running) {
 786             my $res = get_container_disk_usage($vmid);
 787             $d->{disk} = $res->{used};
 788             $d->{maxdisk} = $res->{total};
 789         } else {
 790             $d->{disk} = 0;
 791             # use 4GB by default ??
 792             if (my $rootfs = $conf->{rootfs}) {
 793                 my $rootinfo = parse_ct_mountpoint($rootfs);
 794                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 795             } else {
 796                 $d->{maxdisk} = 4*1024*1024*1024;
 797             }
 798         }
 799
 800         $d->{mem} = 0;
 801         $d->{swap} = 0;
 802         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 803         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 804
 805         $d->{uptime} = 0;
 806         $d->{cpu} = 0;
 807
 808         $d->{netout} = 0;
 809         $d->{netin} = 0;
 810
 811         $d->{diskread} = 0;
 812         $d->{diskwrite} = 0;
 813
 814         $d->{template} = is_template($conf);
 815     }
 816
 817     foreach my $vmid (keys %$list) {
 818         my $d = $list->{$vmid};
 819         next if $d->{status} ne 'running';
 820
 821         my $pid = find_lxc_pid($vmid);
 822         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 823         $d->{uptime} = time - $ctime; # the method lxcfs uses
 824
 825         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 826         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 827
 828         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 829         my @bytes = split(/\n/, $blkio_bytes);
 830         foreach my $byte (@bytes) {
 831             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 832                 $d->{diskread} = $2 if $key eq 'Read';
 833                 $d->{diskwrite} = $2 if $key eq 'Write';
 834             }
 835         }
 836     }
 837
 838     return $list;
 839 }
 840
 841 sub parse_ct_mountpoint {
 842     my ($data, $noerr) = @_;
 843
 844     $data //= '';
 845
 846     my $res;
 847     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 848     if ($@) {
 849         return undef if $noerr;
 850         die $@;
 851     }
 852
 853     if (defined(my $size = $res->{size})) {
 854         $size = PVE::JSONSchema::parse_size($size);
 855         if (!defined($size)) {
 856             return undef if $noerr;
 857             die "invalid size: $size\n";
 858         }
 859         $res->{size} = $size;
 860     }
 861
 862     return $res;
 863 }
 864
 865 sub print_ct_mountpoint {
 866     my ($info, $nomp) = @_;
 867     my $skip = $nomp ? ['mp'] : [];
 868     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 869 }
 870
 871 sub print_lxc_network {
 872     my $net = shift;
 873     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 874 }
 875
 876 sub parse_lxc_network {
 877     my ($data) = @_;
 878
 879     my $res = {};
 880
 881     return $res if !$data;
 882
 883     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 884
 885     $res->{type} = 'veth';
 886     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 887
 888     return $res;
 889 }
 890
 891 sub read_cgroup_value {
 892     my ($group, $vmid, $name, $full) = @_;
 893
 894     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 895
 896     return PVE::Tools::file_get_contents($path) if $full;
 897
 898     return PVE::Tools::file_read_firstline($path);
 899 }
 900
 901 sub write_cgroup_value {
 902    my ($group, $vmid, $name, $value) = @_;
 903
 904    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 905    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 906
 907 }
 908
 909 sub find_lxc_console_pids {
 910
 911     my $res = {};
 912
 913     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 914         my ($pid) = @_;
 915
 916         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 917         return if !$cmdline;
 918
 919         my @args = split(/\0/, $cmdline);
 920
 921         # serach for lxc-console -n <vmid>
 922         return if scalar(@args) != 3;
 923         return if $args[1] ne '-n';
 924         return if $args[2] !~ m/^\d+$/;
 925         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 926
 927         my $vmid = $args[2];
 928
 929         push @{$res->{$vmid}}, $pid;
 930     });
 931
 932     return $res;
 933 }
 934
 935 sub find_lxc_pid {
 936     my ($vmid) = @_;
 937
 938     my $pid = undef;
 939     my $parser = sub {
 940         my $line = shift;
 941         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 942     };
 943     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
 944
 945     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 946
 947     return $pid;
 948 }
 949
 950 my $ipv4_reverse_mask = [
 951     '0.0.0.0',
 952     '128.0.0.0',
 953     '192.0.0.0',
 954     '224.0.0.0',
 955     '240.0.0.0',
 956     '248.0.0.0',
 957     '252.0.0.0',
 958     '254.0.0.0',
 959     '255.0.0.0',
 960     '255.128.0.0',
 961     '255.192.0.0',
 962     '255.224.0.0',
 963     '255.240.0.0',
 964     '255.248.0.0',
 965     '255.252.0.0',
 966     '255.254.0.0',
 967     '255.255.0.0',
 968     '255.255.128.0',
 969     '255.255.192.0',
 970     '255.255.224.0',
 971     '255.255.240.0',
 972     '255.255.248.0',
 973     '255.255.252.0',
 974     '255.255.254.0',
 975     '255.255.255.0',
 976     '255.255.255.128',
 977     '255.255.255.192',
 978     '255.255.255.224',
 979     '255.255.255.240',
 980     '255.255.255.248',
 981     '255.255.255.252',
 982     '255.255.255.254',
 983     '255.255.255.255',
 984 ];
 985
 986 # Note: we cannot use Net:IP, because that only allows strict
 987 # CIDR networks
 988 sub parse_ipv4_cidr {
 989     my ($cidr, $noerr) = @_;
 990
 991     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
 992         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
 993     }
 994
 995     return undef if $noerr;
 996
 997     die "unable to parse ipv4 address/mask\n";
 998 }
 999
1000 sub check_lock {
1001     my ($conf) = @_;
1002
1003     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1004 }
1005
1006 sub check_protection {
1007     my ($vm_conf, $err_msg) = @_;
1008
1009     if ($vm_conf->{protection}) {
1010         die "$err_msg - protection mode enabled\n";
1011     }
1012 }
1013
1014 sub update_lxc_config {
1015     my ($storage_cfg, $vmid, $conf) = @_;
1016
1017     my $dir = "/var/lib/lxc/$vmid";
1018
1019     if ($conf->{template}) {
1020
1021         unlink "$dir/config";
1022
1023         return;
1024     }
1025
1026     my $raw = '';
1027
1028     die "missing 'arch' - internal error" if !$conf->{arch};
1029     $raw .= "lxc.arch = $conf->{arch}\n";
1030
1031     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1032     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1033         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1034     } else {
1035         die "implement me";
1036     }
1037
1038     if (!has_dev_console($conf)) {
1039         $raw .= "lxc.console = none\n";
1040         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1041     }
1042
1043     my $ttycount = get_tty_count($conf);
1044     $raw .= "lxc.tty = $ttycount\n";
1045
1046     # some init scripts expects a linux terminal (turnkey).
1047     $raw .= "lxc.environment = TERM=linux\n";
1048
1049     my $utsname = $conf->{hostname} || "CT$vmid";
1050     $raw .= "lxc.utsname = $utsname\n";
1051
1052     my $memory = $conf->{memory} || 512;
1053     my $swap = $conf->{swap} // 0;
1054
1055     my $lxcmem = int($memory*1024*1024);
1056     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1057
1058     my $lxcswap = int(($memory + $swap)*1024*1024);
1059     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1060
1061     if (my $cpulimit = $conf->{cpulimit}) {
1062         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1063         my $value = int(100000*$cpulimit);
1064         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1065     }
1066
1067     my $shares = $conf->{cpuunits} || 1024;
1068     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1069
1070     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1071     $mountpoint->{mp} = '/';
1072
1073     my ($path, $use_loopdev) = mountpoint_mount_path($mountpoint, $storage_cfg);
1074     $path = "loop:$path" if $use_loopdev;
1075
1076     $raw .= "lxc.rootfs = $path\n";
1077
1078     my $netcount = 0;
1079     foreach my $k (keys %$conf) {
1080         next if $k !~ m/^net(\d+)$/;
1081         my $ind = $1;
1082         my $d = parse_lxc_network($conf->{$k});
1083         $netcount++;
1084         $raw .= "lxc.network.type = veth\n";
1085         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1086         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1087         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1088         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1089     }
1090
1091     if (my $lxcconf = $conf->{lxc}) {
1092         foreach my $entry (@$lxcconf) {
1093             my ($k, $v) = @$entry;
1094             $netcount++ if $k eq 'lxc.network.type';
1095             $raw .= "$k = $v\n";
1096         }
1097     }
1098
1099     $raw .= "lxc.network.type = empty\n" if !$netcount;
1100
1101     File::Path::mkpath("$dir/rootfs");
1102
1103     PVE::Tools::file_set_contents("$dir/config", $raw);
1104 }
1105
1106 # verify and cleanup nameserver list (replace \0 with ' ')
1107 sub verify_nameserver_list {
1108     my ($nameserver_list) = @_;
1109
1110     my @list = ();
1111     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1112         PVE::JSONSchema::pve_verify_ip($server);
1113         push @list, $server;
1114     }
1115
1116     return join(' ', @list);
1117 }
1118
1119 sub verify_searchdomain_list {
1120     my ($searchdomain_list) = @_;
1121
1122     my @list = ();
1123     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1124         # todo: should we add checks for valid dns domains?
1125         push @list, $server;
1126     }
1127
1128     return join(' ', @list);
1129 }
1130
1131 sub add_unused_volume {
1132     my ($config, $volid) = @_;
1133
1134     # skip bind mounts and block devices
1135     return if $volid =~ m|^/|;
1136
1137     my $key;
1138     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1139         my $test = "unused$ind";
1140         if (my $vid = $config->{$test}) {
1141             return if $vid eq $volid; # do not add duplicates
1142         } else {
1143             $key = $test;
1144         }
1145     }
1146
1147     die "To many unused volume - please delete them first.\n" if !$key;
1148
1149     $config->{$key} = $volid;
1150
1151     return $key;
1152 }
1153
1154 sub update_pct_config {
1155     my ($vmid, $conf, $running, $param, $delete) = @_;
1156
1157     my @nohotplug;
1158
1159     my $new_disks = 0;
1160     my @deleted_volumes;
1161
1162     my $rootdir;
1163     if ($running) {
1164         my $pid = find_lxc_pid($vmid);
1165         $rootdir = "/proc/$pid/root";
1166     }
1167
1168     my $hotplug_error = sub {
1169         if ($running) {
1170             push @nohotplug, @_;
1171             return 1;
1172         } else {
1173             return 0;
1174         }
1175     };
1176
1177     if (defined($delete)) {
1178         foreach my $opt (@$delete) {
1179             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1180                 die "unable to delete required option '$opt'\n";
1181             } elsif ($opt eq 'swap') {
1182                 delete $conf->{$opt};
1183                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1184             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1185                 delete $conf->{$opt};
1186             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1187                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1188                 next if $hotplug_error->($opt);
1189                 delete $conf->{$opt};
1190             } elsif ($opt =~ m/^net(\d)$/) {
1191                 delete $conf->{$opt};
1192                 next if !$running;
1193                 my $netid = $1;
1194                 PVE::Network::veth_delete("veth${vmid}i$netid");
1195             } elsif ($opt eq 'protection') {
1196                 delete $conf->{$opt};
1197             } elsif ($opt =~ m/^unused(\d+)$/) {
1198                 next if $hotplug_error->($opt);
1199                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1200                 push @deleted_volumes, $conf->{$opt};
1201                 delete $conf->{$opt};
1202             } elsif ($opt =~ m/^mp(\d+)$/) {
1203                 next if $hotplug_error->($opt);
1204                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1205                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1206                 add_unused_volume($conf, $mountpoint->{volume});
1207                 delete $conf->{$opt};
1208             } else {
1209                 die "implement me"
1210             }
1211             write_config($vmid, $conf) if $running;
1212         }
1213     }
1214
1215     # There's no separate swap size to configure, there's memory and "total"
1216     # memory (iow. memory+swap). This means we have to change them together.
1217     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1218     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1219     if (defined($wanted_memory) || defined($wanted_swap)) {
1220
1221         $wanted_memory //= ($conf->{memory} || 512);
1222         $wanted_swap //=  ($conf->{swap} || 0);
1223
1224         my $total = $wanted_memory + $wanted_swap;
1225         if ($running) {
1226             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1227             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1228         }
1229         $conf->{memory} = $wanted_memory;
1230         $conf->{swap} = $wanted_swap;
1231
1232         write_config($vmid, $conf) if $running;
1233     }
1234
1235     foreach my $opt (keys %$param) {
1236         my $value = $param->{$opt};
1237         if ($opt eq 'hostname') {
1238             $conf->{$opt} = $value;
1239         } elsif ($opt eq 'onboot') {
1240             $conf->{$opt} = $value ? 1 : 0;
1241         } elsif ($opt eq 'startup') {
1242             $conf->{$opt} = $value;
1243         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1244             next if $hotplug_error->($opt);
1245             $conf->{$opt} = $value;
1246         } elsif ($opt eq 'nameserver') {
1247             next if $hotplug_error->($opt);
1248             my $list = verify_nameserver_list($value);
1249             $conf->{$opt} = $list;
1250         } elsif ($opt eq 'searchdomain') {
1251             next if $hotplug_error->($opt);
1252             my $list = verify_searchdomain_list($value);
1253             $conf->{$opt} = $list;
1254         } elsif ($opt eq 'cpulimit') {
1255             next if $hotplug_error->($opt); # FIXME: hotplug
1256             $conf->{$opt} = $value;
1257         } elsif ($opt eq 'cpuunits') {
1258             $conf->{$opt} = $value;
1259             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1260         } elsif ($opt eq 'description') {
1261             $conf->{$opt} = PVE::Tools::encode_text($value);
1262         } elsif ($opt =~ m/^net(\d+)$/) {
1263             my $netid = $1;
1264             my $net = parse_lxc_network($value);
1265             if (!$running) {
1266                 $conf->{$opt} = print_lxc_network($net);
1267             } else {
1268                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1269             }
1270         } elsif ($opt eq 'protection') {
1271             $conf->{$opt} = $value ? 1 : 0;
1272         } elsif ($opt =~ m/^mp(\d+)$/) {
1273             next if $hotplug_error->($opt);
1274             check_protection($conf, "can't update CT $vmid drive '$opt'");
1275             $conf->{$opt} = $value;
1276             $new_disks = 1;
1277         } elsif ($opt eq 'rootfs') {
1278             check_protection($conf, "can't update CT $vmid drive '$opt'");
1279             die "implement me: $opt";
1280         } else {
1281             die "implement me: $opt";
1282         }
1283         write_config($vmid, $conf) if $running;
1284     }
1285
1286     if (@deleted_volumes) {
1287         my $storage_cfg = PVE::Storage::config();
1288         foreach my $volume (@deleted_volumes) {
1289             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1290         }
1291     }
1292
1293     if ($new_disks) {
1294         my $storage_cfg = PVE::Storage::config();
1295         create_disks($storage_cfg, $vmid, $conf, $conf);
1296     }
1297
1298     # This should be the last thing we do here
1299     if ($running && scalar(@nohotplug)) {
1300         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1301     }
1302 }
1303
1304 sub has_dev_console {
1305     my ($conf) = @_;
1306
1307     return !(defined($conf->{console}) && !$conf->{console});
1308 }
1309
1310 sub get_tty_count {
1311     my ($conf) = @_;
1312
1313     return $conf->{tty} // $confdesc->{tty}->{default};
1314 }
1315
1316 sub get_cmode {
1317     my ($conf) = @_;
1318
1319     return $conf->{cmode} // $confdesc->{cmode}->{default};
1320 }
1321
1322 sub get_console_command {
1323     my ($vmid, $conf) = @_;
1324
1325     my $cmode = get_cmode($conf);
1326
1327     if ($cmode eq 'console') {
1328         return ['lxc-console', '-n',  $vmid, '-t', 0];
1329     } elsif ($cmode eq 'tty') {
1330         return ['lxc-console', '-n',  $vmid];
1331     } elsif ($cmode eq 'shell') {
1332         return ['lxc-attach', '--clear-env', '-n', $vmid];
1333     } else {
1334         die "internal error";
1335     }
1336 }
1337
1338 sub get_primary_ips {
1339     my ($conf) = @_;
1340
1341     # return data from net0
1342
1343     return undef if !defined($conf->{net0});
1344     my $net = parse_lxc_network($conf->{net0});
1345
1346     my $ipv4 = $net->{ip};
1347     if ($ipv4) {
1348         if ($ipv4 =~ /^(dhcp|manual)$/) {
1349             $ipv4 = undef
1350         } else {
1351             $ipv4 =~ s!/\d+$!!;
1352         }
1353     }
1354     my $ipv6 = $net->{ip6};
1355     if ($ipv6) {
1356         if ($ipv6 =~ /^(dhcp|manual)$/) {
1357             $ipv6 = undef;
1358         } else {
1359             $ipv6 =~ s!/\d+$!!;
1360         }
1361     }
1362
1363     return ($ipv4, $ipv6);
1364 }
1365
1366 sub delete_mountpoint_volume {
1367     my ($storage_cfg, $vmid, $volume) = @_;
1368
1369     # skip bind mounts and block devices
1370     if ($volume =~ m|^/|) {
1371             return;
1372     }
1373
1374     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1375     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1376 }
1377
1378 sub destroy_lxc_container {
1379     my ($storage_cfg, $vmid, $conf) = @_;
1380
1381     foreach_mountpoint($conf, sub {
1382         my ($ms, $mountpoint) = @_;
1383         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1384     });
1385
1386     rmdir "/var/lib/lxc/$vmid/rootfs";
1387     unlink "/var/lib/lxc/$vmid/config";
1388     rmdir "/var/lib/lxc/$vmid";
1389     destroy_config($vmid);
1390
1391     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1392     #PVE::Tools::run_command($cmd);
1393 }
1394
1395 sub vm_stop_cleanup {
1396     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1397
1398     eval {
1399         if (!$keepActive) {
1400
1401             my $vollist = get_vm_volumes($conf);
1402             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1403         }
1404     };
1405     warn $@ if $@; # avoid errors - just warn
1406 }
1407
1408 my $safe_num_ne = sub {
1409     my ($a, $b) = @_;
1410
1411     return 0 if !defined($a) && !defined($b);
1412     return 1 if !defined($a);
1413     return 1 if !defined($b);
1414
1415     return $a != $b;
1416 };
1417
1418 my $safe_string_ne = sub {
1419     my ($a, $b) = @_;
1420
1421     return 0 if !defined($a) && !defined($b);
1422     return 1 if !defined($a);
1423     return 1 if !defined($b);
1424
1425     return $a ne $b;
1426 };
1427
1428 sub update_net {
1429     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1430
1431     if ($newnet->{type} ne 'veth') {
1432         # for when there are physical interfaces
1433         die "cannot update interface of type $newnet->{type}";
1434     }
1435
1436     my $veth = "veth${vmid}i${netid}";
1437     my $eth = $newnet->{name};
1438
1439     if (my $oldnetcfg = $conf->{$opt}) {
1440         my $oldnet = parse_lxc_network($oldnetcfg);
1441
1442         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1443             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1444
1445             PVE::Network::veth_delete($veth);
1446             delete $conf->{$opt};
1447             write_config($vmid, $conf);
1448
1449             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1450
1451         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1452                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1453                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1454
1455                 if ($oldnet->{bridge}) {
1456                     PVE::Network::tap_unplug($veth);
1457                     foreach (qw(bridge tag firewall)) {
1458                         delete $oldnet->{$_};
1459                     }
1460                     $conf->{$opt} = print_lxc_network($oldnet);
1461                     write_config($vmid, $conf);
1462                 }
1463
1464                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1465                 foreach (qw(bridge tag firewall)) {
1466                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1467                 }
1468                 $conf->{$opt} = print_lxc_network($oldnet);
1469                 write_config($vmid, $conf);
1470         }
1471     } else {
1472         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1473     }
1474
1475     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1476 }
1477
1478 sub hotplug_net {
1479     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1480
1481     my $veth = "veth${vmid}i${netid}";
1482     my $vethpeer = $veth . "p";
1483     my $eth = $newnet->{name};
1484
1485     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1486     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1487
1488     # attach peer in container
1489     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1490     PVE::Tools::run_command($cmd);
1491
1492     # link up peer in container
1493     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1494     PVE::Tools::run_command($cmd);
1495
1496     my $done = { type => 'veth' };
1497     foreach (qw(bridge tag firewall hwaddr name)) {
1498         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1499     }
1500     $conf->{$opt} = print_lxc_network($done);
1501
1502     write_config($vmid, $conf);
1503 }
1504
1505 sub update_ipconfig {
1506     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1507
1508     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1509
1510     my $optdata = parse_lxc_network($conf->{$opt});
1511     my $deleted = [];
1512     my $added = [];
1513     my $nscmd = sub {
1514         my $cmdargs = shift;
1515         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1516     };
1517     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1518
1519     my $change_ip_config = sub {
1520         my ($ipversion) = @_;
1521
1522         my $family_opt = "-$ipversion";
1523         my $suffix = $ipversion == 4 ? '' : $ipversion;
1524         my $gw= "gw$suffix";
1525         my $ip= "ip$suffix";
1526
1527         my $newip = $newnet->{$ip};
1528         my $newgw = $newnet->{$gw};
1529         my $oldip = $optdata->{$ip};
1530
1531         my $change_ip = &$safe_string_ne($oldip, $newip);
1532         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1533
1534         return if !$change_ip && !$change_gw;
1535
1536         # step 1: add new IP, if this fails we cancel
1537         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1538             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1539             if (my $err = $@) {
1540                 warn $err;
1541                 return;
1542             }
1543         }
1544
1545         # step 2: replace gateway
1546         #   If this fails we delete the added IP and cancel.
1547         #   If it succeeds we save the config and delete the old IP, ignoring
1548         #   errors. The config is then saved.
1549         # Note: 'ip route replace' can add
1550         if ($change_gw) {
1551             if ($newgw) {
1552                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1553                 if (my $err = $@) {
1554                     warn $err;
1555                     # the route was not replaced, the old IP is still available
1556                     # rollback (delete new IP) and cancel
1557                     if ($change_ip) {
1558                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1559                         warn $@ if $@; # no need to die here
1560                     }
1561                     return;
1562                 }
1563             } else {
1564                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1565                 # if the route was not deleted, the guest might have deleted it manually
1566                 # warn and continue
1567                 warn $@ if $@;
1568             }
1569         }
1570
1571         # from this point on we save the configuration
1572         # step 3: delete old IP ignoring errors
1573         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1574             # We need to enable promote_secondaries, otherwise our newly added
1575             # address will be removed along with the old one.
1576             my $promote = 0;
1577             eval {
1578                 if ($ipversion == 4) {
1579                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1580                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1581                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1582                 }
1583                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1584             };
1585             warn $@ if $@; # no need to die here
1586
1587             if ($ipversion == 4) {
1588                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1589             }
1590         }
1591
1592         foreach my $property ($ip, $gw) {
1593             if ($newnet->{$property}) {
1594                 $optdata->{$property} = $newnet->{$property};
1595             } else {
1596                 delete $optdata->{$property};
1597             }
1598         }
1599         $conf->{$opt} = print_lxc_network($optdata);
1600         write_config($vmid, $conf);
1601         $lxc_setup->setup_network($conf);
1602     };
1603
1604     &$change_ip_config(4);
1605     &$change_ip_config(6);
1606
1607 }
1608
1609 # Internal snapshots
1610
1611 # NOTE: Snapshot create/delete involves several non-atomic
1612 # action, and can take a long time.
1613 # So we try to avoid locking the file and use 'lock' variable
1614 # inside the config file instead.
1615
1616 my $snapshot_copy_config = sub {
1617     my ($source, $dest) = @_;
1618
1619     foreach my $k (keys %$source) {
1620         next if $k eq 'snapshots';
1621         next if $k eq 'snapstate';
1622         next if $k eq 'snaptime';
1623         next if $k eq 'vmstate';
1624         next if $k eq 'lock';
1625         next if $k eq 'digest';
1626         next if $k eq 'description';
1627
1628         $dest->{$k} = $source->{$k};
1629     }
1630 };
1631
1632 my $snapshot_prepare = sub {
1633     my ($vmid, $snapname, $comment) = @_;
1634
1635     my $snap;
1636
1637     my $updatefn =  sub {
1638
1639         my $conf = load_config($vmid);
1640
1641         die "you can't take a snapshot if it's a template\n"
1642             if is_template($conf);
1643
1644         check_lock($conf);
1645
1646         $conf->{lock} = 'snapshot';
1647
1648         die "snapshot name '$snapname' already used\n"
1649             if defined($conf->{snapshots}->{$snapname});
1650
1651         my $storecfg = PVE::Storage::config();
1652         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1653
1654         $snap = $conf->{snapshots}->{$snapname} = {};
1655
1656         &$snapshot_copy_config($conf, $snap);
1657
1658         $snap->{'snapstate'} = "prepare";
1659         $snap->{'snaptime'} = time();
1660         $snap->{'description'} = $comment if $comment;
1661         $conf->{snapshots}->{$snapname} = $snap;
1662
1663         write_config($vmid, $conf);
1664     };
1665
1666     lock_container($vmid, 10, $updatefn);
1667
1668     return $snap;
1669 };
1670
1671 my $snapshot_commit = sub {
1672     my ($vmid, $snapname) = @_;
1673
1674     my $updatefn = sub {
1675
1676         my $conf = load_config($vmid);
1677
1678         die "missing snapshot lock\n"
1679             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1680
1681         die "snapshot '$snapname' does not exist\n"
1682             if !defined($conf->{snapshots}->{$snapname});
1683
1684         die "wrong snapshot state\n"
1685             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1686                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1687
1688         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1689         delete $conf->{lock};
1690         $conf->{parent} = $snapname;
1691
1692         write_config($vmid, $conf);
1693     };
1694
1695     lock_container($vmid, 10 ,$updatefn);
1696 };
1697
1698 sub has_feature {
1699     my ($feature, $conf, $storecfg, $snapname) = @_;
1700
1701     my $err;
1702
1703     foreach_mountpoint($conf, sub {
1704         my ($ms, $mountpoint) = @_;
1705
1706         return if $err; # skip further test
1707
1708         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1709
1710         # TODO: implement support for mountpoints
1711         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1712             if $ms ne 'rootfs';
1713     });
1714
1715     return $err ? 0 : 1;
1716 }
1717
1718 sub snapshot_create {
1719     my ($vmid, $snapname, $comment) = @_;
1720
1721     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1722
1723     my $conf = load_config($vmid);
1724
1725     my $running = check_running($vmid);
1726     eval {
1727         if ($running) {
1728             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1729             PVE::Tools::run_command(['/bin/sync']);
1730         };
1731
1732         my $storecfg = PVE::Storage::config();
1733         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1734         my $volid = $rootinfo->{volume};
1735
1736         if ($running) {
1737             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1738         };
1739
1740         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1741         &$snapshot_commit($vmid, $snapname);
1742     };
1743     if(my $err = $@) {
1744         snapshot_delete($vmid, $snapname, 1);
1745         die "$err\n";
1746     }
1747 }
1748
1749 sub snapshot_delete {
1750     my ($vmid, $snapname, $force) = @_;
1751
1752     my $snap;
1753
1754     my $conf;
1755
1756     my $updatefn =  sub {
1757
1758         $conf = load_config($vmid);
1759
1760         die "you can't delete a snapshot if vm is a template\n"
1761             if is_template($conf);
1762
1763         $snap = $conf->{snapshots}->{$snapname};
1764
1765         check_lock($conf);
1766
1767         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1768
1769         $snap->{snapstate} = 'delete';
1770
1771         write_config($vmid, $conf);
1772     };
1773
1774     lock_container($vmid, 10, $updatefn);
1775
1776     my $storecfg = PVE::Storage::config();
1777
1778     my $del_snap =  sub {
1779
1780         check_lock($conf);
1781
1782         if ($conf->{parent} eq $snapname) {
1783             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1784                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1785             } else {
1786                 delete $conf->{parent};
1787             }
1788         }
1789
1790         delete $conf->{snapshots}->{$snapname};
1791
1792         write_config($vmid, $conf);
1793     };
1794
1795     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1796     my $rootinfo = parse_ct_mountpoint($rootfs);
1797     my $volid = $rootinfo->{volume};
1798
1799     eval {
1800         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1801     };
1802     my $err = $@;
1803
1804     if(!$err || ($err && $force)) {
1805         lock_container($vmid, 10, $del_snap);
1806         if ($err) {
1807             die "Can't delete snapshot: $vmid $snapname $err\n";
1808         }
1809     }
1810 }
1811
1812 sub snapshot_rollback {
1813     my ($vmid, $snapname) = @_;
1814
1815     my $storecfg = PVE::Storage::config();
1816
1817     my $conf = load_config($vmid);
1818
1819     die "you can't rollback if vm is a template\n" if is_template($conf);
1820
1821     my $snap = $conf->{snapshots}->{$snapname};
1822
1823     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1824
1825     my $rootfs = $snap->{rootfs};
1826     my $rootinfo = parse_ct_mountpoint($rootfs);
1827     my $volid = $rootinfo->{volume};
1828
1829     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1830
1831     my $updatefn = sub {
1832
1833         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1834             if $snap->{snapstate};
1835
1836         check_lock($conf);
1837
1838         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1839
1840         die "unable to rollback vm $vmid: vm is running\n"
1841             if check_running($vmid);
1842
1843         $conf->{lock} = 'rollback';
1844
1845         my $forcemachine;
1846
1847         # copy snapshot config to current config
1848
1849         my $tmp_conf = $conf;
1850         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1851         $conf->{snapshots} = $tmp_conf->{snapshots};
1852         delete $conf->{snaptime};
1853         delete $conf->{snapname};
1854         $conf->{parent} = $snapname;
1855
1856         write_config($vmid, $conf);
1857     };
1858
1859     my $unlockfn = sub {
1860         delete $conf->{lock};
1861         write_config($vmid, $conf);
1862     };
1863
1864     lock_container($vmid, 10, $updatefn);
1865
1866     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1867
1868     lock_container($vmid, 5, $unlockfn);
1869 }
1870
1871 sub template_create {
1872     my ($vmid, $conf) = @_;
1873
1874     my $storecfg = PVE::Storage::config();
1875
1876     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1877     my $volid = $rootinfo->{volume};
1878
1879     die "Template feature is not available for '$volid'\n"
1880         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1881
1882     PVE::Storage::activate_volumes($storecfg, [$volid]);
1883
1884     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1885     $rootinfo->{volume} = $template_volid;
1886     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1887
1888     write_config($vmid, $conf);
1889 }
1890
1891 sub is_template {
1892     my ($conf) = @_;
1893
1894     return 1 if defined $conf->{template} && $conf->{template} == 1;
1895 }
1896
1897 sub mountpoint_names {
1898     my ($reverse) = @_;
1899
1900     my @names = ('rootfs');
1901
1902     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1903         push @names, "mp$i";
1904     }
1905
1906     return $reverse ? reverse @names : @names;
1907 }
1908
1909 # The container might have *different* symlinks than the host. realpath/abs_path
1910 # use the actual filesystem to resolve links.
1911 sub sanitize_mountpoint {
1912     my ($mp) = @_;
1913     $mp = '/' . $mp; # we always start with a slash
1914     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1915     $mp =~ s@/\./@@g; # collapse /./
1916     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1917     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1918     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1919     return $mp;
1920 }
1921
1922 sub foreach_mountpoint_full {
1923     my ($conf, $reverse, $func) = @_;
1924
1925     foreach my $key (mountpoint_names($reverse)) {
1926         my $value = $conf->{$key};
1927         next if !defined($value);
1928         my $mountpoint = parse_ct_mountpoint($value, 1);
1929         next if !defined($mountpoint);
1930
1931         # just to be sure: rootfs is /
1932         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1933         $mountpoint->{mp} = sanitize_mountpoint($path);
1934
1935         $path = $mountpoint->{volume};
1936         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1937
1938         &$func($key, $mountpoint);
1939     }
1940 }
1941
1942 sub foreach_mountpoint {
1943     my ($conf, $func) = @_;
1944
1945     foreach_mountpoint_full($conf, 0, $func);
1946 }
1947
1948 sub foreach_mountpoint_reverse {
1949     my ($conf, $func) = @_;
1950
1951     foreach_mountpoint_full($conf, 1, $func);
1952 }
1953
1954 sub check_ct_modify_config_perm {
1955     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1956
1957     return 1 if $authuser ne 'root@pam';
1958
1959     foreach my $opt (@$key_list) {
1960
1961         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1962             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1963         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1964             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1965         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1966             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1967         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1968                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1969             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1970         } else {
1971             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1972         }
1973     }
1974
1975     return 1;
1976 }
1977
1978 sub umount_all {
1979     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
1980
1981     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1982     my $volid_list = get_vm_volumes($conf);
1983
1984     foreach_mountpoint_reverse($conf, sub {
1985         my ($ms, $mountpoint) = @_;
1986
1987         my $volid = $mountpoint->{volume};
1988         my $mount = $mountpoint->{mp};
1989
1990         return if !$volid || !$mount;
1991
1992         my $mount_path = "$rootdir/$mount";
1993         $mount_path =~ s!/+!/!g;
1994
1995         return if !PVE::ProcFSTools::is_mounted($mount_path);
1996
1997         eval {
1998             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1999         };
2000         if (my $err = $@) {
2001             if ($noerr) {
2002                 warn $err;
2003             } else {
2004                 die $err;
2005             }
2006         }
2007     });
2008 }
2009
2010 sub mount_all {
2011     my ($vmid, $storage_cfg, $conf) = @_;
2012
2013     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2014     File::Path::make_path($rootdir);
2015
2016     my $volid_list = get_vm_volumes($conf);
2017     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2018
2019     eval {
2020         foreach_mountpoint($conf, sub {
2021             my ($ms, $mountpoint) = @_;
2022
2023             my $volid = $mountpoint->{volume};
2024             my $mount = $mountpoint->{mp};
2025
2026             return if !$volid || !$mount;
2027
2028             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2029             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2030                 PVE::Storage::parse_volname($storage_cfg, $volid);
2031
2032             die "unable to mount base volume - internal error" if $isBase;
2033
2034             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2035         });
2036     };
2037     if (my $err = $@) {
2038         warn "mounting container failed - $err";
2039         umount_all($vmid, $storage_cfg, $conf, 1);
2040     }
2041
2042     return $rootdir;
2043 }
2044
2045
2046 sub mountpoint_mount_path {
2047     my ($mountpoint, $storage_cfg, $snapname) = @_;
2048
2049     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2050 }
2051
2052 my $check_mount_path = sub {
2053     my ($path) = @_;
2054     $path = File::Spec->canonpath($path);
2055     my $real = Cwd::realpath($path);
2056     if ($real ne $path) {
2057         die "mount path modified by symlink: $path != $real";
2058     }
2059 };
2060
2061 # use $rootdir = undef to just return the corresponding mount path
2062 sub mountpoint_mount {
2063     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2064
2065     my $volid = $mountpoint->{volume};
2066     my $mount = $mountpoint->{mp};
2067
2068     return if !$volid || !$mount;
2069
2070     my $mount_path;
2071
2072     if (defined($rootdir)) {
2073         $rootdir =~ s!/+$!!;
2074         $mount_path = "$rootdir/$mount";
2075         $mount_path =~ s!/+!/!g;
2076         &$check_mount_path($mount_path);
2077         File::Path::mkpath($mount_path);
2078     }
2079
2080     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2081
2082     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2083
2084     if ($storage) {
2085
2086         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2087         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2088
2089         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2090             PVE::Storage::parse_volname($storage_cfg, $volid);
2091
2092         if ($format eq 'subvol') {
2093             if ($mount_path) {
2094                 if ($snapname) {
2095                     if ($scfg->{type} eq 'zfspool') {
2096                         my $path_arg = $path;
2097                         $path_arg =~ s!^/+!!;
2098                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2099                     } else {
2100                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2101                     }
2102                 } else {
2103                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2104                 }
2105             }
2106             return wantarray ? ($path, 0) : $path;
2107         } elsif ($format eq 'raw') {
2108             my $use_loopdev = 0;
2109             my @extra_opts;
2110             if ($scfg->{path}) {
2111                 push @extra_opts, '-o', 'loop';
2112                 $use_loopdev = 1;
2113             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2114                 # do nothing
2115             } else {
2116                 die "unsupported storage type '$scfg->{type}'\n";
2117             }
2118             if ($mount_path) {
2119                 if ($isBase || defined($snapname)) {
2120                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2121                 } else {
2122                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2123                 }
2124             }
2125             return wantarray ? ($path, $use_loopdev) : $path;
2126         } else {
2127             die "unsupported image format '$format'\n";
2128         }
2129     } elsif ($volid =~ m|^/dev/.+|) {
2130         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2131         return wantarray ? ($volid, 0) : $volid;
2132     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2133         &$check_mount_path($volid);
2134         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2135         return wantarray ? ($volid, 0) : $volid;
2136     }
2137
2138     die "unsupported storage";
2139 }
2140
2141 sub get_vm_volumes {
2142     my ($conf, $excludes) = @_;
2143
2144     my $vollist = [];
2145
2146     foreach_mountpoint($conf, sub {
2147         my ($ms, $mountpoint) = @_;
2148
2149         return if $excludes && $ms eq $excludes;
2150
2151         my $volid = $mountpoint->{volume};
2152
2153         return if !$volid || $volid =~ m|^/|;
2154
2155         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2156         return if !$sid;
2157
2158         push @$vollist, $volid;
2159     });
2160
2161     return $vollist;
2162 }
2163
2164 sub mkfs {
2165     my ($dev) = @_;
2166
2167     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2168 }
2169
2170 sub format_disk {
2171     my ($storage_cfg, $volid) = @_;
2172
2173     if ($volid =~ m!^/dev/.+!) {
2174         mkfs($volid);
2175         return;
2176     }
2177
2178     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2179
2180     die "cannot format volume '$volid' with no storage\n" if !$storage;
2181
2182     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2183
2184     my $path = PVE::Storage::path($storage_cfg, $volid);
2185
2186     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2187         PVE::Storage::parse_volname($storage_cfg, $volid);
2188
2189     die "cannot format volume '$volid' (format == $format)\n"
2190         if $format ne 'raw';
2191
2192     mkfs($path);
2193 }
2194
2195 sub destroy_disks {
2196     my ($storecfg, $vollist) = @_;
2197
2198     foreach my $volid (@$vollist) {
2199         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2200         warn $@ if $@;
2201     }
2202 }
2203
2204 sub create_disks {
2205     my ($storecfg, $vmid, $settings, $conf) = @_;
2206
2207     my $vollist = [];
2208
2209     eval {
2210         foreach_mountpoint($settings, sub {
2211             my ($ms, $mountpoint) = @_;
2212
2213             my $volid = $mountpoint->{volume};
2214             my $mp = $mountpoint->{mp};
2215
2216             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2217
2218             return if !$storage;
2219
2220             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2221                 my ($storeid, $size_gb) = ($1, $2);
2222
2223                 my $size_kb = int(${size_gb}*1024) * 1024;
2224
2225                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2226                 # fixme: use better naming ct-$vmid-disk-X.raw?
2227
2228                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2229                     if ($size_kb > 0) {
2230                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2231                                                            undef, $size_kb);
2232                         format_disk($storecfg, $volid);
2233                     } else {
2234                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2235                                                            undef, 0);
2236                     }
2237                 } elsif ($scfg->{type} eq 'zfspool') {
2238
2239                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2240                                                        undef, $size_kb);
2241                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2242
2243                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2244                     format_disk($storecfg, $volid);
2245
2246                 } elsif ($scfg->{type} eq 'rbd') {
2247
2248                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2249                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2250                     format_disk($storecfg, $volid);
2251                 } else {
2252                     die "unable to create containers on storage type '$scfg->{type}'\n";
2253                 }
2254                 push @$vollist, $volid;
2255                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2256                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2257             } else {
2258                 # use specified/existing volid
2259             }
2260         });
2261     };
2262     # free allocated images on error
2263     if (my $err = $@) {
2264         destroy_disks($storecfg, $vollist);
2265         die $err;
2266     }
2267     return $vollist;
2268 }
2269
2270 # bash completion helper
2271
2272 sub complete_os_templates {
2273     my ($cmdname, $pname, $cvalue) = @_;
2274
2275     my $cfg = PVE::Storage::config();
2276
2277     my $storeid;
2278
2279     if ($cvalue =~ m/^([^:]+):/) {
2280         $storeid = $1;
2281     }
2282
2283     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2284     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2285
2286     my $res = [];
2287     foreach my $id (keys %$data) {
2288         foreach my $item (@{$data->{$id}}) {
2289             push @$res, $item->{volid} if defined($item->{volid});
2290         }
2291     }
2292
2293     return $res;
2294 }
2295
2296 my $complete_ctid_full = sub {
2297     my ($running) = @_;
2298
2299     my $idlist = vmstatus();
2300
2301     my $active_hash = list_active_containers();
2302
2303     my $res = [];
2304
2305     foreach my $id (keys %$idlist) {
2306         my $d = $idlist->{$id};
2307         if (defined($running)) {
2308             next if $d->{template};
2309             next if $running && !$active_hash->{$id};
2310             next if !$running && $active_hash->{$id};
2311         }
2312         push @$res, $id;
2313
2314     }
2315     return $res;
2316 };
2317
2318 sub complete_ctid {
2319     return &$complete_ctid_full();
2320 }
2321
2322 sub complete_ctid_stopped {
2323     return &$complete_ctid_full(0);
2324 }
2325
2326 sub complete_ctid_running {
2327     return &$complete_ctid_full(1);
2328 }
2329
2330 1;