src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21
  22 use Data::Dumper;
  23
  24 my $nodename = PVE::INotify::nodename();
  25
  26 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  27
  28 my $rootfs_desc = {
  29     volume => {
  30         type => 'string',
  31         default_key => 1,
  32         format_description => 'volume',
  33         description => 'Volume, device or directory to mount into the container.',
  34     },
  35     backup => {
  36         type => 'boolean',
  37         format_description => '[1|0]',
  38         description => 'Whether to include the mountpoint in backups.',
  39         optional => 1,
  40     },
  41     size => {
  42         type => 'disk-size',
  43         format_description => 'DiskSize',
  44         description => 'Volume size (read only value).',
  45         optional => 1,
  46     },
  47 };
  48
  49 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  50     type => 'string', format => $rootfs_desc,
  51     description => "Use volume as container root.",
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  56     description => "The name of the snapshot.",
  57     type => 'string', format => 'pve-configid',
  58     maxLength => 40,
  59 });
  60
  61 my $confdesc = {
  62     lock => {
  63         optional => 1,
  64         type => 'string',
  65         description => "Lock/unlock the VM.",
  66         enum => [qw(migrate backup snapshot rollback)],
  67     },
  68     onboot => {
  69         optional => 1,
  70         type => 'boolean',
  71         description => "Specifies whether a VM will be started during system bootup.",
  72         default => 0,
  73     },
  74     startup => get_standard_option('pve-startup-order'),
  75     template => {
  76         optional => 1,
  77         type => 'boolean',
  78         description => "Enable/disable Template.",
  79         default => 0,
  80     },
  81     arch => {
  82         optional => 1,
  83         type => 'string',
  84         enum => ['amd64', 'i386'],
  85         description => "OS architecture type.",
  86         default => 'amd64',
  87     },
  88     ostype => {
  89         optional => 1,
  90         type => 'string',
  91         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  92         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  93     },
  94     console => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Attach a console device (/dev/console) to the container.",
  98         default => 1,
  99     },
 100     tty => {
 101         optional => 1,
 102         type => 'integer',
 103         description => "Specify the number of tty available to the container",
 104         minimum => 0,
 105         maximum => 6,
 106         default => 2,
 107     },
 108     cpulimit => {
 109         optional => 1,
 110         type => 'number',
 111         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 112         minimum => 0,
 113         maximum => 128,
 114         default => 0,
 115     },
 116     cpuunits => {
 117         optional => 1,
 118         type => 'integer',
 119         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 120         minimum => 0,
 121         maximum => 500000,
 122         default => 1024,
 123     },
 124     memory => {
 125         optional => 1,
 126         type => 'integer',
 127         description => "Amount of RAM for the VM in MB.",
 128         minimum => 16,
 129         default => 512,
 130     },
 131     swap => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "Amount of SWAP for the VM in MB.",
 135         minimum => 0,
 136         default => 512,
 137     },
 138     hostname => {
 139         optional => 1,
 140         description => "Set a host name for the container.",
 141         type => 'string', format => 'dns-name',
 142         maxLength => 255,
 143     },
 144     description => {
 145         optional => 1,
 146         type => 'string',
 147         description => "Container description. Only used on the configuration web interface.",
 148     },
 149     searchdomain => {
 150         optional => 1,
 151         type => 'string', format => 'dns-name-list',
 152         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 153     },
 154     nameserver => {
 155         optional => 1,
 156         type => 'string', format => 'address-list',
 157         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 158     },
 159     rootfs => get_standard_option('pve-ct-rootfs'),
 160     parent => {
 161         optional => 1,
 162         type => 'string', format => 'pve-configid',
 163         maxLength => 40,
 164         description => "Parent snapshot name. This is used internally, and should not be modified.",
 165     },
 166     snaptime => {
 167         optional => 1,
 168         description => "Timestamp for snapshots.",
 169         type => 'integer',
 170         minimum => 0,
 171     },
 172     cmode => {
 173         optional => 1,
 174         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 175         type => 'string',
 176         enum => ['shell', 'console', 'tty'],
 177         default => 'tty',
 178     },
 179     protection => {
 180         optional => 1,
 181         type => 'boolean',
 182         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 183         default => 0,
 184     },
 185 };
 186
 187 my $valid_lxc_conf_keys = {
 188     'lxc.include' => 1,
 189     'lxc.arch' => 1,
 190     'lxc.utsname' => 1,
 191     'lxc.haltsignal' => 1,
 192     'lxc.rebootsignal' => 1,
 193     'lxc.stopsignal' => 1,
 194     'lxc.init_cmd' => 1,
 195     'lxc.network.type' => 1,
 196     'lxc.network.flags' => 1,
 197     'lxc.network.link' => 1,
 198     'lxc.network.mtu' => 1,
 199     'lxc.network.name' => 1,
 200     'lxc.network.hwaddr' => 1,
 201     'lxc.network.ipv4' => 1,
 202     'lxc.network.ipv4.gateway' => 1,
 203     'lxc.network.ipv6' => 1,
 204     'lxc.network.ipv6.gateway' => 1,
 205     'lxc.network.script.up' => 1,
 206     'lxc.network.script.down' => 1,
 207     'lxc.pts' => 1,
 208     'lxc.console.logfile' => 1,
 209     'lxc.console' => 1,
 210     'lxc.tty' => 1,
 211     'lxc.devttydir' => 1,
 212     'lxc.hook.autodev' => 1,
 213     'lxc.autodev' => 1,
 214     'lxc.kmsg' => 1,
 215     'lxc.mount' => 1,
 216     'lxc.mount.entry' => 1,
 217     'lxc.mount.auto' => 1,
 218     'lxc.rootfs' => 1,
 219     'lxc.rootfs.mount' => 1,
 220     'lxc.rootfs.options' => 1,
 221     # lxc.cgroup.*
 222     'lxc.cap.drop' => 1,
 223     'lxc.cap.keep' => 1,
 224     'lxc.aa_profile' => 1,
 225     'lxc.aa_allow_incomplete' => 1,
 226     'lxc.se_context' => 1,
 227     'lxc.seccomp' => 1,
 228     'lxc.id_map' => 1,
 229     'lxc.hook.pre-start' => 1,
 230     'lxc.hook.pre-mount' => 1,
 231     'lxc.hook.mount' => 1,
 232     'lxc.hook.start' => 1,
 233     'lxc.hook.post-stop' => 1,
 234     'lxc.hook.clone' => 1,
 235     'lxc.hook.destroy' => 1,
 236     'lxc.loglevel' => 1,
 237     'lxc.logfile' => 1,
 238     'lxc.start.auto' => 1,
 239     'lxc.start.delay' => 1,
 240     'lxc.start.order' => 1,
 241     'lxc.group' => 1,
 242     'lxc.environment' => 1,
 243     'lxc.' => 1,
 244     'lxc.' => 1,
 245     'lxc.' => 1,
 246     'lxc.' => 1,
 247 };
 248
 249 my $netconf_desc = {
 250     type => {
 251         type => 'string',
 252         optional => 1,
 253         description => "Network interface type.",
 254         enum => [qw(veth)],
 255     },
 256     name => {
 257         type => 'string',
 258         format_description => 'String',
 259         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 260         pattern => '[-_.\w\d]+',
 261     },
 262     bridge => {
 263         type => 'string',
 264         format_description => 'vmbr<Number>',
 265         description => 'Bridge to attach the network device to.',
 266         pattern => '[-_.\w\d]+',
 267     },
 268     hwaddr => {
 269         type => 'string',
 270         format_description => 'MAC',
 271         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 272         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 273         optional => 1,
 274     },
 275     mtu => {
 276         type => 'integer',
 277         format_description => 'Number',
 278         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 279         minimum => 64, # minimum ethernet frame is 64 bytes
 280         optional => 1,
 281     },
 282     ip => {
 283         type => 'string',
 284         format => 'pve-ipv4-config',
 285         format_description => 'IPv4Format/CIDR',
 286         description => 'IPv4 address in CIDR format.',
 287         optional => 1,
 288     },
 289     gw => {
 290         type => 'string',
 291         format => 'ipv4',
 292         format_description => 'GatewayIPv4',
 293         description => 'Default gateway for IPv4 traffic.',
 294         optional => 1,
 295     },
 296     ip6 => {
 297         type => 'string',
 298         format => 'pve-ipv6-config',
 299         format_description => 'IPv6Format/CIDR',
 300         description => 'IPv6 address in CIDR format.',
 301         optional => 1,
 302     },
 303     gw6 => {
 304         type => 'string',
 305         format => 'ipv6',
 306         format_description => 'GatewayIPv6',
 307         description => 'Default gateway for IPv6 traffic.',
 308         optional => 1,
 309     },
 310     firewall => {
 311         type => 'boolean',
 312         format_description => '[1|0]',
 313         description => "Controls whether this interface's firewall rules should be used.",
 314         optional => 1,
 315     },
 316     tag => {
 317         type => 'integer',
 318         format_description => 'VlanNo',
 319         minimum => '2',
 320         maximum => '4094',
 321         description => "VLAN tag foro this interface.",
 322         optional => 1,
 323     },
 324 };
 325 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 326
 327 my $MAX_LXC_NETWORKS = 10;
 328 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 329     $confdesc->{"net$i"} = {
 330         optional => 1,
 331         type => 'string', format => $netconf_desc,
 332         description => "Specifies network interfaces for the container.",
 333     };
 334 }
 335
 336 my $mp_desc = {
 337     %$rootfs_desc,
 338     mp => {
 339         type => 'string',
 340         format_description => 'Path',
 341         description => 'Path to the mountpoint as seen from inside the container.',
 342         optional => 1,
 343     },
 344 };
 345 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 346
 347 my $unuseddesc = {
 348     optional => 1,
 349     type => 'string', format => 'pve-volume-id',
 350     description => "Reference to unused volumes.",
 351 };
 352
 353 my $MAX_MOUNT_POINTS = 10;
 354 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 355     $confdesc->{"mp$i"} = {
 356         optional => 1,
 357         type => 'string', format => $mp_desc,
 358         description => "Use volume as container mount point (experimental feature).",
 359         optional => 1,
 360     };
 361 }
 362
 363 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 364 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 365     $confdesc->{"unused$i"} = $unuseddesc;
 366 }
 367
 368 sub write_pct_config {
 369     my ($filename, $conf) = @_;
 370
 371     delete $conf->{snapstate}; # just to be sure
 372
 373     my $generate_raw_config = sub {
 374         my ($conf) = @_;
 375
 376         my $raw = '';
 377
 378         # add description as comment to top of file
 379         my $descr = $conf->{description} || '';
 380         foreach my $cl (split(/\n/, $descr)) {
 381             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 382         }
 383
 384         foreach my $key (sort keys %$conf) {
 385             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 386                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 387             my $value = $conf->{$key};
 388             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 389             $raw .= "$key: $value\n";
 390         }
 391
 392         if (my $lxcconf = $conf->{lxc}) {
 393             foreach my $entry (@$lxcconf) {
 394                 my ($k, $v) = @$entry;
 395                 $raw .= "$k: $v\n";
 396             }
 397         }
 398
 399         return $raw;
 400     };
 401
 402     my $raw = &$generate_raw_config($conf);
 403
 404     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 405         $raw .= "\n[$snapname]\n";
 406         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 407     }
 408
 409     return $raw;
 410 }
 411
 412 sub check_type {
 413     my ($key, $value) = @_;
 414
 415     die "unknown setting '$key'\n" if !$confdesc->{$key};
 416
 417     my $type = $confdesc->{$key}->{type};
 418
 419     if (!defined($value)) {
 420         die "got undefined value\n";
 421     }
 422
 423     if ($value =~ m/[\n\r]/) {
 424         die "property contains a line feed\n";
 425     }
 426
 427     if ($type eq 'boolean') {
 428         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 429         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 430         die "type check ('boolean') failed - got '$value'\n";
 431     } elsif ($type eq 'integer') {
 432         return int($1) if $value =~ m/^(\d+)$/;
 433         die "type check ('integer') failed - got '$value'\n";
 434     } elsif ($type eq 'number') {
 435         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 436         die "type check ('number') failed - got '$value'\n";
 437     } elsif ($type eq 'string') {
 438         if (my $fmt = $confdesc->{$key}->{format}) {
 439             PVE::JSONSchema::check_format($fmt, $value);
 440             return $value;
 441         }
 442         return $value;
 443     } else {
 444         die "internal error"
 445     }
 446 }
 447
 448 sub parse_pct_config {
 449     my ($filename, $raw) = @_;
 450
 451     return undef if !defined($raw);
 452
 453     my $res = {
 454         digest => Digest::SHA::sha1_hex($raw),
 455         snapshots => {},
 456     };
 457
 458     $filename =~ m|/lxc/(\d+).conf$|
 459         || die "got strange filename '$filename'";
 460
 461     my $vmid = $1;
 462
 463     my $conf = $res;
 464     my $descr = '';
 465     my $section = '';
 466
 467     my @lines = split(/\n/, $raw);
 468     foreach my $line (@lines) {
 469         next if $line =~ m/^\s*$/;
 470
 471         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 472             $section = $1;
 473             $conf->{description} = $descr if $descr;
 474             $descr = '';
 475             $conf = $res->{snapshots}->{$section} = {};
 476             next;
 477         }
 478
 479         if ($line =~ m/^\#(.*)\s*$/) {
 480             $descr .= PVE::Tools::decode_text($1) . "\n";
 481             next;
 482         }
 483
 484         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 485             my $key = $1;
 486             my $value = $3;
 487             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 488                 push @{$conf->{lxc}}, [$key, $value];
 489             } else {
 490                 warn "vm $vmid - unable to parse config: $line\n";
 491             }
 492         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 493             $descr .= PVE::Tools::decode_text($2);
 494         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 495             $conf->{snapstate} = $1;
 496         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 497             my $key = $1;
 498             my $value = $2;
 499             eval { $value = check_type($key, $value); };
 500             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 501             $conf->{$key} = $value;
 502         } else {
 503             warn "vm $vmid - unable to parse config: $line\n";
 504         }
 505     }
 506
 507     $conf->{description} = $descr if $descr;
 508
 509     delete $res->{snapstate}; # just to be sure
 510
 511     return $res;
 512 }
 513
 514 sub config_list {
 515     my $vmlist = PVE::Cluster::get_vmlist();
 516     my $res = {};
 517     return $res if !$vmlist || !$vmlist->{ids};
 518     my $ids = $vmlist->{ids};
 519
 520     foreach my $vmid (keys %$ids) {
 521         next if !$vmid; # skip CT0
 522         my $d = $ids->{$vmid};
 523         next if !$d->{node} || $d->{node} ne $nodename;
 524         next if !$d->{type} || $d->{type} ne 'lxc';
 525         $res->{$vmid}->{type} = 'lxc';
 526     }
 527     return $res;
 528 }
 529
 530 sub cfs_config_path {
 531     my ($vmid, $node) = @_;
 532
 533     $node = $nodename if !$node;
 534     return "nodes/$node/lxc/$vmid.conf";
 535 }
 536
 537 sub config_file {
 538     my ($vmid, $node) = @_;
 539
 540     my $cfspath = cfs_config_path($vmid, $node);
 541     return "/etc/pve/$cfspath";
 542 }
 543
 544 sub load_config {
 545     my ($vmid, $node) = @_;
 546
 547     $node = $nodename if !$node;
 548     my $cfspath = cfs_config_path($vmid, $node);
 549
 550     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 551     die "container $vmid does not exists\n" if !defined($conf);
 552
 553     return $conf;
 554 }
 555
 556 sub create_config {
 557     my ($vmid, $conf) = @_;
 558
 559     my $dir = "/etc/pve/nodes/$nodename/lxc";
 560     mkdir $dir;
 561
 562     write_config($vmid, $conf);
 563 }
 564
 565 sub destroy_config {
 566     my ($vmid) = @_;
 567
 568     unlink config_file($vmid, $nodename);
 569 }
 570
 571 sub write_config {
 572     my ($vmid, $conf) = @_;
 573
 574     my $cfspath = cfs_config_path($vmid);
 575
 576     PVE::Cluster::cfs_write_file($cfspath, $conf);
 577 }
 578
 579 # flock: we use one file handle per process, so lock file
 580 # can be called multiple times and succeeds for the same process.
 581
 582 my $lock_handles =  {};
 583 my $lockdir = "/run/lock/lxc";
 584
 585 sub lock_filename {
 586     my ($vmid) = @_;
 587
 588     return "$lockdir/pve-config-${vmid}.lock";
 589 }
 590
 591 sub lock_aquire {
 592     my ($vmid, $timeout) = @_;
 593
 594     $timeout = 10 if !$timeout;
 595     my $mode = LOCK_EX;
 596
 597     my $filename = lock_filename($vmid);
 598
 599     mkdir $lockdir if !-d $lockdir;
 600
 601     my $lock_func = sub {
 602         if (!$lock_handles->{$$}->{$filename}) {
 603             my $fh = new IO::File(">>$filename") ||
 604                 die "can't open file - $!\n";
 605             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 606         }
 607
 608         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 609             print STDERR "trying to aquire lock...";
 610             my $success;
 611             while(1) {
 612                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 613                 # try again on EINTR (see bug #273)
 614                 if ($success || ($! != EINTR)) {
 615                     last;
 616                 }
 617             }
 618             if (!$success) {
 619                 print STDERR " failed\n";
 620                 die "can't aquire lock - $!\n";
 621             }
 622
 623             print STDERR " OK\n";
 624         }
 625
 626         $lock_handles->{$$}->{$filename}->{refcount}++;
 627     };
 628
 629     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 630     my $err = $@;
 631     if ($err) {
 632         die "can't lock file '$filename' - $err";
 633     }
 634 }
 635
 636 sub lock_release {
 637     my ($vmid) = @_;
 638
 639     my $filename = lock_filename($vmid);
 640
 641     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 642         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 643         if ($refcount <= 0) {
 644             $lock_handles->{$$}->{$filename} = undef;
 645             close ($fh);
 646         }
 647     }
 648 }
 649
 650 sub lock_container {
 651     my ($vmid, $timeout, $code, @param) = @_;
 652
 653     my $res;
 654
 655     lock_aquire($vmid, $timeout);
 656     eval { $res = &$code(@param) };
 657     my $err = $@;
 658     lock_release($vmid);
 659
 660     die $err if $err;
 661
 662     return $res;
 663 }
 664
 665 sub option_exists {
 666     my ($name) = @_;
 667
 668     return defined($confdesc->{$name});
 669 }
 670
 671 # add JSON properties for create and set function
 672 sub json_config_properties {
 673     my $prop = shift;
 674
 675     foreach my $opt (keys %$confdesc) {
 676         next if $opt eq 'parent' || $opt eq 'snaptime';
 677         next if $prop->{$opt};
 678         $prop->{$opt} = $confdesc->{$opt};
 679     }
 680
 681     return $prop;
 682 }
 683
 684 sub json_config_properties_no_rootfs {
 685     my $prop = shift;
 686
 687     foreach my $opt (keys %$confdesc) {
 688         next if $prop->{$opt};
 689         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 690         $prop->{$opt} = $confdesc->{$opt};
 691     }
 692
 693     return $prop;
 694 }
 695
 696 # container status helpers
 697
 698 sub list_active_containers {
 699
 700     my $filename = "/proc/net/unix";
 701
 702     # similar test is used by lcxcontainers.c: list_active_containers
 703     my $res = {};
 704
 705     my $fh = IO::File->new ($filename, "r");
 706     return $res if !$fh;
 707
 708     while (defined(my $line = <$fh>)) {
 709         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 710             my $path = $1;
 711             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 712                 $res->{$1} = 1;
 713             }
 714         }
 715     }
 716
 717     close($fh);
 718
 719     return $res;
 720 }
 721
 722 # warning: this is slow
 723 sub check_running {
 724     my ($vmid) = @_;
 725
 726     my $active_hash = list_active_containers();
 727
 728     return 1 if defined($active_hash->{$vmid});
 729
 730     return undef;
 731 }
 732
 733 sub get_container_disk_usage {
 734     my ($vmid) = @_;
 735
 736     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 737
 738     my $res = {
 739         total => 0,
 740         used => 0,
 741         avail => 0,
 742     };
 743
 744     my $parser = sub {
 745         my $line = shift;
 746         if (my ($fsid, $total, $used, $avail) = $line =~
 747             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 748             $res = {
 749                 total => $total,
 750                 used => $used,
 751                 avail => $avail,
 752             };
 753         }
 754     };
 755     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 756     warn $@ if $@;
 757
 758     return $res;
 759 }
 760
 761 sub vmstatus {
 762     my ($opt_vmid) = @_;
 763
 764     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 765
 766     my $active_hash = list_active_containers();
 767
 768     foreach my $vmid (keys %$list) {
 769         my $d = $list->{$vmid};
 770
 771         my $running = defined($active_hash->{$vmid});
 772
 773         $d->{status} = $running ? 'running' : 'stopped';
 774
 775         my $cfspath = cfs_config_path($vmid);
 776         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 777
 778         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 779         $d->{name} =~ s/[\s]//g;
 780
 781         $d->{cpus} = $conf->{cpulimit} // 0;
 782
 783         if ($running) {
 784             my $res = get_container_disk_usage($vmid);
 785             $d->{disk} = $res->{used};
 786             $d->{maxdisk} = $res->{total};
 787         } else {
 788             $d->{disk} = 0;
 789             # use 4GB by default ??
 790             if (my $rootfs = $conf->{rootfs}) {
 791                 my $rootinfo = parse_ct_mountpoint($rootfs);
 792                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 793             } else {
 794                 $d->{maxdisk} = 4*1024*1024*1024;
 795             }
 796         }
 797
 798         $d->{mem} = 0;
 799         $d->{swap} = 0;
 800         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 801         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 802
 803         $d->{uptime} = 0;
 804         $d->{cpu} = 0;
 805
 806         $d->{netout} = 0;
 807         $d->{netin} = 0;
 808
 809         $d->{diskread} = 0;
 810         $d->{diskwrite} = 0;
 811
 812         $d->{template} = is_template($conf);
 813     }
 814
 815     foreach my $vmid (keys %$list) {
 816         my $d = $list->{$vmid};
 817         next if $d->{status} ne 'running';
 818
 819         my $pid = find_lxc_pid($vmid);
 820         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 821         $d->{uptime} = time - $ctime; # the method lxcfs uses
 822
 823         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 824         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 825
 826         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 827         my @bytes = split(/\n/, $blkio_bytes);
 828         foreach my $byte (@bytes) {
 829             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 830                 $d->{diskread} = $2 if $key eq 'Read';
 831                 $d->{diskwrite} = $2 if $key eq 'Write';
 832             }
 833         }
 834     }
 835
 836     return $list;
 837 }
 838
 839 my $parse_size = sub {
 840     my ($value) = @_;
 841
 842     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 843     my ($size, $unit) = ($1, $3);
 844     if ($unit) {
 845         if ($unit eq 'K') {
 846             $size = $size * 1024;
 847         } elsif ($unit eq 'M') {
 848             $size = $size * 1024 * 1024;
 849         } elsif ($unit eq 'G') {
 850             $size = $size * 1024 * 1024 * 1024;
 851         }
 852     }
 853     return int($size);
 854 };
 855
 856 my $format_size = sub {
 857     my ($size) = @_;
 858
 859     $size = int($size);
 860
 861     my $kb = int($size/1024);
 862     return $size if $kb*1024 != $size;
 863
 864     my $mb = int($kb/1024);
 865     return "${kb}K" if $mb*1024 != $kb;
 866
 867     my $gb = int($mb/1024);
 868     return "${mb}M" if $gb*1024 != $mb;
 869
 870     return "${gb}G";
 871 };
 872
 873 sub parse_ct_mountpoint {
 874     my ($data) = @_;
 875
 876     $data //= '';
 877
 878     my $res;
 879     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 880     if ($@) {
 881         warn $@;
 882         return undef;
 883     }
 884
 885     return undef if !defined($res->{volume});
 886
 887     if ($res->{size}) {
 888         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 889     }
 890
 891     return $res;
 892 }
 893
 894 sub print_ct_mountpoint {
 895     my ($info, $nomp) = @_;
 896     my $skip = $nomp ? ['mp'] : [];
 897     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 898 }
 899
 900 sub print_lxc_network {
 901     my $net = shift;
 902     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 903 }
 904
 905 sub parse_lxc_network {
 906     my ($data) = @_;
 907
 908     my $res = {};
 909
 910     return $res if !$data;
 911
 912     eval { $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data) };
 913     if ($@) {
 914         warn $@;
 915         return undef;
 916     }
 917
 918     $res->{type} = 'veth';
 919     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 920
 921     return $res;
 922 }
 923
 924 sub read_cgroup_value {
 925     my ($group, $vmid, $name, $full) = @_;
 926
 927     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 928
 929     return PVE::Tools::file_get_contents($path) if $full;
 930
 931     return PVE::Tools::file_read_firstline($path);
 932 }
 933
 934 sub write_cgroup_value {
 935    my ($group, $vmid, $name, $value) = @_;
 936
 937    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 938    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 939
 940 }
 941
 942 sub find_lxc_console_pids {
 943
 944     my $res = {};
 945
 946     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 947         my ($pid) = @_;
 948
 949         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 950         return if !$cmdline;
 951
 952         my @args = split(/\0/, $cmdline);
 953
 954         # serach for lxc-console -n <vmid>
 955         return if scalar(@args) != 3;
 956         return if $args[1] ne '-n';
 957         return if $args[2] !~ m/^\d+$/;
 958         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 959
 960         my $vmid = $args[2];
 961
 962         push @{$res->{$vmid}}, $pid;
 963     });
 964
 965     return $res;
 966 }
 967
 968 sub find_lxc_pid {
 969     my ($vmid) = @_;
 970
 971     my $pid = undef;
 972     my $parser = sub {
 973         my $line = shift;
 974         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 975     };
 976     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
 977
 978     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 979
 980     return $pid;
 981 }
 982
 983 my $ipv4_reverse_mask = [
 984     '0.0.0.0',
 985     '128.0.0.0',
 986     '192.0.0.0',
 987     '224.0.0.0',
 988     '240.0.0.0',
 989     '248.0.0.0',
 990     '252.0.0.0',
 991     '254.0.0.0',
 992     '255.0.0.0',
 993     '255.128.0.0',
 994     '255.192.0.0',
 995     '255.224.0.0',
 996     '255.240.0.0',
 997     '255.248.0.0',
 998     '255.252.0.0',
 999     '255.254.0.0',
1000     '255.255.0.0',
1001     '255.255.128.0',
1002     '255.255.192.0',
1003     '255.255.224.0',
1004     '255.255.240.0',
1005     '255.255.248.0',
1006     '255.255.252.0',
1007     '255.255.254.0',
1008     '255.255.255.0',
1009     '255.255.255.128',
1010     '255.255.255.192',
1011     '255.255.255.224',
1012     '255.255.255.240',
1013     '255.255.255.248',
1014     '255.255.255.252',
1015     '255.255.255.254',
1016     '255.255.255.255',
1017 ];
1018
1019 # Note: we cannot use Net:IP, because that only allows strict
1020 # CIDR networks
1021 sub parse_ipv4_cidr {
1022     my ($cidr, $noerr) = @_;
1023
1024     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
1025         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
1026     }
1027
1028     return undef if $noerr;
1029
1030     die "unable to parse ipv4 address/mask\n";
1031 }
1032
1033 sub check_lock {
1034     my ($conf) = @_;
1035
1036     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1037 }
1038
1039 sub check_protection {
1040     my ($vm_conf, $err_msg) = @_;
1041
1042     if ($vm_conf->{protection}) {
1043         die "$err_msg - protection mode enabled\n";
1044     }
1045 }
1046
1047 sub update_lxc_config {
1048     my ($storage_cfg, $vmid, $conf) = @_;
1049
1050     my $dir = "/var/lib/lxc/$vmid";
1051
1052     if ($conf->{template}) {
1053
1054         unlink "$dir/config";
1055
1056         return;
1057     }
1058
1059     my $raw = '';
1060
1061     die "missing 'arch' - internal error" if !$conf->{arch};
1062     $raw .= "lxc.arch = $conf->{arch}\n";
1063
1064     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1065     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1066         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1067     } else {
1068         die "implement me";
1069     }
1070
1071     if (!has_dev_console($conf)) {
1072         $raw .= "lxc.console = none\n";
1073         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1074     }
1075
1076     my $ttycount = get_tty_count($conf);
1077     $raw .= "lxc.tty = $ttycount\n";
1078
1079     # some init scripts expects a linux terminal (turnkey).
1080     $raw .= "lxc.environment = TERM=linux\n";
1081
1082     my $utsname = $conf->{hostname} || "CT$vmid";
1083     $raw .= "lxc.utsname = $utsname\n";
1084
1085     my $memory = $conf->{memory} || 512;
1086     my $swap = $conf->{swap} // 0;
1087
1088     my $lxcmem = int($memory*1024*1024);
1089     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1090
1091     my $lxcswap = int(($memory + $swap)*1024*1024);
1092     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1093
1094     if (my $cpulimit = $conf->{cpulimit}) {
1095         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1096         my $value = int(100000*$cpulimit);
1097         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1098     }
1099
1100     my $shares = $conf->{cpuunits} || 1024;
1101     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1102
1103     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1104     $mountpoint->{mp} = '/';
1105
1106     my ($path, $use_loopdev) = mountpoint_mount_path($mountpoint, $storage_cfg);
1107     $path = "loop:$path" if $use_loopdev;
1108
1109     $raw .= "lxc.rootfs = $path\n";
1110
1111     my $netcount = 0;
1112     foreach my $k (keys %$conf) {
1113         next if $k !~ m/^net(\d+)$/;
1114         my $ind = $1;
1115         my $d = parse_lxc_network($conf->{$k});
1116         $netcount++;
1117         $raw .= "lxc.network.type = veth\n";
1118         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1119         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1120         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1121         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1122     }
1123
1124     if (my $lxcconf = $conf->{lxc}) {
1125         foreach my $entry (@$lxcconf) {
1126             my ($k, $v) = @$entry;
1127             $netcount++ if $k eq 'lxc.network.type';
1128             $raw .= "$k = $v\n";
1129         }
1130     }
1131
1132     $raw .= "lxc.network.type = empty\n" if !$netcount;
1133
1134     File::Path::mkpath("$dir/rootfs");
1135
1136     PVE::Tools::file_set_contents("$dir/config", $raw);
1137 }
1138
1139 # verify and cleanup nameserver list (replace \0 with ' ')
1140 sub verify_nameserver_list {
1141     my ($nameserver_list) = @_;
1142
1143     my @list = ();
1144     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1145         PVE::JSONSchema::pve_verify_ip($server);
1146         push @list, $server;
1147     }
1148
1149     return join(' ', @list);
1150 }
1151
1152 sub verify_searchdomain_list {
1153     my ($searchdomain_list) = @_;
1154
1155     my @list = ();
1156     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1157         # todo: should we add checks for valid dns domains?
1158         push @list, $server;
1159     }
1160
1161     return join(' ', @list);
1162 }
1163
1164 sub add_unused_volume {
1165     my ($config, $volid) = @_;
1166
1167     my $key;
1168     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1169         my $test = "unused$ind";
1170         if (my $vid = $config->{$test}) {
1171             return if $vid eq $volid; # do not add duplicates
1172         } else {
1173             $key = $test;
1174         }
1175     }
1176
1177     die "To many unused volume - please delete them first.\n" if !$key;
1178
1179     $config->{$key} = $volid;
1180
1181     return $key;
1182 }
1183
1184 sub update_pct_config {
1185     my ($vmid, $conf, $running, $param, $delete) = @_;
1186
1187     my @nohotplug;
1188
1189     my $new_disks = 0;
1190     my @deleted_volumes;
1191
1192     my $rootdir;
1193     if ($running) {
1194         my $pid = find_lxc_pid($vmid);
1195         $rootdir = "/proc/$pid/root";
1196     }
1197
1198     if (defined($delete)) {
1199         foreach my $opt (@$delete) {
1200             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1201                 die "unable to delete required option '$opt'\n";
1202             } elsif ($opt eq 'swap') {
1203                 delete $conf->{$opt};
1204                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1205             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1206                 delete $conf->{$opt};
1207             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1208                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1209                 delete $conf->{$opt};
1210                 push @nohotplug, $opt;
1211                 next if $running;
1212             } elsif ($opt =~ m/^net(\d)$/) {
1213                 delete $conf->{$opt};
1214                 next if !$running;
1215                 my $netid = $1;
1216                 PVE::Network::veth_delete("veth${vmid}i$netid");
1217             } elsif ($opt eq 'protection') {
1218                 delete $conf->{$opt};
1219             } elsif ($opt =~ m/^unused(\d+)$/) {
1220                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1221                 push @deleted_volumes, $conf->{$opt};
1222                 delete $conf->{$opt};
1223                 push @nohotplug, $opt;
1224                 next if $running;
1225             } elsif ($opt =~ m/^mp(\d+)$/) {
1226                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1227                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1228                 add_unused_volume($conf, $mountpoint->{volume});
1229                 delete $conf->{$opt};
1230                 push @nohotplug, $opt;
1231                 next if $running;
1232             } else {
1233                 die "implement me"
1234             }
1235             write_config($vmid, $conf) if $running;
1236         }
1237     }
1238
1239     # There's no separate swap size to configure, there's memory and "total"
1240     # memory (iow. memory+swap). This means we have to change them together.
1241     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1242     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1243     if (defined($wanted_memory) || defined($wanted_swap)) {
1244
1245         $wanted_memory //= ($conf->{memory} || 512);
1246         $wanted_swap //=  ($conf->{swap} || 0);
1247
1248         my $total = $wanted_memory + $wanted_swap;
1249         if ($running) {
1250             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1251             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1252         }
1253         $conf->{memory} = $wanted_memory;
1254         $conf->{swap} = $wanted_swap;
1255
1256         write_config($vmid, $conf) if $running;
1257     }
1258
1259     foreach my $opt (keys %$param) {
1260         my $value = $param->{$opt};
1261         if ($opt eq 'hostname') {
1262             $conf->{$opt} = $value;
1263         } elsif ($opt eq 'onboot') {
1264             $conf->{$opt} = $value ? 1 : 0;
1265         } elsif ($opt eq 'startup') {
1266             $conf->{$opt} = $value;
1267         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1268             $conf->{$opt} = $value;
1269             push @nohotplug, $opt;
1270             next if $running;
1271         } elsif ($opt eq 'nameserver') {
1272             my $list = verify_nameserver_list($value);
1273             $conf->{$opt} = $list;
1274             push @nohotplug, $opt;
1275             next if $running;
1276         } elsif ($opt eq 'searchdomain') {
1277             my $list = verify_searchdomain_list($value);
1278             $conf->{$opt} = $list;
1279             push @nohotplug, $opt;
1280             next if $running;
1281         } elsif ($opt eq 'cpulimit') {
1282             $conf->{$opt} = $value;
1283             push @nohotplug, $opt; # fixme: hotplug
1284             next;
1285         } elsif ($opt eq 'cpuunits') {
1286             $conf->{$opt} = $value;
1287             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1288         } elsif ($opt eq 'description') {
1289             $conf->{$opt} = PVE::Tools::encode_text($value);
1290         } elsif ($opt =~ m/^net(\d+)$/) {
1291             my $netid = $1;
1292             my $net = parse_lxc_network($value);
1293             if (!$running) {
1294                 $conf->{$opt} = print_lxc_network($net);
1295             } else {
1296                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1297             }
1298         } elsif ($opt eq 'protection') {
1299             $conf->{$opt} = $value ? 1 : 0;
1300         } elsif ($opt =~ m/^mp(\d+)$/) {
1301             check_protection($conf, "can't update CT $vmid drive '$opt'");
1302             $conf->{$opt} = $value;
1303             $new_disks = 1;
1304             push @nohotplug, $opt;
1305             next;
1306         } elsif ($opt eq 'rootfs') {
1307             check_protection($conf, "can't update CT $vmid drive '$opt'");
1308             die "implement me: $opt";
1309         } else {
1310             die "implement me: $opt";
1311         }
1312         write_config($vmid, $conf) if $running;
1313     }
1314
1315     if ($running && scalar(@nohotplug)) {
1316         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1317     }
1318
1319     if (@deleted_volumes) {
1320         my $storage_cfg = PVE::Storage::config();
1321         foreach my $volume (@deleted_volumes) {
1322             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1323         }
1324     }
1325
1326     if ($new_disks) {
1327         my $storage_cfg = PVE::Storage::config();
1328         create_disks($storage_cfg, $vmid, $conf, $conf);
1329     }
1330 }
1331
1332 sub has_dev_console {
1333     my ($conf) = @_;
1334
1335     return !(defined($conf->{console}) && !$conf->{console});
1336 }
1337
1338 sub get_tty_count {
1339     my ($conf) = @_;
1340
1341     return $conf->{tty} // $confdesc->{tty}->{default};
1342 }
1343
1344 sub get_cmode {
1345     my ($conf) = @_;
1346
1347     return $conf->{cmode} // $confdesc->{cmode}->{default};
1348 }
1349
1350 sub get_console_command {
1351     my ($vmid, $conf) = @_;
1352
1353     my $cmode = get_cmode($conf);
1354
1355     if ($cmode eq 'console') {
1356         return ['lxc-console', '-n',  $vmid, '-t', 0];
1357     } elsif ($cmode eq 'tty') {
1358         return ['lxc-console', '-n',  $vmid];
1359     } elsif ($cmode eq 'shell') {
1360         return ['lxc-attach', '--clear-env', '-n', $vmid];
1361     } else {
1362         die "internal error";
1363     }
1364 }
1365
1366 sub get_primary_ips {
1367     my ($conf) = @_;
1368
1369     # return data from net0
1370
1371     return undef if !defined($conf->{net0});
1372     my $net = parse_lxc_network($conf->{net0});
1373
1374     my $ipv4 = $net->{ip};
1375     if ($ipv4) {
1376         if ($ipv4 =~ /^(dhcp|manual)$/) {
1377             $ipv4 = undef
1378         } else {
1379             $ipv4 =~ s!/\d+$!!;
1380         }
1381     }
1382     my $ipv6 = $net->{ip6};
1383     if ($ipv6) {
1384         if ($ipv6 =~ /^(dhcp|manual)$/) {
1385             $ipv6 = undef;
1386         } else {
1387             $ipv6 =~ s!/\d+$!!;
1388         }
1389     }
1390
1391     return ($ipv4, $ipv6);
1392 }
1393
1394 sub delete_mountpoint_volume {
1395     my ($storage_cfg, $vmid, $volume) = @_;
1396
1397     # skip bind mounts and block devices
1398     if ($volume =~ m|^/|) {
1399             return;
1400     }
1401
1402     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1403     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1404 }
1405
1406 sub destroy_lxc_container {
1407     my ($storage_cfg, $vmid, $conf) = @_;
1408
1409     foreach_mountpoint($conf, sub {
1410         my ($ms, $mountpoint) = @_;
1411         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1412     });
1413
1414     rmdir "/var/lib/lxc/$vmid/rootfs";
1415     unlink "/var/lib/lxc/$vmid/config";
1416     rmdir "/var/lib/lxc/$vmid";
1417     destroy_config($vmid);
1418
1419     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1420     #PVE::Tools::run_command($cmd);
1421 }
1422
1423 sub vm_stop_cleanup {
1424     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1425
1426     eval {
1427         if (!$keepActive) {
1428
1429             my $vollist = get_vm_volumes($conf);
1430             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1431         }
1432     };
1433     warn $@ if $@; # avoid errors - just warn
1434 }
1435
1436 my $safe_num_ne = sub {
1437     my ($a, $b) = @_;
1438
1439     return 0 if !defined($a) && !defined($b);
1440     return 1 if !defined($a);
1441     return 1 if !defined($b);
1442
1443     return $a != $b;
1444 };
1445
1446 my $safe_string_ne = sub {
1447     my ($a, $b) = @_;
1448
1449     return 0 if !defined($a) && !defined($b);
1450     return 1 if !defined($a);
1451     return 1 if !defined($b);
1452
1453     return $a ne $b;
1454 };
1455
1456 sub update_net {
1457     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1458
1459     if ($newnet->{type} ne 'veth') {
1460         # for when there are physical interfaces
1461         die "cannot update interface of type $newnet->{type}";
1462     }
1463
1464     my $veth = "veth${vmid}i${netid}";
1465     my $eth = $newnet->{name};
1466
1467     if (my $oldnetcfg = $conf->{$opt}) {
1468         my $oldnet = parse_lxc_network($oldnetcfg);
1469
1470         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1471             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1472
1473             PVE::Network::veth_delete($veth);
1474             delete $conf->{$opt};
1475             write_config($vmid, $conf);
1476
1477             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1478
1479         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1480                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1481                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1482
1483                 if ($oldnet->{bridge}) {
1484                     PVE::Network::tap_unplug($veth);
1485                     foreach (qw(bridge tag firewall)) {
1486                         delete $oldnet->{$_};
1487                     }
1488                     $conf->{$opt} = print_lxc_network($oldnet);
1489                     write_config($vmid, $conf);
1490                 }
1491
1492                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1493                 foreach (qw(bridge tag firewall)) {
1494                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1495                 }
1496                 $conf->{$opt} = print_lxc_network($oldnet);
1497                 write_config($vmid, $conf);
1498         }
1499     } else {
1500         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1501     }
1502
1503     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1504 }
1505
1506 sub hotplug_net {
1507     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1508
1509     my $veth = "veth${vmid}i${netid}";
1510     my $vethpeer = $veth . "p";
1511     my $eth = $newnet->{name};
1512
1513     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1514     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1515
1516     # attach peer in container
1517     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1518     PVE::Tools::run_command($cmd);
1519
1520     # link up peer in container
1521     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1522     PVE::Tools::run_command($cmd);
1523
1524     my $done = { type => 'veth' };
1525     foreach (qw(bridge tag firewall hwaddr name)) {
1526         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1527     }
1528     $conf->{$opt} = print_lxc_network($done);
1529
1530     write_config($vmid, $conf);
1531 }
1532
1533 sub update_ipconfig {
1534     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1535
1536     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1537
1538     my $optdata = parse_lxc_network($conf->{$opt});
1539     my $deleted = [];
1540     my $added = [];
1541     my $nscmd = sub {
1542         my $cmdargs = shift;
1543         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1544     };
1545     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1546
1547     my $change_ip_config = sub {
1548         my ($ipversion) = @_;
1549
1550         my $family_opt = "-$ipversion";
1551         my $suffix = $ipversion == 4 ? '' : $ipversion;
1552         my $gw= "gw$suffix";
1553         my $ip= "ip$suffix";
1554
1555         my $newip = $newnet->{$ip};
1556         my $newgw = $newnet->{$gw};
1557         my $oldip = $optdata->{$ip};
1558
1559         my $change_ip = &$safe_string_ne($oldip, $newip);
1560         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1561
1562         return if !$change_ip && !$change_gw;
1563
1564         # step 1: add new IP, if this fails we cancel
1565         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1566             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1567             if (my $err = $@) {
1568                 warn $err;
1569                 return;
1570             }
1571         }
1572
1573         # step 2: replace gateway
1574         #   If this fails we delete the added IP and cancel.
1575         #   If it succeeds we save the config and delete the old IP, ignoring
1576         #   errors. The config is then saved.
1577         # Note: 'ip route replace' can add
1578         if ($change_gw) {
1579             if ($newgw) {
1580                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1581                 if (my $err = $@) {
1582                     warn $err;
1583                     # the route was not replaced, the old IP is still available
1584                     # rollback (delete new IP) and cancel
1585                     if ($change_ip) {
1586                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1587                         warn $@ if $@; # no need to die here
1588                     }
1589                     return;
1590                 }
1591             } else {
1592                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1593                 # if the route was not deleted, the guest might have deleted it manually
1594                 # warn and continue
1595                 warn $@ if $@;
1596             }
1597         }
1598
1599         # from this point on we save the configuration
1600         # step 3: delete old IP ignoring errors
1601         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1602             # We need to enable promote_secondaries, otherwise our newly added
1603             # address will be removed along with the old one.
1604             my $promote = 0;
1605             eval {
1606                 if ($ipversion == 4) {
1607                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1608                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1609                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1610                 }
1611                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1612             };
1613             warn $@ if $@; # no need to die here
1614
1615             if ($ipversion == 4) {
1616                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1617             }
1618         }
1619
1620         foreach my $property ($ip, $gw) {
1621             if ($newnet->{$property}) {
1622                 $optdata->{$property} = $newnet->{$property};
1623             } else {
1624                 delete $optdata->{$property};
1625             }
1626         }
1627         $conf->{$opt} = print_lxc_network($optdata);
1628         write_config($vmid, $conf);
1629         $lxc_setup->setup_network($conf);
1630     };
1631
1632     &$change_ip_config(4);
1633     &$change_ip_config(6);
1634
1635 }
1636
1637 # Internal snapshots
1638
1639 # NOTE: Snapshot create/delete involves several non-atomic
1640 # action, and can take a long time.
1641 # So we try to avoid locking the file and use 'lock' variable
1642 # inside the config file instead.
1643
1644 my $snapshot_copy_config = sub {
1645     my ($source, $dest) = @_;
1646
1647     foreach my $k (keys %$source) {
1648         next if $k eq 'snapshots';
1649         next if $k eq 'snapstate';
1650         next if $k eq 'snaptime';
1651         next if $k eq 'vmstate';
1652         next if $k eq 'lock';
1653         next if $k eq 'digest';
1654         next if $k eq 'description';
1655
1656         $dest->{$k} = $source->{$k};
1657     }
1658 };
1659
1660 my $snapshot_prepare = sub {
1661     my ($vmid, $snapname, $comment) = @_;
1662
1663     my $snap;
1664
1665     my $updatefn =  sub {
1666
1667         my $conf = load_config($vmid);
1668
1669         die "you can't take a snapshot if it's a template\n"
1670             if is_template($conf);
1671
1672         check_lock($conf);
1673
1674         $conf->{lock} = 'snapshot';
1675
1676         die "snapshot name '$snapname' already used\n"
1677             if defined($conf->{snapshots}->{$snapname});
1678
1679         my $storecfg = PVE::Storage::config();
1680         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1681
1682         $snap = $conf->{snapshots}->{$snapname} = {};
1683
1684         &$snapshot_copy_config($conf, $snap);
1685
1686         $snap->{'snapstate'} = "prepare";
1687         $snap->{'snaptime'} = time();
1688         $snap->{'description'} = $comment if $comment;
1689         $conf->{snapshots}->{$snapname} = $snap;
1690
1691         write_config($vmid, $conf);
1692     };
1693
1694     lock_container($vmid, 10, $updatefn);
1695
1696     return $snap;
1697 };
1698
1699 my $snapshot_commit = sub {
1700     my ($vmid, $snapname) = @_;
1701
1702     my $updatefn = sub {
1703
1704         my $conf = load_config($vmid);
1705
1706         die "missing snapshot lock\n"
1707             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1708
1709         die "snapshot '$snapname' does not exist\n"
1710             if !defined($conf->{snapshots}->{$snapname});
1711
1712         die "wrong snapshot state\n"
1713             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1714                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1715
1716         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1717         delete $conf->{lock};
1718         $conf->{parent} = $snapname;
1719
1720         write_config($vmid, $conf);
1721     };
1722
1723     lock_container($vmid, 10 ,$updatefn);
1724 };
1725
1726 sub has_feature {
1727     my ($feature, $conf, $storecfg, $snapname) = @_;
1728
1729     my $err;
1730
1731     foreach_mountpoint($conf, sub {
1732         my ($ms, $mountpoint) = @_;
1733
1734         return if $err; # skip further test
1735
1736         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1737
1738         # TODO: implement support for mountpoints
1739         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1740             if $ms ne 'rootfs';
1741     });
1742
1743     return $err ? 0 : 1;
1744 }
1745
1746 sub snapshot_create {
1747     my ($vmid, $snapname, $comment) = @_;
1748
1749     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1750
1751     my $conf = load_config($vmid);
1752
1753     my $running = check_running($vmid);
1754     eval {
1755         if ($running) {
1756             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1757             PVE::Tools::run_command(['/bin/sync']);
1758         };
1759
1760         my $storecfg = PVE::Storage::config();
1761         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1762         my $volid = $rootinfo->{volume};
1763
1764         if ($running) {
1765             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1766         };
1767
1768         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1769         &$snapshot_commit($vmid, $snapname);
1770     };
1771     if(my $err = $@) {
1772         snapshot_delete($vmid, $snapname, 1);
1773         die "$err\n";
1774     }
1775 }
1776
1777 sub snapshot_delete {
1778     my ($vmid, $snapname, $force) = @_;
1779
1780     my $snap;
1781
1782     my $conf;
1783
1784     my $updatefn =  sub {
1785
1786         $conf = load_config($vmid);
1787
1788         die "you can't delete a snapshot if vm is a template\n"
1789             if is_template($conf);
1790
1791         $snap = $conf->{snapshots}->{$snapname};
1792
1793         check_lock($conf);
1794
1795         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1796
1797         $snap->{snapstate} = 'delete';
1798
1799         write_config($vmid, $conf);
1800     };
1801
1802     lock_container($vmid, 10, $updatefn);
1803
1804     my $storecfg = PVE::Storage::config();
1805
1806     my $del_snap =  sub {
1807
1808         check_lock($conf);
1809
1810         if ($conf->{parent} eq $snapname) {
1811             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1812                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1813             } else {
1814                 delete $conf->{parent};
1815             }
1816         }
1817
1818         delete $conf->{snapshots}->{$snapname};
1819
1820         write_config($vmid, $conf);
1821     };
1822
1823     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1824     my $rootinfo = parse_ct_mountpoint($rootfs);
1825     my $volid = $rootinfo->{volume};
1826
1827     eval {
1828         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1829     };
1830     my $err = $@;
1831
1832     if(!$err || ($err && $force)) {
1833         lock_container($vmid, 10, $del_snap);
1834         if ($err) {
1835             die "Can't delete snapshot: $vmid $snapname $err\n";
1836         }
1837     }
1838 }
1839
1840 sub snapshot_rollback {
1841     my ($vmid, $snapname) = @_;
1842
1843     my $storecfg = PVE::Storage::config();
1844
1845     my $conf = load_config($vmid);
1846
1847     die "you can't rollback if vm is a template\n" if is_template($conf);
1848
1849     my $snap = $conf->{snapshots}->{$snapname};
1850
1851     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1852
1853     my $rootfs = $snap->{rootfs};
1854     my $rootinfo = parse_ct_mountpoint($rootfs);
1855     my $volid = $rootinfo->{volume};
1856
1857     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1858
1859     my $updatefn = sub {
1860
1861         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1862             if $snap->{snapstate};
1863
1864         check_lock($conf);
1865
1866         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1867
1868         die "unable to rollback vm $vmid: vm is running\n"
1869             if check_running($vmid);
1870
1871         $conf->{lock} = 'rollback';
1872
1873         my $forcemachine;
1874
1875         # copy snapshot config to current config
1876
1877         my $tmp_conf = $conf;
1878         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1879         $conf->{snapshots} = $tmp_conf->{snapshots};
1880         delete $conf->{snaptime};
1881         delete $conf->{snapname};
1882         $conf->{parent} = $snapname;
1883
1884         write_config($vmid, $conf);
1885     };
1886
1887     my $unlockfn = sub {
1888         delete $conf->{lock};
1889         write_config($vmid, $conf);
1890     };
1891
1892     lock_container($vmid, 10, $updatefn);
1893
1894     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1895
1896     lock_container($vmid, 5, $unlockfn);
1897 }
1898
1899 sub template_create {
1900     my ($vmid, $conf) = @_;
1901
1902     my $storecfg = PVE::Storage::config();
1903
1904     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1905     my $volid = $rootinfo->{volume};
1906
1907     die "Template feature is not available for '$volid'\n"
1908         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1909
1910     PVE::Storage::activate_volumes($storecfg, [$volid]);
1911
1912     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1913     $rootinfo->{volume} = $template_volid;
1914     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1915
1916     write_config($vmid, $conf);
1917 }
1918
1919 sub is_template {
1920     my ($conf) = @_;
1921
1922     return 1 if defined $conf->{template} && $conf->{template} == 1;
1923 }
1924
1925 sub mountpoint_names {
1926     my ($reverse) = @_;
1927
1928     my @names = ('rootfs');
1929
1930     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1931         push @names, "mp$i";
1932     }
1933
1934     return $reverse ? reverse @names : @names;
1935 }
1936
1937 # The container might have *different* symlinks than the host. realpath/abs_path
1938 # use the actual filesystem to resolve links.
1939 sub sanitize_mountpoint {
1940     my ($mp) = @_;
1941     $mp = '/' . $mp; # we always start with a slash
1942     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1943     $mp =~ s@/\./@@g; # collapse /./
1944     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1945     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1946     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1947     return $mp;
1948 }
1949
1950 sub foreach_mountpoint_full {
1951     my ($conf, $reverse, $func) = @_;
1952
1953     foreach my $key (mountpoint_names($reverse)) {
1954         my $value = $conf->{$key};
1955         next if !defined($value);
1956         my $mountpoint = parse_ct_mountpoint($value);
1957
1958         # just to be sure: rootfs is /
1959         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1960         $mountpoint->{mp} = sanitize_mountpoint($path);
1961
1962         $path = $mountpoint->{volume};
1963         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1964
1965         &$func($key, $mountpoint);
1966     }
1967 }
1968
1969 sub foreach_mountpoint {
1970     my ($conf, $func) = @_;
1971
1972     foreach_mountpoint_full($conf, 0, $func);
1973 }
1974
1975 sub foreach_mountpoint_reverse {
1976     my ($conf, $func) = @_;
1977
1978     foreach_mountpoint_full($conf, 1, $func);
1979 }
1980
1981 sub check_ct_modify_config_perm {
1982     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1983
1984     return 1 if $authuser ne 'root@pam';
1985
1986     foreach my $opt (@$key_list) {
1987
1988         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1989             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1990         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1991             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1992         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1993             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1994         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1995                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1996             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1997         } else {
1998             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1999         }
2000     }
2001
2002     return 1;
2003 }
2004
2005 sub umount_all {
2006     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2007
2008     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2009     my $volid_list = get_vm_volumes($conf);
2010
2011     foreach_mountpoint_reverse($conf, sub {
2012         my ($ms, $mountpoint) = @_;
2013
2014         my $volid = $mountpoint->{volume};
2015         my $mount = $mountpoint->{mp};
2016
2017         return if !$volid || !$mount;
2018
2019         my $mount_path = "$rootdir/$mount";
2020         $mount_path =~ s!/+!/!g;
2021
2022         return if !PVE::ProcFSTools::is_mounted($mount_path);
2023
2024         eval {
2025             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2026         };
2027         if (my $err = $@) {
2028             if ($noerr) {
2029                 warn $err;
2030             } else {
2031                 die $err;
2032             }
2033         }
2034     });
2035 }
2036
2037 sub mount_all {
2038     my ($vmid, $storage_cfg, $conf) = @_;
2039
2040     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2041     File::Path::make_path($rootdir);
2042
2043     my $volid_list = get_vm_volumes($conf);
2044     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2045
2046     eval {
2047         foreach_mountpoint($conf, sub {
2048             my ($ms, $mountpoint) = @_;
2049
2050             my $volid = $mountpoint->{volume};
2051             my $mount = $mountpoint->{mp};
2052
2053             return if !$volid || !$mount;
2054
2055             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2056             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2057                 PVE::Storage::parse_volname($storage_cfg, $volid);
2058
2059             die "unable to mount base volume - internal error" if $isBase;
2060
2061             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2062         });
2063     };
2064     if (my $err = $@) {
2065         warn "mounting container failed - $err";
2066         umount_all($vmid, $storage_cfg, $conf, 1);
2067     }
2068
2069     return $rootdir;
2070 }
2071
2072
2073 sub mountpoint_mount_path {
2074     my ($mountpoint, $storage_cfg, $snapname) = @_;
2075
2076     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2077 }
2078
2079 my $check_mount_path = sub {
2080     my ($path) = @_;
2081     $path = File::Spec->canonpath($path);
2082     my $real = Cwd::realpath($path);
2083     if ($real ne $path) {
2084         die "mount path modified by symlink: $path != $real";
2085     }
2086 };
2087
2088 # use $rootdir = undef to just return the corresponding mount path
2089 sub mountpoint_mount {
2090     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2091
2092     my $volid = $mountpoint->{volume};
2093     my $mount = $mountpoint->{mp};
2094
2095     return if !$volid || !$mount;
2096
2097     my $mount_path;
2098
2099     if (defined($rootdir)) {
2100         $rootdir =~ s!/+$!!;
2101         $mount_path = "$rootdir/$mount";
2102         $mount_path =~ s!/+!/!g;
2103         &$check_mount_path($mount_path);
2104         File::Path::mkpath($mount_path);
2105     }
2106
2107     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2108
2109     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2110
2111     if ($storage) {
2112
2113         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2114         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2115
2116         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2117             PVE::Storage::parse_volname($storage_cfg, $volid);
2118
2119         if ($format eq 'subvol') {
2120             if ($mount_path) {
2121                 if ($snapname) {
2122                     if ($scfg->{type} eq 'zfspool') {
2123                         my $path_arg = $path;
2124                         $path_arg =~ s!^/+!!;
2125                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', '-t', 'zfs', $path_arg, $mount_path]);
2126                     } else {
2127                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2128                     }
2129                 } else {
2130                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2131                 }
2132             }
2133             return wantarray ? ($path, 0) : $path;
2134         } elsif ($format eq 'raw') {
2135             my $use_loopdev = 0;
2136             my @extra_opts;
2137             if ($scfg->{path}) {
2138                 push @extra_opts, '-o', 'loop';
2139                 $use_loopdev = 1;
2140             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2141                 # do nothing
2142             } else {
2143                 die "unsupported storage type '$scfg->{type}'\n";
2144             }
2145             if ($mount_path) {
2146                 if ($isBase || defined($snapname)) {
2147                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2148                 } else {
2149                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2150                 }
2151             }
2152             return wantarray ? ($path, $use_loopdev) : $path;
2153         } else {
2154             die "unsupported image format '$format'\n";
2155         }
2156     } elsif ($volid =~ m|^/dev/.+|) {
2157         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2158         return wantarray ? ($volid, 0) : $volid;
2159     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2160         &$check_mount_path($volid);
2161         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2162         return wantarray ? ($volid, 0) : $volid;
2163     }
2164
2165     die "unsupported storage";
2166 }
2167
2168 sub get_vm_volumes {
2169     my ($conf, $excludes) = @_;
2170
2171     my $vollist = [];
2172
2173     foreach_mountpoint($conf, sub {
2174         my ($ms, $mountpoint) = @_;
2175
2176         return if $excludes && $ms eq $excludes;
2177
2178         my $volid = $mountpoint->{volume};
2179
2180         return if !$volid || $volid =~ m|^/|;
2181
2182         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2183         return if !$sid;
2184
2185         push @$vollist, $volid;
2186     });
2187
2188     return $vollist;
2189 }
2190
2191 sub mkfs {
2192     my ($dev) = @_;
2193
2194     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2195 }
2196
2197 sub format_disk {
2198     my ($storage_cfg, $volid) = @_;
2199
2200     if ($volid =~ m!^/dev/.+!) {
2201         mkfs($volid);
2202         return;
2203     }
2204
2205     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2206
2207     die "cannot format volume '$volid' with no storage\n" if !$storage;
2208
2209     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2210
2211     my $path = PVE::Storage::path($storage_cfg, $volid);
2212
2213     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2214         PVE::Storage::parse_volname($storage_cfg, $volid);
2215
2216     die "cannot format volume '$volid' (format == $format)\n"
2217         if $format ne 'raw';
2218
2219     mkfs($path);
2220 }
2221
2222 sub destroy_disks {
2223     my ($storecfg, $vollist) = @_;
2224
2225     foreach my $volid (@$vollist) {
2226         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2227         warn $@ if $@;
2228     }
2229 }
2230
2231 sub create_disks {
2232     my ($storecfg, $vmid, $settings, $conf) = @_;
2233
2234     my $vollist = [];
2235
2236     eval {
2237         foreach_mountpoint($settings, sub {
2238             my ($ms, $mountpoint) = @_;
2239
2240             my $volid = $mountpoint->{volume};
2241             my $mp = $mountpoint->{mp};
2242
2243             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2244
2245             return if !$storage;
2246
2247             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2248                 my ($storeid, $size_gb) = ($1, $2);
2249
2250                 my $size_kb = int(${size_gb}*1024) * 1024;
2251
2252                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2253                 # fixme: use better naming ct-$vmid-disk-X.raw?
2254
2255                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2256                     if ($size_kb > 0) {
2257                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2258                                                            undef, $size_kb);
2259                         format_disk($storecfg, $volid);
2260                     } else {
2261                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2262                                                            undef, 0);
2263                     }
2264                 } elsif ($scfg->{type} eq 'zfspool') {
2265
2266                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2267                                                        undef, $size_kb);
2268                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2269
2270                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2271                     format_disk($storecfg, $volid);
2272
2273                 } elsif ($scfg->{type} eq 'rbd') {
2274
2275                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2276                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2277                     format_disk($storecfg, $volid);
2278                 } else {
2279                     die "unable to create containers on storage type '$scfg->{type}'\n";
2280                 }
2281                 push @$vollist, $volid;
2282                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2283                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2284             } else {
2285                 # use specified/existing volid
2286             }
2287         });
2288     };
2289     # free allocated images on error
2290     if (my $err = $@) {
2291         destroy_disks($storecfg, $vollist);
2292         die $err;
2293     }
2294     return $vollist;
2295 }
2296
2297 # bash completion helper
2298
2299 sub complete_os_templates {
2300     my ($cmdname, $pname, $cvalue) = @_;
2301
2302     my $cfg = PVE::Storage::config();
2303
2304     my $storeid;
2305
2306     if ($cvalue =~ m/^([^:]+):/) {
2307         $storeid = $1;
2308     }
2309
2310     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2311     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2312
2313     my $res = [];
2314     foreach my $id (keys %$data) {
2315         foreach my $item (@{$data->{$id}}) {
2316             push @$res, $item->{volid} if defined($item->{volid});
2317         }
2318     }
2319
2320     return $res;
2321 }
2322
2323 my $complete_ctid_full = sub {
2324     my ($running) = @_;
2325
2326     my $idlist = vmstatus();
2327
2328     my $active_hash = list_active_containers();
2329
2330     my $res = [];
2331
2332     foreach my $id (keys %$idlist) {
2333         my $d = $idlist->{$id};
2334         if (defined($running)) {
2335             next if $d->{template};
2336             next if $running && !$active_hash->{$id};
2337             next if !$running && $active_hash->{$id};
2338         }
2339         push @$res, $id;
2340
2341     }
2342     return $res;
2343 };
2344
2345 sub complete_ctid {
2346     return &$complete_ctid_full();
2347 }
2348
2349 sub complete_ctid_stopped {
2350     return &$complete_ctid_full(0);
2351 }
2352
2353 sub complete_ctid_running {
2354     return &$complete_ctid_full(1);
2355 }
2356
2357 1;