src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--totals', '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format_description => 'volume',
  42         description => 'Volume, device or directory to mount into the container.',
  43     },
  44     backup => {
  45         type => 'boolean',
  46         format_description => '[1|0]',
  47         description => 'Whether to include the mountpoint in backups.',
  48         optional => 1,
  49     },
  50     size => {
  51         type => 'string',
  52         format => 'disk-size',
  53         format_description => 'DiskSize',
  54         description => 'Volume size (read only value).',
  55         optional => 1,
  56     },
  57 };
  58
  59 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  60     type => 'string', format => $rootfs_desc,
  61     description => "Use volume as container root.",
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 my $confdesc = {
  72     lock => {
  73         optional => 1,
  74         type => 'string',
  75         description => "Lock/unlock the VM.",
  76         enum => [qw(migrate backup snapshot rollback)],
  77     },
  78     onboot => {
  79         optional => 1,
  80         type => 'boolean',
  81         description => "Specifies whether a VM will be started during system bootup.",
  82         default => 0,
  83     },
  84     startup => get_standard_option('pve-startup-order'),
  85     template => {
  86         optional => 1,
  87         type => 'boolean',
  88         description => "Enable/disable Template.",
  89         default => 0,
  90     },
  91     arch => {
  92         optional => 1,
  93         type => 'string',
  94         enum => ['amd64', 'i386'],
  95         description => "OS architecture type.",
  96         default => 'amd64',
  97     },
  98     ostype => {
  99         optional => 1,
 100         type => 'string',
 101         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
 102         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 103     },
 104     console => {
 105         optional => 1,
 106         type => 'boolean',
 107         description => "Attach a console device (/dev/console) to the container.",
 108         default => 1,
 109     },
 110     tty => {
 111         optional => 1,
 112         type => 'integer',
 113         description => "Specify the number of tty available to the container",
 114         minimum => 0,
 115         maximum => 6,
 116         default => 2,
 117     },
 118     cpulimit => {
 119         optional => 1,
 120         type => 'number',
 121         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 122         minimum => 0,
 123         maximum => 128,
 124         default => 0,
 125     },
 126     cpuunits => {
 127         optional => 1,
 128         type => 'integer',
 129         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 130         minimum => 0,
 131         maximum => 500000,
 132         default => 1024,
 133     },
 134     memory => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Amount of RAM for the VM in MB.",
 138         minimum => 16,
 139         default => 512,
 140     },
 141     swap => {
 142         optional => 1,
 143         type => 'integer',
 144         description => "Amount of SWAP for the VM in MB.",
 145         minimum => 0,
 146         default => 512,
 147     },
 148     hostname => {
 149         optional => 1,
 150         description => "Set a host name for the container.",
 151         type => 'string', format => 'dns-name',
 152         maxLength => 255,
 153     },
 154     description => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Container description. Only used on the configuration web interface.",
 158     },
 159     searchdomain => {
 160         optional => 1,
 161         type => 'string', format => 'dns-name-list',
 162         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 163     },
 164     nameserver => {
 165         optional => 1,
 166         type => 'string', format => 'address-list',
 167         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 168     },
 169     rootfs => get_standard_option('pve-ct-rootfs'),
 170     parent => {
 171         optional => 1,
 172         type => 'string', format => 'pve-configid',
 173         maxLength => 40,
 174         description => "Parent snapshot name. This is used internally, and should not be modified.",
 175     },
 176     snaptime => {
 177         optional => 1,
 178         description => "Timestamp for snapshots.",
 179         type => 'integer',
 180         minimum => 0,
 181     },
 182     cmode => {
 183         optional => 1,
 184         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 185         type => 'string',
 186         enum => ['shell', 'console', 'tty'],
 187         default => 'tty',
 188     },
 189     protection => {
 190         optional => 1,
 191         type => 'boolean',
 192         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 193         default => 0,
 194     },
 195 };
 196
 197 my $valid_lxc_conf_keys = {
 198     'lxc.include' => 1,
 199     'lxc.arch' => 1,
 200     'lxc.utsname' => 1,
 201     'lxc.haltsignal' => 1,
 202     'lxc.rebootsignal' => 1,
 203     'lxc.stopsignal' => 1,
 204     'lxc.init_cmd' => 1,
 205     'lxc.network.type' => 1,
 206     'lxc.network.flags' => 1,
 207     'lxc.network.link' => 1,
 208     'lxc.network.mtu' => 1,
 209     'lxc.network.name' => 1,
 210     'lxc.network.hwaddr' => 1,
 211     'lxc.network.ipv4' => 1,
 212     'lxc.network.ipv4.gateway' => 1,
 213     'lxc.network.ipv6' => 1,
 214     'lxc.network.ipv6.gateway' => 1,
 215     'lxc.network.script.up' => 1,
 216     'lxc.network.script.down' => 1,
 217     'lxc.pts' => 1,
 218     'lxc.console.logfile' => 1,
 219     'lxc.console' => 1,
 220     'lxc.tty' => 1,
 221     'lxc.devttydir' => 1,
 222     'lxc.hook.autodev' => 1,
 223     'lxc.autodev' => 1,
 224     'lxc.kmsg' => 1,
 225     'lxc.mount' => 1,
 226     'lxc.mount.entry' => 1,
 227     'lxc.mount.auto' => 1,
 228     'lxc.rootfs' => 1,
 229     'lxc.rootfs.mount' => 1,
 230     'lxc.rootfs.options' => 1,
 231     # lxc.cgroup.*
 232     'lxc.cap.drop' => 1,
 233     'lxc.cap.keep' => 1,
 234     'lxc.aa_profile' => 1,
 235     'lxc.aa_allow_incomplete' => 1,
 236     'lxc.se_context' => 1,
 237     'lxc.seccomp' => 1,
 238     'lxc.id_map' => 1,
 239     'lxc.hook.pre-start' => 1,
 240     'lxc.hook.pre-mount' => 1,
 241     'lxc.hook.mount' => 1,
 242     'lxc.hook.start' => 1,
 243     'lxc.hook.stop' => 1,
 244     'lxc.hook.post-stop' => 1,
 245     'lxc.hook.clone' => 1,
 246     'lxc.hook.destroy' => 1,
 247     'lxc.loglevel' => 1,
 248     'lxc.logfile' => 1,
 249     'lxc.start.auto' => 1,
 250     'lxc.start.delay' => 1,
 251     'lxc.start.order' => 1,
 252     'lxc.group' => 1,
 253     'lxc.environment' => 1,
 254     'lxc.' => 1,
 255     'lxc.' => 1,
 256     'lxc.' => 1,
 257     'lxc.' => 1,
 258 };
 259
 260 my $netconf_desc = {
 261     type => {
 262         type => 'string',
 263         optional => 1,
 264         description => "Network interface type.",
 265         enum => [qw(veth)],
 266     },
 267     name => {
 268         type => 'string',
 269         format_description => 'String',
 270         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 271         pattern => '[-_.\w\d]+',
 272     },
 273     bridge => {
 274         type => 'string',
 275         format_description => 'vmbr<Number>',
 276         description => 'Bridge to attach the network device to.',
 277         pattern => '[-_.\w\d]+',
 278         optional => 1,
 279     },
 280     hwaddr => {
 281         type => 'string',
 282         format_description => 'MAC',
 283         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 284         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 285         optional => 1,
 286     },
 287     mtu => {
 288         type => 'integer',
 289         format_description => 'Number',
 290         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 291         minimum => 64, # minimum ethernet frame is 64 bytes
 292         optional => 1,
 293     },
 294     ip => {
 295         type => 'string',
 296         format => 'pve-ipv4-config',
 297         format_description => 'IPv4Format/CIDR',
 298         description => 'IPv4 address in CIDR format.',
 299         optional => 1,
 300     },
 301     gw => {
 302         type => 'string',
 303         format => 'ipv4',
 304         format_description => 'GatewayIPv4',
 305         description => 'Default gateway for IPv4 traffic.',
 306         optional => 1,
 307     },
 308     ip6 => {
 309         type => 'string',
 310         format => 'pve-ipv6-config',
 311         format_description => 'IPv6Format/CIDR',
 312         description => 'IPv6 address in CIDR format.',
 313         optional => 1,
 314     },
 315     gw6 => {
 316         type => 'string',
 317         format => 'ipv6',
 318         format_description => 'GatewayIPv6',
 319         description => 'Default gateway for IPv6 traffic.',
 320         optional => 1,
 321     },
 322     firewall => {
 323         type => 'boolean',
 324         format_description => '[1|0]',
 325         description => "Controls whether this interface's firewall rules should be used.",
 326         optional => 1,
 327     },
 328     tag => {
 329         type => 'integer',
 330         format_description => 'VlanNo',
 331         minimum => '2',
 332         maximum => '4094',
 333         description => "VLAN tag foro this interface.",
 334         optional => 1,
 335     },
 336 };
 337 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 338
 339 my $MAX_LXC_NETWORKS = 10;
 340 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 341     $confdesc->{"net$i"} = {
 342         optional => 1,
 343         type => 'string', format => $netconf_desc,
 344         description => "Specifies network interfaces for the container.",
 345     };
 346 }
 347
 348 my $mp_desc = {
 349     %$rootfs_desc,
 350     mp => {
 351         type => 'string',
 352         format_description => 'Path',
 353         description => 'Path to the mountpoint as seen from inside the container.',
 354         optional => 1,
 355     },
 356 };
 357 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 358
 359 my $unuseddesc = {
 360     optional => 1,
 361     type => 'string', format => 'pve-volume-id',
 362     description => "Reference to unused volumes.",
 363 };
 364
 365 my $MAX_MOUNT_POINTS = 10;
 366 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 367     $confdesc->{"mp$i"} = {
 368         optional => 1,
 369         type => 'string', format => $mp_desc,
 370         description => "Use volume as container mount point (experimental feature).",
 371         optional => 1,
 372     };
 373 }
 374
 375 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 376 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 377     $confdesc->{"unused$i"} = $unuseddesc;
 378 }
 379
 380 sub write_pct_config {
 381     my ($filename, $conf) = @_;
 382
 383     delete $conf->{snapstate}; # just to be sure
 384
 385     my $generate_raw_config = sub {
 386         my ($conf) = @_;
 387
 388         my $raw = '';
 389
 390         # add description as comment to top of file
 391         my $descr = $conf->{description} || '';
 392         foreach my $cl (split(/\n/, $descr)) {
 393             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 394         }
 395
 396         foreach my $key (sort keys %$conf) {
 397             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 398                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 399             my $value = $conf->{$key};
 400             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 401             $raw .= "$key: $value\n";
 402         }
 403
 404         if (my $lxcconf = $conf->{lxc}) {
 405             foreach my $entry (@$lxcconf) {
 406                 my ($k, $v) = @$entry;
 407                 $raw .= "$k: $v\n";
 408             }
 409         }
 410
 411         return $raw;
 412     };
 413
 414     my $raw = &$generate_raw_config($conf);
 415
 416     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 417         $raw .= "\n[$snapname]\n";
 418         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 419     }
 420
 421     return $raw;
 422 }
 423
 424 sub check_type {
 425     my ($key, $value) = @_;
 426
 427     die "unknown setting '$key'\n" if !$confdesc->{$key};
 428
 429     my $type = $confdesc->{$key}->{type};
 430
 431     if (!defined($value)) {
 432         die "got undefined value\n";
 433     }
 434
 435     if ($value =~ m/[\n\r]/) {
 436         die "property contains a line feed\n";
 437     }
 438
 439     if ($type eq 'boolean') {
 440         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 441         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 442         die "type check ('boolean') failed - got '$value'\n";
 443     } elsif ($type eq 'integer') {
 444         return int($1) if $value =~ m/^(\d+)$/;
 445         die "type check ('integer') failed - got '$value'\n";
 446     } elsif ($type eq 'number') {
 447         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 448         die "type check ('number') failed - got '$value'\n";
 449     } elsif ($type eq 'string') {
 450         if (my $fmt = $confdesc->{$key}->{format}) {
 451             PVE::JSONSchema::check_format($fmt, $value);
 452             return $value;
 453         }
 454         return $value;
 455     } else {
 456         die "internal error"
 457     }
 458 }
 459
 460 sub parse_pct_config {
 461     my ($filename, $raw) = @_;
 462
 463     return undef if !defined($raw);
 464
 465     my $res = {
 466         digest => Digest::SHA::sha1_hex($raw),
 467         snapshots => {},
 468     };
 469
 470     $filename =~ m|/lxc/(\d+).conf$|
 471         || die "got strange filename '$filename'";
 472
 473     my $vmid = $1;
 474
 475     my $conf = $res;
 476     my $descr = '';
 477     my $section = '';
 478
 479     my @lines = split(/\n/, $raw);
 480     foreach my $line (@lines) {
 481         next if $line =~ m/^\s*$/;
 482
 483         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 484             $section = $1;
 485             $conf->{description} = $descr if $descr;
 486             $descr = '';
 487             $conf = $res->{snapshots}->{$section} = {};
 488             next;
 489         }
 490
 491         if ($line =~ m/^\#(.*)\s*$/) {
 492             $descr .= PVE::Tools::decode_text($1) . "\n";
 493             next;
 494         }
 495
 496         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 497             my $key = $1;
 498             my $value = $3;
 499             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 500                 push @{$conf->{lxc}}, [$key, $value];
 501             } else {
 502                 warn "vm $vmid - unable to parse config: $line\n";
 503             }
 504         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 505             $descr .= PVE::Tools::decode_text($2);
 506         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 507             $conf->{snapstate} = $1;
 508         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 509             my $key = $1;
 510             my $value = $2;
 511             eval { $value = check_type($key, $value); };
 512             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 513             $conf->{$key} = $value;
 514         } else {
 515             warn "vm $vmid - unable to parse config: $line\n";
 516         }
 517     }
 518
 519     $conf->{description} = $descr if $descr;
 520
 521     delete $res->{snapstate}; # just to be sure
 522
 523     return $res;
 524 }
 525
 526 sub config_list {
 527     my $vmlist = PVE::Cluster::get_vmlist();
 528     my $res = {};
 529     return $res if !$vmlist || !$vmlist->{ids};
 530     my $ids = $vmlist->{ids};
 531
 532     foreach my $vmid (keys %$ids) {
 533         next if !$vmid; # skip CT0
 534         my $d = $ids->{$vmid};
 535         next if !$d->{node} || $d->{node} ne $nodename;
 536         next if !$d->{type} || $d->{type} ne 'lxc';
 537         $res->{$vmid}->{type} = 'lxc';
 538     }
 539     return $res;
 540 }
 541
 542 sub cfs_config_path {
 543     my ($vmid, $node) = @_;
 544
 545     $node = $nodename if !$node;
 546     return "nodes/$node/lxc/$vmid.conf";
 547 }
 548
 549 sub config_file {
 550     my ($vmid, $node) = @_;
 551
 552     my $cfspath = cfs_config_path($vmid, $node);
 553     return "/etc/pve/$cfspath";
 554 }
 555
 556 sub load_config {
 557     my ($vmid, $node) = @_;
 558
 559     $node = $nodename if !$node;
 560     my $cfspath = cfs_config_path($vmid, $node);
 561
 562     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 563     die "container $vmid does not exists\n" if !defined($conf);
 564
 565     return $conf;
 566 }
 567
 568 sub create_config {
 569     my ($vmid, $conf) = @_;
 570
 571     my $dir = "/etc/pve/nodes/$nodename/lxc";
 572     mkdir $dir;
 573
 574     write_config($vmid, $conf);
 575 }
 576
 577 sub destroy_config {
 578     my ($vmid) = @_;
 579
 580     unlink config_file($vmid, $nodename);
 581 }
 582
 583 sub write_config {
 584     my ($vmid, $conf) = @_;
 585
 586     my $cfspath = cfs_config_path($vmid);
 587
 588     PVE::Cluster::cfs_write_file($cfspath, $conf);
 589 }
 590
 591 # flock: we use one file handle per process, so lock file
 592 # can be called multiple times and succeeds for the same process.
 593
 594 my $lock_handles =  {};
 595 my $lockdir = "/run/lock/lxc";
 596
 597 sub lock_filename {
 598     my ($vmid) = @_;
 599
 600     return "$lockdir/pve-config-${vmid}.lock";
 601 }
 602
 603 sub lock_aquire {
 604     my ($vmid, $timeout) = @_;
 605
 606     $timeout = 10 if !$timeout;
 607     my $mode = LOCK_EX;
 608
 609     my $filename = lock_filename($vmid);
 610
 611     mkdir $lockdir if !-d $lockdir;
 612
 613     my $lock_func = sub {
 614         if (!$lock_handles->{$$}->{$filename}) {
 615             my $fh = new IO::File(">>$filename") ||
 616                 die "can't open file - $!\n";
 617             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 618         }
 619
 620         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 621             print STDERR "trying to aquire lock...";
 622             my $success;
 623             while(1) {
 624                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 625                 # try again on EINTR (see bug #273)
 626                 if ($success || ($! != EINTR)) {
 627                     last;
 628                 }
 629             }
 630             if (!$success) {
 631                 print STDERR " failed\n";
 632                 die "can't aquire lock - $!\n";
 633             }
 634
 635             print STDERR " OK\n";
 636         }
 637
 638         $lock_handles->{$$}->{$filename}->{refcount}++;
 639     };
 640
 641     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 642     my $err = $@;
 643     if ($err) {
 644         die "can't lock file '$filename' - $err";
 645     }
 646 }
 647
 648 sub lock_release {
 649     my ($vmid) = @_;
 650
 651     my $filename = lock_filename($vmid);
 652
 653     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 654         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 655         if ($refcount <= 0) {
 656             $lock_handles->{$$}->{$filename} = undef;
 657             close ($fh);
 658         }
 659     }
 660 }
 661
 662 sub lock_container {
 663     my ($vmid, $timeout, $code, @param) = @_;
 664
 665     my $res;
 666
 667     lock_aquire($vmid, $timeout);
 668     eval { $res = &$code(@param) };
 669     my $err = $@;
 670     lock_release($vmid);
 671
 672     die $err if $err;
 673
 674     return $res;
 675 }
 676
 677 sub option_exists {
 678     my ($name) = @_;
 679
 680     return defined($confdesc->{$name});
 681 }
 682
 683 # add JSON properties for create and set function
 684 sub json_config_properties {
 685     my $prop = shift;
 686
 687     foreach my $opt (keys %$confdesc) {
 688         next if $opt eq 'parent' || $opt eq 'snaptime';
 689         next if $prop->{$opt};
 690         $prop->{$opt} = $confdesc->{$opt};
 691     }
 692
 693     return $prop;
 694 }
 695
 696 sub json_config_properties_no_rootfs {
 697     my $prop = shift;
 698
 699     foreach my $opt (keys %$confdesc) {
 700         next if $prop->{$opt};
 701         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 702         $prop->{$opt} = $confdesc->{$opt};
 703     }
 704
 705     return $prop;
 706 }
 707
 708 # container status helpers
 709
 710 sub list_active_containers {
 711
 712     my $filename = "/proc/net/unix";
 713
 714     # similar test is used by lcxcontainers.c: list_active_containers
 715     my $res = {};
 716
 717     my $fh = IO::File->new ($filename, "r");
 718     return $res if !$fh;
 719
 720     while (defined(my $line = <$fh>)) {
 721         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 722             my $path = $1;
 723             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 724                 $res->{$1} = 1;
 725             }
 726         }
 727     }
 728
 729     close($fh);
 730
 731     return $res;
 732 }
 733
 734 # warning: this is slow
 735 sub check_running {
 736     my ($vmid) = @_;
 737
 738     my $active_hash = list_active_containers();
 739
 740     return 1 if defined($active_hash->{$vmid});
 741
 742     return undef;
 743 }
 744
 745 sub get_container_disk_usage {
 746     my ($vmid, $pid) = @_;
 747
 748     return PVE::Tools::df("/proc/$pid/root/", 1);
 749 }
 750
 751 my $last_proc_vmid_stat;
 752
 753 my $parse_cpuacct_stat = sub {
 754     my ($vmid) = @_;
 755
 756     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 757
 758     my $stat = {};
 759
 760     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 761
 762         $stat->{utime} = $1;
 763         $stat->{stime} = $2;
 764
 765     }
 766
 767     return $stat;
 768 };
 769
 770 sub vmstatus {
 771     my ($opt_vmid) = @_;
 772
 773     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 774
 775     my $active_hash = list_active_containers();
 776
 777     my $cpucount = $cpuinfo->{cpus} || 1;
 778
 779     my $cdtime = gettimeofday;
 780
 781     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 782
 783     foreach my $vmid (keys %$list) {
 784         my $d = $list->{$vmid};
 785
 786         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 787         warn $@ if $@; # ignore errors (consider them stopped)
 788
 789         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 790
 791         my $cfspath = cfs_config_path($vmid);
 792         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 793
 794         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 795         $d->{name} =~ s/[\s]//g;
 796
 797         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 798
 799         if ($d->{pid}) {
 800             my $res = get_container_disk_usage($vmid, $d->{pid});
 801             $d->{disk} = $res->{used};
 802             $d->{maxdisk} = $res->{total};
 803         } else {
 804             $d->{disk} = 0;
 805             # use 4GB by default ??
 806             if (my $rootfs = $conf->{rootfs}) {
 807                 my $rootinfo = parse_ct_mountpoint($rootfs);
 808                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 809             } else {
 810                 $d->{maxdisk} = 4*1024*1024*1024;
 811             }
 812         }
 813
 814         $d->{mem} = 0;
 815         $d->{swap} = 0;
 816         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 817         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 818
 819         $d->{uptime} = 0;
 820         $d->{cpu} = 0;
 821
 822         $d->{netout} = 0;
 823         $d->{netin} = 0;
 824
 825         $d->{diskread} = 0;
 826         $d->{diskwrite} = 0;
 827
 828         $d->{template} = is_template($conf);
 829     }
 830
 831     foreach my $vmid (keys %$list) {
 832         my $d = $list->{$vmid};
 833         my $pid = $d->{pid};
 834
 835         next if !$pid; # skip stopped CTs
 836
 837         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 838         $d->{uptime} = time - $ctime; # the method lxcfs uses
 839
 840         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 841         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 842
 843         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 844         my @bytes = split(/\n/, $blkio_bytes);
 845         foreach my $byte (@bytes) {
 846             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 847                 $d->{diskread} = $2 if $key eq 'Read';
 848                 $d->{diskwrite} = $2 if $key eq 'Write';
 849             }
 850         }
 851
 852         my $pstat = &$parse_cpuacct_stat($vmid);
 853
 854         my $used = $pstat->{utime} + $pstat->{stime};
 855
 856         my $old = $last_proc_vmid_stat->{$vmid};
 857         if (!$old) {
 858             $last_proc_vmid_stat->{$vmid} = {
 859                 time => $cdtime,
 860                 used => $used,
 861                 cpu => 0,
 862             };
 863             next;
 864         }
 865
 866         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 867
 868         if ($dtime > 1000) {
 869             my $dutime = $used -  $old->{used};
 870
 871             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 872             $last_proc_vmid_stat->{$vmid} = {
 873                 time => $cdtime,
 874                 used => $used,
 875                 cpu => $d->{cpu},
 876             };
 877         } else {
 878             $d->{cpu} = $old->{cpu};
 879         }
 880     }
 881
 882     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 883
 884     foreach my $dev (keys %$netdev) {
 885         next if $dev !~ m/^veth([1-9]\d*)i/;
 886         my $vmid = $1;
 887         my $d = $list->{$vmid};
 888
 889         next if !$d;
 890
 891         $d->{netout} += $netdev->{$dev}->{receive};
 892         $d->{netin} += $netdev->{$dev}->{transmit};
 893
 894     }
 895
 896     return $list;
 897 }
 898
 899 sub parse_ct_mountpoint {
 900     my ($data, $noerr) = @_;
 901
 902     $data //= '';
 903
 904     my $res;
 905     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 906     if ($@) {
 907         return undef if $noerr;
 908         die $@;
 909     }
 910
 911     if (defined(my $size = $res->{size})) {
 912         $size = PVE::JSONSchema::parse_size($size);
 913         if (!defined($size)) {
 914             return undef if $noerr;
 915             die "invalid size: $size\n";
 916         }
 917         $res->{size} = $size;
 918     }
 919
 920     return $res;
 921 }
 922
 923 sub print_ct_mountpoint {
 924     my ($info, $nomp) = @_;
 925     my $skip = $nomp ? ['mp'] : [];
 926     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 927 }
 928
 929 sub print_lxc_network {
 930     my $net = shift;
 931     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 932 }
 933
 934 sub parse_lxc_network {
 935     my ($data) = @_;
 936
 937     my $res = {};
 938
 939     return $res if !$data;
 940
 941     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 942
 943     $res->{type} = 'veth';
 944     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 945
 946     return $res;
 947 }
 948
 949 sub read_cgroup_value {
 950     my ($group, $vmid, $name, $full) = @_;
 951
 952     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 953
 954     return PVE::Tools::file_get_contents($path) if $full;
 955
 956     return PVE::Tools::file_read_firstline($path);
 957 }
 958
 959 sub write_cgroup_value {
 960    my ($group, $vmid, $name, $value) = @_;
 961
 962    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 963    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 964
 965 }
 966
 967 sub find_lxc_console_pids {
 968
 969     my $res = {};
 970
 971     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 972         my ($pid) = @_;
 973
 974         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 975         return if !$cmdline;
 976
 977         my @args = split(/\0/, $cmdline);
 978
 979         # serach for lxc-console -n <vmid>
 980         return if scalar(@args) != 3;
 981         return if $args[1] ne '-n';
 982         return if $args[2] !~ m/^\d+$/;
 983         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 984
 985         my $vmid = $args[2];
 986
 987         push @{$res->{$vmid}}, $pid;
 988     });
 989
 990     return $res;
 991 }
 992
 993 sub find_lxc_pid {
 994     my ($vmid) = @_;
 995
 996     my $pid = undef;
 997     my $parser = sub {
 998         my $line = shift;
 999         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1000     };
1001     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1002
1003     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1004
1005     return $pid;
1006 }
1007
1008 # Note: we cannot use Net:IP, because that only allows strict
1009 # CIDR networks
1010 sub parse_ipv4_cidr {
1011     my ($cidr, $noerr) = @_;
1012
1013     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1014         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1015     }
1016
1017     return undef if $noerr;
1018
1019     die "unable to parse ipv4 address/mask\n";
1020 }
1021
1022 sub check_lock {
1023     my ($conf) = @_;
1024
1025     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1026 }
1027
1028 sub check_protection {
1029     my ($vm_conf, $err_msg) = @_;
1030
1031     if ($vm_conf->{protection}) {
1032         die "$err_msg - protection mode enabled\n";
1033     }
1034 }
1035
1036 sub update_lxc_config {
1037     my ($storage_cfg, $vmid, $conf) = @_;
1038
1039     my $dir = "/var/lib/lxc/$vmid";
1040
1041     if ($conf->{template}) {
1042
1043         unlink "$dir/config";
1044
1045         return;
1046     }
1047
1048     my $raw = '';
1049
1050     die "missing 'arch' - internal error" if !$conf->{arch};
1051     $raw .= "lxc.arch = $conf->{arch}\n";
1052
1053     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1054     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1055         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1056     } else {
1057         die "implement me";
1058     }
1059
1060     if (!has_dev_console($conf)) {
1061         $raw .= "lxc.console = none\n";
1062         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1063     }
1064
1065     my $ttycount = get_tty_count($conf);
1066     $raw .= "lxc.tty = $ttycount\n";
1067
1068     # some init scripts expects a linux terminal (turnkey).
1069     $raw .= "lxc.environment = TERM=linux\n";
1070
1071     my $utsname = $conf->{hostname} || "CT$vmid";
1072     $raw .= "lxc.utsname = $utsname\n";
1073
1074     my $memory = $conf->{memory} || 512;
1075     my $swap = $conf->{swap} // 0;
1076
1077     my $lxcmem = int($memory*1024*1024);
1078     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1079
1080     my $lxcswap = int(($memory + $swap)*1024*1024);
1081     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1082
1083     if (my $cpulimit = $conf->{cpulimit}) {
1084         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1085         my $value = int(100000*$cpulimit);
1086         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1087     }
1088
1089     my $shares = $conf->{cpuunits} || 1024;
1090     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1091
1092     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1093     $mountpoint->{mp} = '/';
1094
1095     $raw .= "lxc.rootfs = $dir/rootfs\n";
1096
1097     my $netcount = 0;
1098     foreach my $k (keys %$conf) {
1099         next if $k !~ m/^net(\d+)$/;
1100         my $ind = $1;
1101         my $d = parse_lxc_network($conf->{$k});
1102         $netcount++;
1103         $raw .= "lxc.network.type = veth\n";
1104         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1105         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1106         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1107         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1108     }
1109
1110     if (my $lxcconf = $conf->{lxc}) {
1111         foreach my $entry (@$lxcconf) {
1112             my ($k, $v) = @$entry;
1113             $netcount++ if $k eq 'lxc.network.type';
1114             $raw .= "$k = $v\n";
1115         }
1116     }
1117
1118     $raw .= "lxc.network.type = empty\n" if !$netcount;
1119
1120     File::Path::mkpath("$dir/rootfs");
1121
1122     PVE::Tools::file_set_contents("$dir/config", $raw);
1123 }
1124
1125 # verify and cleanup nameserver list (replace \0 with ' ')
1126 sub verify_nameserver_list {
1127     my ($nameserver_list) = @_;
1128
1129     my @list = ();
1130     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1131         PVE::JSONSchema::pve_verify_ip($server);
1132         push @list, $server;
1133     }
1134
1135     return join(' ', @list);
1136 }
1137
1138 sub verify_searchdomain_list {
1139     my ($searchdomain_list) = @_;
1140
1141     my @list = ();
1142     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1143         # todo: should we add checks for valid dns domains?
1144         push @list, $server;
1145     }
1146
1147     return join(' ', @list);
1148 }
1149
1150 sub add_unused_volume {
1151     my ($config, $volid) = @_;
1152
1153     # skip bind mounts and block devices
1154     return if $volid =~ m|^/|;
1155
1156     my $key;
1157     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1158         my $test = "unused$ind";
1159         if (my $vid = $config->{$test}) {
1160             return if $vid eq $volid; # do not add duplicates
1161         } else {
1162             $key = $test;
1163         }
1164     }
1165
1166     die "To many unused volume - please delete them first.\n" if !$key;
1167
1168     $config->{$key} = $volid;
1169
1170     return $key;
1171 }
1172
1173 sub update_pct_config {
1174     my ($vmid, $conf, $running, $param, $delete) = @_;
1175
1176     my @nohotplug;
1177
1178     my $new_disks = 0;
1179     my @deleted_volumes;
1180
1181     my $rootdir;
1182     if ($running) {
1183         my $pid = find_lxc_pid($vmid);
1184         $rootdir = "/proc/$pid/root";
1185     }
1186
1187     my $hotplug_error = sub {
1188         if ($running) {
1189             push @nohotplug, @_;
1190             return 1;
1191         } else {
1192             return 0;
1193         }
1194     };
1195
1196     if (defined($delete)) {
1197         foreach my $opt (@$delete) {
1198             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1199                 die "unable to delete required option '$opt'\n";
1200             } elsif ($opt eq 'swap') {
1201                 delete $conf->{$opt};
1202                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1203             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1204                 delete $conf->{$opt};
1205             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1206                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1207                 next if $hotplug_error->($opt);
1208                 delete $conf->{$opt};
1209             } elsif ($opt =~ m/^net(\d)$/) {
1210                 delete $conf->{$opt};
1211                 next if !$running;
1212                 my $netid = $1;
1213                 PVE::Network::veth_delete("veth${vmid}i$netid");
1214             } elsif ($opt eq 'protection') {
1215                 delete $conf->{$opt};
1216             } elsif ($opt =~ m/^unused(\d+)$/) {
1217                 next if $hotplug_error->($opt);
1218                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1219                 push @deleted_volumes, $conf->{$opt};
1220                 delete $conf->{$opt};
1221             } elsif ($opt =~ m/^mp(\d+)$/) {
1222                 next if $hotplug_error->($opt);
1223                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1224                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1225                 add_unused_volume($conf, $mountpoint->{volume});
1226                 delete $conf->{$opt};
1227             } else {
1228                 die "implement me"
1229             }
1230             write_config($vmid, $conf) if $running;
1231         }
1232     }
1233
1234     # There's no separate swap size to configure, there's memory and "total"
1235     # memory (iow. memory+swap). This means we have to change them together.
1236     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1237     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1238     if (defined($wanted_memory) || defined($wanted_swap)) {
1239
1240         $wanted_memory //= ($conf->{memory} || 512);
1241         $wanted_swap //=  ($conf->{swap} || 0);
1242
1243         my $total = $wanted_memory + $wanted_swap;
1244         if ($running) {
1245             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1246             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1247         }
1248         $conf->{memory} = $wanted_memory;
1249         $conf->{swap} = $wanted_swap;
1250
1251         write_config($vmid, $conf) if $running;
1252     }
1253
1254     foreach my $opt (keys %$param) {
1255         my $value = $param->{$opt};
1256         if ($opt eq 'hostname') {
1257             $conf->{$opt} = $value;
1258         } elsif ($opt eq 'onboot') {
1259             $conf->{$opt} = $value ? 1 : 0;
1260         } elsif ($opt eq 'startup') {
1261             $conf->{$opt} = $value;
1262         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1263             next if $hotplug_error->($opt);
1264             $conf->{$opt} = $value;
1265         } elsif ($opt eq 'nameserver') {
1266             next if $hotplug_error->($opt);
1267             my $list = verify_nameserver_list($value);
1268             $conf->{$opt} = $list;
1269         } elsif ($opt eq 'searchdomain') {
1270             next if $hotplug_error->($opt);
1271             my $list = verify_searchdomain_list($value);
1272             $conf->{$opt} = $list;
1273         } elsif ($opt eq 'cpulimit') {
1274             next if $hotplug_error->($opt); # FIXME: hotplug
1275             $conf->{$opt} = $value;
1276         } elsif ($opt eq 'cpuunits') {
1277             $conf->{$opt} = $value;
1278             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1279         } elsif ($opt eq 'description') {
1280             $conf->{$opt} = PVE::Tools::encode_text($value);
1281         } elsif ($opt =~ m/^net(\d+)$/) {
1282             my $netid = $1;
1283             my $net = parse_lxc_network($value);
1284             if (!$running) {
1285                 $conf->{$opt} = print_lxc_network($net);
1286             } else {
1287                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1288             }
1289         } elsif ($opt eq 'protection') {
1290             $conf->{$opt} = $value ? 1 : 0;
1291         } elsif ($opt =~ m/^mp(\d+)$/) {
1292             next if $hotplug_error->($opt);
1293             check_protection($conf, "can't update CT $vmid drive '$opt'");
1294             $conf->{$opt} = $value;
1295             $new_disks = 1;
1296         } elsif ($opt eq 'rootfs') {
1297             check_protection($conf, "can't update CT $vmid drive '$opt'");
1298             die "implement me: $opt";
1299         } else {
1300             die "implement me: $opt";
1301         }
1302         write_config($vmid, $conf) if $running;
1303     }
1304
1305     if (@deleted_volumes) {
1306         my $storage_cfg = PVE::Storage::config();
1307         foreach my $volume (@deleted_volumes) {
1308             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1309         }
1310     }
1311
1312     if ($new_disks) {
1313         my $storage_cfg = PVE::Storage::config();
1314         create_disks($storage_cfg, $vmid, $conf, $conf);
1315     }
1316
1317     # This should be the last thing we do here
1318     if ($running && scalar(@nohotplug)) {
1319         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1320     }
1321 }
1322
1323 sub has_dev_console {
1324     my ($conf) = @_;
1325
1326     return !(defined($conf->{console}) && !$conf->{console});
1327 }
1328
1329 sub get_tty_count {
1330     my ($conf) = @_;
1331
1332     return $conf->{tty} // $confdesc->{tty}->{default};
1333 }
1334
1335 sub get_cmode {
1336     my ($conf) = @_;
1337
1338     return $conf->{cmode} // $confdesc->{cmode}->{default};
1339 }
1340
1341 sub get_console_command {
1342     my ($vmid, $conf) = @_;
1343
1344     my $cmode = get_cmode($conf);
1345
1346     if ($cmode eq 'console') {
1347         return ['lxc-console', '-n',  $vmid, '-t', 0];
1348     } elsif ($cmode eq 'tty') {
1349         return ['lxc-console', '-n',  $vmid];
1350     } elsif ($cmode eq 'shell') {
1351         return ['lxc-attach', '--clear-env', '-n', $vmid];
1352     } else {
1353         die "internal error";
1354     }
1355 }
1356
1357 sub get_primary_ips {
1358     my ($conf) = @_;
1359
1360     # return data from net0
1361
1362     return undef if !defined($conf->{net0});
1363     my $net = parse_lxc_network($conf->{net0});
1364
1365     my $ipv4 = $net->{ip};
1366     if ($ipv4) {
1367         if ($ipv4 =~ /^(dhcp|manual)$/) {
1368             $ipv4 = undef
1369         } else {
1370             $ipv4 =~ s!/\d+$!!;
1371         }
1372     }
1373     my $ipv6 = $net->{ip6};
1374     if ($ipv6) {
1375         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1376             $ipv6 = undef;
1377         } else {
1378             $ipv6 =~ s!/\d+$!!;
1379         }
1380     }
1381
1382     return ($ipv4, $ipv6);
1383 }
1384
1385 sub delete_mountpoint_volume {
1386     my ($storage_cfg, $vmid, $volume) = @_;
1387
1388     # skip bind mounts and block devices
1389     if ($volume =~ m|^/|) {
1390             return;
1391     }
1392
1393     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1394     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1395 }
1396
1397 sub destroy_lxc_container {
1398     my ($storage_cfg, $vmid, $conf) = @_;
1399
1400     foreach_mountpoint($conf, sub {
1401         my ($ms, $mountpoint) = @_;
1402         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1403     });
1404
1405     rmdir "/var/lib/lxc/$vmid/rootfs";
1406     unlink "/var/lib/lxc/$vmid/config";
1407     rmdir "/var/lib/lxc/$vmid";
1408     destroy_config($vmid);
1409
1410     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1411     #PVE::Tools::run_command($cmd);
1412 }
1413
1414 sub vm_stop_cleanup {
1415     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1416
1417     eval {
1418         if (!$keepActive) {
1419
1420             my $vollist = get_vm_volumes($conf);
1421             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1422         }
1423     };
1424     warn $@ if $@; # avoid errors - just warn
1425 }
1426
1427 my $safe_num_ne = sub {
1428     my ($a, $b) = @_;
1429
1430     return 0 if !defined($a) && !defined($b);
1431     return 1 if !defined($a);
1432     return 1 if !defined($b);
1433
1434     return $a != $b;
1435 };
1436
1437 my $safe_string_ne = sub {
1438     my ($a, $b) = @_;
1439
1440     return 0 if !defined($a) && !defined($b);
1441     return 1 if !defined($a);
1442     return 1 if !defined($b);
1443
1444     return $a ne $b;
1445 };
1446
1447 sub update_net {
1448     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1449
1450     if ($newnet->{type} ne 'veth') {
1451         # for when there are physical interfaces
1452         die "cannot update interface of type $newnet->{type}";
1453     }
1454
1455     my $veth = "veth${vmid}i${netid}";
1456     my $eth = $newnet->{name};
1457
1458     if (my $oldnetcfg = $conf->{$opt}) {
1459         my $oldnet = parse_lxc_network($oldnetcfg);
1460
1461         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1462             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1463
1464             PVE::Network::veth_delete($veth);
1465             delete $conf->{$opt};
1466             write_config($vmid, $conf);
1467
1468             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1469
1470         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1471                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1472                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1473
1474                 if ($oldnet->{bridge}) {
1475                     PVE::Network::tap_unplug($veth);
1476                     foreach (qw(bridge tag firewall)) {
1477                         delete $oldnet->{$_};
1478                     }
1479                     $conf->{$opt} = print_lxc_network($oldnet);
1480                     write_config($vmid, $conf);
1481                 }
1482
1483                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1484                 foreach (qw(bridge tag firewall)) {
1485                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1486                 }
1487                 $conf->{$opt} = print_lxc_network($oldnet);
1488                 write_config($vmid, $conf);
1489         }
1490     } else {
1491         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1492     }
1493
1494     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1495 }
1496
1497 sub hotplug_net {
1498     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1499
1500     my $veth = "veth${vmid}i${netid}";
1501     my $vethpeer = $veth . "p";
1502     my $eth = $newnet->{name};
1503
1504     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1505     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1506
1507     # attach peer in container
1508     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1509     PVE::Tools::run_command($cmd);
1510
1511     # link up peer in container
1512     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1513     PVE::Tools::run_command($cmd);
1514
1515     my $done = { type => 'veth' };
1516     foreach (qw(bridge tag firewall hwaddr name)) {
1517         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1518     }
1519     $conf->{$opt} = print_lxc_network($done);
1520
1521     write_config($vmid, $conf);
1522 }
1523
1524 sub update_ipconfig {
1525     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1526
1527     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1528
1529     my $optdata = parse_lxc_network($conf->{$opt});
1530     my $deleted = [];
1531     my $added = [];
1532     my $nscmd = sub {
1533         my $cmdargs = shift;
1534         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1535     };
1536     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1537
1538     my $change_ip_config = sub {
1539         my ($ipversion) = @_;
1540
1541         my $family_opt = "-$ipversion";
1542         my $suffix = $ipversion == 4 ? '' : $ipversion;
1543         my $gw= "gw$suffix";
1544         my $ip= "ip$suffix";
1545
1546         my $newip = $newnet->{$ip};
1547         my $newgw = $newnet->{$gw};
1548         my $oldip = $optdata->{$ip};
1549
1550         my $change_ip = &$safe_string_ne($oldip, $newip);
1551         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1552
1553         return if !$change_ip && !$change_gw;
1554
1555         # step 1: add new IP, if this fails we cancel
1556         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1557         if ($change_ip && $is_real_ip) {
1558             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1559             if (my $err = $@) {
1560                 warn $err;
1561                 return;
1562             }
1563         }
1564
1565         # step 2: replace gateway
1566         #   If this fails we delete the added IP and cancel.
1567         #   If it succeeds we save the config and delete the old IP, ignoring
1568         #   errors. The config is then saved.
1569         # Note: 'ip route replace' can add
1570         if ($change_gw) {
1571             if ($newgw) {
1572                 eval {
1573                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1574                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1575                     }
1576                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1577                 };
1578                 if (my $err = $@) {
1579                     warn $err;
1580                     # the route was not replaced, the old IP is still available
1581                     # rollback (delete new IP) and cancel
1582                     if ($change_ip) {
1583                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1584                         warn $@ if $@; # no need to die here
1585                     }
1586                     return;
1587                 }
1588             } else {
1589                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1590                 # if the route was not deleted, the guest might have deleted it manually
1591                 # warn and continue
1592                 warn $@ if $@;
1593             }
1594         }
1595
1596         # from this point on we save the configuration
1597         # step 3: delete old IP ignoring errors
1598         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1599             # We need to enable promote_secondaries, otherwise our newly added
1600             # address will be removed along with the old one.
1601             my $promote = 0;
1602             eval {
1603                 if ($ipversion == 4) {
1604                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1605                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1606                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1607                 }
1608                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1609             };
1610             warn $@ if $@; # no need to die here
1611
1612             if ($ipversion == 4) {
1613                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1614             }
1615         }
1616
1617         foreach my $property ($ip, $gw) {
1618             if ($newnet->{$property}) {
1619                 $optdata->{$property} = $newnet->{$property};
1620             } else {
1621                 delete $optdata->{$property};
1622             }
1623         }
1624         $conf->{$opt} = print_lxc_network($optdata);
1625         write_config($vmid, $conf);
1626         $lxc_setup->setup_network($conf);
1627     };
1628
1629     &$change_ip_config(4);
1630     &$change_ip_config(6);
1631
1632 }
1633
1634 # Internal snapshots
1635
1636 # NOTE: Snapshot create/delete involves several non-atomic
1637 # action, and can take a long time.
1638 # So we try to avoid locking the file and use 'lock' variable
1639 # inside the config file instead.
1640
1641 my $snapshot_copy_config = sub {
1642     my ($source, $dest) = @_;
1643
1644     foreach my $k (keys %$source) {
1645         next if $k eq 'snapshots';
1646         next if $k eq 'snapstate';
1647         next if $k eq 'snaptime';
1648         next if $k eq 'vmstate';
1649         next if $k eq 'lock';
1650         next if $k eq 'digest';
1651         next if $k eq 'description';
1652
1653         $dest->{$k} = $source->{$k};
1654     }
1655 };
1656
1657 my $snapshot_prepare = sub {
1658     my ($vmid, $snapname, $comment) = @_;
1659
1660     my $snap;
1661
1662     my $updatefn =  sub {
1663
1664         my $conf = load_config($vmid);
1665
1666         die "you can't take a snapshot if it's a template\n"
1667             if is_template($conf);
1668
1669         check_lock($conf);
1670
1671         $conf->{lock} = 'snapshot';
1672
1673         die "snapshot name '$snapname' already used\n"
1674             if defined($conf->{snapshots}->{$snapname});
1675
1676         my $storecfg = PVE::Storage::config();
1677         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1678
1679         $snap = $conf->{snapshots}->{$snapname} = {};
1680
1681         &$snapshot_copy_config($conf, $snap);
1682
1683         $snap->{'snapstate'} = "prepare";
1684         $snap->{'snaptime'} = time();
1685         $snap->{'description'} = $comment if $comment;
1686         $conf->{snapshots}->{$snapname} = $snap;
1687
1688         write_config($vmid, $conf);
1689     };
1690
1691     lock_container($vmid, 10, $updatefn);
1692
1693     return $snap;
1694 };
1695
1696 my $snapshot_commit = sub {
1697     my ($vmid, $snapname) = @_;
1698
1699     my $updatefn = sub {
1700
1701         my $conf = load_config($vmid);
1702
1703         die "missing snapshot lock\n"
1704             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1705
1706         die "snapshot '$snapname' does not exist\n"
1707             if !defined($conf->{snapshots}->{$snapname});
1708
1709         die "wrong snapshot state\n"
1710             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1711                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1712
1713         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1714         delete $conf->{lock};
1715         $conf->{parent} = $snapname;
1716
1717         write_config($vmid, $conf);
1718     };
1719
1720     lock_container($vmid, 10 ,$updatefn);
1721 };
1722
1723 sub has_feature {
1724     my ($feature, $conf, $storecfg, $snapname) = @_;
1725
1726     my $err;
1727
1728     foreach_mountpoint($conf, sub {
1729         my ($ms, $mountpoint) = @_;
1730
1731         return if $err; # skip further test
1732
1733         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1734
1735         # TODO: implement support for mountpoints
1736         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1737             if $ms ne 'rootfs';
1738     });
1739
1740     return $err ? 0 : 1;
1741 }
1742
1743 sub snapshot_create {
1744     my ($vmid, $snapname, $comment) = @_;
1745
1746     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1747
1748     my $conf = load_config($vmid);
1749
1750     my $running = check_running($vmid);
1751     eval {
1752         if ($running) {
1753             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1754             PVE::Tools::run_command(['/bin/sync']);
1755         };
1756
1757         my $storecfg = PVE::Storage::config();
1758         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1759         my $volid = $rootinfo->{volume};
1760
1761         if ($running) {
1762             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1763         };
1764
1765         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1766         &$snapshot_commit($vmid, $snapname);
1767     };
1768     if(my $err = $@) {
1769         snapshot_delete($vmid, $snapname, 1);
1770         die "$err\n";
1771     }
1772 }
1773
1774 sub snapshot_delete {
1775     my ($vmid, $snapname, $force) = @_;
1776
1777     my $snap;
1778
1779     my $conf;
1780
1781     my $updatefn =  sub {
1782
1783         $conf = load_config($vmid);
1784
1785         die "you can't delete a snapshot if vm is a template\n"
1786             if is_template($conf);
1787
1788         $snap = $conf->{snapshots}->{$snapname};
1789
1790         check_lock($conf);
1791
1792         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1793
1794         $snap->{snapstate} = 'delete';
1795
1796         write_config($vmid, $conf);
1797     };
1798
1799     lock_container($vmid, 10, $updatefn);
1800
1801     my $storecfg = PVE::Storage::config();
1802
1803     my $del_snap =  sub {
1804
1805         check_lock($conf);
1806
1807         if ($conf->{parent} eq $snapname) {
1808             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1809                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1810             } else {
1811                 delete $conf->{parent};
1812             }
1813         }
1814
1815         delete $conf->{snapshots}->{$snapname};
1816
1817         write_config($vmid, $conf);
1818     };
1819
1820     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1821     my $rootinfo = parse_ct_mountpoint($rootfs);
1822     my $volid = $rootinfo->{volume};
1823
1824     eval {
1825         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1826     };
1827     my $err = $@;
1828
1829     if(!$err || ($err && $force)) {
1830         lock_container($vmid, 10, $del_snap);
1831         if ($err) {
1832             die "Can't delete snapshot: $vmid $snapname $err\n";
1833         }
1834     }
1835 }
1836
1837 sub snapshot_rollback {
1838     my ($vmid, $snapname) = @_;
1839
1840     my $storecfg = PVE::Storage::config();
1841
1842     my $conf = load_config($vmid);
1843
1844     die "you can't rollback if vm is a template\n" if is_template($conf);
1845
1846     my $snap = $conf->{snapshots}->{$snapname};
1847
1848     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1849
1850     my $rootfs = $snap->{rootfs};
1851     my $rootinfo = parse_ct_mountpoint($rootfs);
1852     my $volid = $rootinfo->{volume};
1853
1854     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1855
1856     my $updatefn = sub {
1857
1858         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1859             if $snap->{snapstate};
1860
1861         check_lock($conf);
1862
1863         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1864
1865         die "unable to rollback vm $vmid: vm is running\n"
1866             if check_running($vmid);
1867
1868         $conf->{lock} = 'rollback';
1869
1870         my $forcemachine;
1871
1872         # copy snapshot config to current config
1873
1874         my $tmp_conf = $conf;
1875         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1876         $conf->{snapshots} = $tmp_conf->{snapshots};
1877         delete $conf->{snaptime};
1878         delete $conf->{snapname};
1879         $conf->{parent} = $snapname;
1880
1881         write_config($vmid, $conf);
1882     };
1883
1884     my $unlockfn = sub {
1885         delete $conf->{lock};
1886         write_config($vmid, $conf);
1887     };
1888
1889     lock_container($vmid, 10, $updatefn);
1890
1891     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1892
1893     lock_container($vmid, 5, $unlockfn);
1894 }
1895
1896 sub template_create {
1897     my ($vmid, $conf) = @_;
1898
1899     my $storecfg = PVE::Storage::config();
1900
1901     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1902     my $volid = $rootinfo->{volume};
1903
1904     die "Template feature is not available for '$volid'\n"
1905         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1906
1907     PVE::Storage::activate_volumes($storecfg, [$volid]);
1908
1909     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1910     $rootinfo->{volume} = $template_volid;
1911     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1912
1913     write_config($vmid, $conf);
1914 }
1915
1916 sub is_template {
1917     my ($conf) = @_;
1918
1919     return 1 if defined $conf->{template} && $conf->{template} == 1;
1920 }
1921
1922 sub mountpoint_names {
1923     my ($reverse) = @_;
1924
1925     my @names = ('rootfs');
1926
1927     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1928         push @names, "mp$i";
1929     }
1930
1931     return $reverse ? reverse @names : @names;
1932 }
1933
1934 # The container might have *different* symlinks than the host. realpath/abs_path
1935 # use the actual filesystem to resolve links.
1936 sub sanitize_mountpoint {
1937     my ($mp) = @_;
1938     $mp = '/' . $mp; # we always start with a slash
1939     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1940     $mp =~ s@/\./@@g; # collapse /./
1941     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1942     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1943     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1944     return $mp;
1945 }
1946
1947 sub foreach_mountpoint_full {
1948     my ($conf, $reverse, $func) = @_;
1949
1950     foreach my $key (mountpoint_names($reverse)) {
1951         my $value = $conf->{$key};
1952         next if !defined($value);
1953         my $mountpoint = parse_ct_mountpoint($value, 1);
1954         next if !defined($mountpoint);
1955
1956         # just to be sure: rootfs is /
1957         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1958         $mountpoint->{mp} = sanitize_mountpoint($path);
1959
1960         $path = $mountpoint->{volume};
1961         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1962
1963         &$func($key, $mountpoint);
1964     }
1965 }
1966
1967 sub foreach_mountpoint {
1968     my ($conf, $func) = @_;
1969
1970     foreach_mountpoint_full($conf, 0, $func);
1971 }
1972
1973 sub foreach_mountpoint_reverse {
1974     my ($conf, $func) = @_;
1975
1976     foreach_mountpoint_full($conf, 1, $func);
1977 }
1978
1979 sub check_ct_modify_config_perm {
1980     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1981
1982     return 1 if $authuser ne 'root@pam';
1983
1984     foreach my $opt (@$key_list) {
1985
1986         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1987             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1988         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1989             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1990         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1991             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1992         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1993                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1994             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1995         } else {
1996             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1997         }
1998     }
1999
2000     return 1;
2001 }
2002
2003 sub umount_all {
2004     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2005
2006     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2007     my $volid_list = get_vm_volumes($conf);
2008
2009     foreach_mountpoint_reverse($conf, sub {
2010         my ($ms, $mountpoint) = @_;
2011
2012         my $volid = $mountpoint->{volume};
2013         my $mount = $mountpoint->{mp};
2014
2015         return if !$volid || !$mount;
2016
2017         my $mount_path = "$rootdir/$mount";
2018         $mount_path =~ s!/+!/!g;
2019
2020         return if !PVE::ProcFSTools::is_mounted($mount_path);
2021
2022         eval {
2023             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2024         };
2025         if (my $err = $@) {
2026             if ($noerr) {
2027                 warn $err;
2028             } else {
2029                 die $err;
2030             }
2031         }
2032     });
2033 }
2034
2035 sub mount_all {
2036     my ($vmid, $storage_cfg, $conf) = @_;
2037
2038     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2039     File::Path::make_path($rootdir);
2040
2041     my $volid_list = get_vm_volumes($conf);
2042     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2043
2044     eval {
2045         foreach_mountpoint($conf, sub {
2046             my ($ms, $mountpoint) = @_;
2047
2048             my $volid = $mountpoint->{volume};
2049             my $mount = $mountpoint->{mp};
2050
2051             return if !$volid || !$mount;
2052
2053             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2054             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2055                 PVE::Storage::parse_volname($storage_cfg, $volid);
2056
2057             die "unable to mount base volume - internal error" if $isBase;
2058
2059             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2060         });
2061     };
2062     if (my $err = $@) {
2063         warn "mounting container failed - $err";
2064         umount_all($vmid, $storage_cfg, $conf, 1);
2065     }
2066
2067     return $rootdir;
2068 }
2069
2070
2071 sub mountpoint_mount_path {
2072     my ($mountpoint, $storage_cfg, $snapname) = @_;
2073
2074     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2075 }
2076
2077 my $check_mount_path = sub {
2078     my ($path) = @_;
2079     $path = File::Spec->canonpath($path);
2080     my $real = Cwd::realpath($path);
2081     if ($real ne $path) {
2082         die "mount path modified by symlink: $path != $real";
2083     }
2084 };
2085
2086 # use $rootdir = undef to just return the corresponding mount path
2087 sub mountpoint_mount {
2088     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2089
2090     my $volid = $mountpoint->{volume};
2091     my $mount = $mountpoint->{mp};
2092
2093     return if !$volid || !$mount;
2094
2095     my $mount_path;
2096
2097     if (defined($rootdir)) {
2098         $rootdir =~ s!/+$!!;
2099         $mount_path = "$rootdir/$mount";
2100         $mount_path =~ s!/+!/!g;
2101         &$check_mount_path($mount_path);
2102         File::Path::mkpath($mount_path);
2103     }
2104
2105     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2106
2107     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2108
2109     if ($storage) {
2110
2111         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2112         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2113
2114         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2115             PVE::Storage::parse_volname($storage_cfg, $volid);
2116
2117         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2118
2119         if ($format eq 'subvol') {
2120             if ($mount_path) {
2121                 if ($snapname) {
2122                     if ($scfg->{type} eq 'zfspool') {
2123                         my $path_arg = $path;
2124                         $path_arg =~ s!^/+!!;
2125                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2126                     } else {
2127                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2128                     }
2129                 } else {
2130                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2131                 }
2132             }
2133             return wantarray ? ($path, 0) : $path;
2134         } elsif ($format eq 'raw' || $format eq 'iso') {
2135             my $use_loopdev = 0;
2136             my @extra_opts;
2137             if ($scfg->{path}) {
2138                 push @extra_opts, '-o', 'loop';
2139                 $use_loopdev = 1;
2140             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2141                 # do nothing
2142             } else {
2143                 die "unsupported storage type '$scfg->{type}'\n";
2144             }
2145             if ($mount_path) {
2146                 if ($format eq 'iso') {
2147                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2148                 } elsif ($isBase || defined($snapname)) {
2149                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2150                 } else {
2151                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2152                 }
2153             }
2154             return wantarray ? ($path, $use_loopdev) : $path;
2155         } else {
2156             die "unsupported image format '$format'\n";
2157         }
2158     } elsif ($volid =~ m|^/dev/.+|) {
2159         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2160         return wantarray ? ($volid, 0) : $volid;
2161     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2162         &$check_mount_path($volid);
2163         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2164         return wantarray ? ($volid, 0) : $volid;
2165     }
2166
2167     die "unsupported storage";
2168 }
2169
2170 sub get_vm_volumes {
2171     my ($conf, $excludes) = @_;
2172
2173     my $vollist = [];
2174
2175     foreach_mountpoint($conf, sub {
2176         my ($ms, $mountpoint) = @_;
2177
2178         return if $excludes && $ms eq $excludes;
2179
2180         my $volid = $mountpoint->{volume};
2181
2182         return if !$volid || $volid =~ m|^/|;
2183
2184         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2185         return if !$sid;
2186
2187         push @$vollist, $volid;
2188     });
2189
2190     return $vollist;
2191 }
2192
2193 sub mkfs {
2194     my ($dev) = @_;
2195
2196     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2197 }
2198
2199 sub format_disk {
2200     my ($storage_cfg, $volid) = @_;
2201
2202     if ($volid =~ m!^/dev/.+!) {
2203         mkfs($volid);
2204         return;
2205     }
2206
2207     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2208
2209     die "cannot format volume '$volid' with no storage\n" if !$storage;
2210
2211     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2212
2213     my $path = PVE::Storage::path($storage_cfg, $volid);
2214
2215     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2216         PVE::Storage::parse_volname($storage_cfg, $volid);
2217
2218     die "cannot format volume '$volid' (format == $format)\n"
2219         if $format ne 'raw';
2220
2221     mkfs($path);
2222 }
2223
2224 sub destroy_disks {
2225     my ($storecfg, $vollist) = @_;
2226
2227     foreach my $volid (@$vollist) {
2228         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2229         warn $@ if $@;
2230     }
2231 }
2232
2233 sub create_disks {
2234     my ($storecfg, $vmid, $settings, $conf) = @_;
2235
2236     my $vollist = [];
2237
2238     eval {
2239         foreach_mountpoint($settings, sub {
2240             my ($ms, $mountpoint) = @_;
2241
2242             my $volid = $mountpoint->{volume};
2243             my $mp = $mountpoint->{mp};
2244
2245             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2246
2247             return if !$storage;
2248
2249             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2250                 my ($storeid, $size_gb) = ($1, $2);
2251
2252                 my $size_kb = int(${size_gb}*1024) * 1024;
2253
2254                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2255                 # fixme: use better naming ct-$vmid-disk-X.raw?
2256
2257                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2258                     if ($size_kb > 0) {
2259                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2260                                                            undef, $size_kb);
2261                         format_disk($storecfg, $volid);
2262                     } else {
2263                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2264                                                            undef, 0);
2265                     }
2266                 } elsif ($scfg->{type} eq 'zfspool') {
2267
2268                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2269                                                        undef, $size_kb);
2270                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2271
2272                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2273                     format_disk($storecfg, $volid);
2274
2275                 } elsif ($scfg->{type} eq 'rbd') {
2276
2277                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2278                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2279                     format_disk($storecfg, $volid);
2280                 } else {
2281                     die "unable to create containers on storage type '$scfg->{type}'\n";
2282                 }
2283                 push @$vollist, $volid;
2284                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2285                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2286             } else {
2287                 # use specified/existing volid
2288             }
2289         });
2290     };
2291     # free allocated images on error
2292     if (my $err = $@) {
2293         destroy_disks($storecfg, $vollist);
2294         die $err;
2295     }
2296     return $vollist;
2297 }
2298
2299 # bash completion helper
2300
2301 sub complete_os_templates {
2302     my ($cmdname, $pname, $cvalue) = @_;
2303
2304     my $cfg = PVE::Storage::config();
2305
2306     my $storeid;
2307
2308     if ($cvalue =~ m/^([^:]+):/) {
2309         $storeid = $1;
2310     }
2311
2312     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2313     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2314
2315     my $res = [];
2316     foreach my $id (keys %$data) {
2317         foreach my $item (@{$data->{$id}}) {
2318             push @$res, $item->{volid} if defined($item->{volid});
2319         }
2320     }
2321
2322     return $res;
2323 }
2324
2325 my $complete_ctid_full = sub {
2326     my ($running) = @_;
2327
2328     my $idlist = vmstatus();
2329
2330     my $active_hash = list_active_containers();
2331
2332     my $res = [];
2333
2334     foreach my $id (keys %$idlist) {
2335         my $d = $idlist->{$id};
2336         if (defined($running)) {
2337             next if $d->{template};
2338             next if $running && !$active_hash->{$id};
2339             next if !$running && $active_hash->{$id};
2340         }
2341         push @$res, $id;
2342
2343     }
2344     return $res;
2345 };
2346
2347 sub complete_ctid {
2348     return &$complete_ctid_full();
2349 }
2350
2351 sub complete_ctid_stopped {
2352     return &$complete_ctid_full(0);
2353 }
2354
2355 sub complete_ctid_running {
2356     return &$complete_ctid_full(1);
2357 }
2358
2359 sub parse_id_maps {
2360     my ($conf) = @_;
2361
2362     my $id_map = [];
2363     my $rootuid = 0;
2364     my $rootgid = 0;
2365
2366     my $lxc = $conf->{lxc};
2367     foreach my $entry (@$lxc) {
2368         my ($key, $value) = @$entry;
2369         next if $key ne 'lxc.id_map';
2370         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2371             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2372             push @$id_map, [$type, $ct, $host, $length];
2373             if ($ct == 0) {
2374                 $rootuid = $host if $type eq 'u';
2375                 $rootgid = $host if $type eq 'g';
2376             }
2377         } else {
2378             die "failed to parse id_map: $value\n";
2379         }
2380     }
2381
2382     if (!@$id_map && $conf->{unprivileged}) {
2383         # Should we read them from /etc/subuid?
2384         $id_map = [ ['u', '0', '100000', '65536'],
2385                     ['g', '0', '100000', '65536'] ];
2386         $rootuid = $rootgid = 100000;
2387     }
2388
2389     return ($id_map, $rootuid, $rootgid);
2390 }
2391
2392 1;