src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format_description => 'volume',
  42         description => 'Volume, device or directory to mount into the container.',
  43     },
  44     backup => {
  45         type => 'boolean',
  46         format_description => '[1|0]',
  47         description => 'Whether to include the mountpoint in backups.',
  48         optional => 1,
  49     },
  50     size => {
  51         type => 'string',
  52         format => 'disk-size',
  53         format_description => 'DiskSize',
  54         description => 'Volume size (read only value).',
  55         optional => 1,
  56     },
  57 };
  58
  59 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  60     type => 'string', format => $rootfs_desc,
  61     description => "Use volume as container root.",
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 my $confdesc = {
  72     lock => {
  73         optional => 1,
  74         type => 'string',
  75         description => "Lock/unlock the VM.",
  76         enum => [qw(migrate backup snapshot rollback)],
  77     },
  78     onboot => {
  79         optional => 1,
  80         type => 'boolean',
  81         description => "Specifies whether a VM will be started during system bootup.",
  82         default => 0,
  83     },
  84     startup => get_standard_option('pve-startup-order'),
  85     template => {
  86         optional => 1,
  87         type => 'boolean',
  88         description => "Enable/disable Template.",
  89         default => 0,
  90     },
  91     arch => {
  92         optional => 1,
  93         type => 'string',
  94         enum => ['amd64', 'i386'],
  95         description => "OS architecture type.",
  96         default => 'amd64',
  97     },
  98     ostype => {
  99         optional => 1,
 100         type => 'string',
 101         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux'],
 102         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 103     },
 104     console => {
 105         optional => 1,
 106         type => 'boolean',
 107         description => "Attach a console device (/dev/console) to the container.",
 108         default => 1,
 109     },
 110     tty => {
 111         optional => 1,
 112         type => 'integer',
 113         description => "Specify the number of tty available to the container",
 114         minimum => 0,
 115         maximum => 6,
 116         default => 2,
 117     },
 118     cpulimit => {
 119         optional => 1,
 120         type => 'number',
 121         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 122         minimum => 0,
 123         maximum => 128,
 124         default => 0,
 125     },
 126     cpuunits => {
 127         optional => 1,
 128         type => 'integer',
 129         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 130         minimum => 0,
 131         maximum => 500000,
 132         default => 1024,
 133     },
 134     memory => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Amount of RAM for the VM in MB.",
 138         minimum => 16,
 139         default => 512,
 140     },
 141     swap => {
 142         optional => 1,
 143         type => 'integer',
 144         description => "Amount of SWAP for the VM in MB.",
 145         minimum => 0,
 146         default => 512,
 147     },
 148     hostname => {
 149         optional => 1,
 150         description => "Set a host name for the container.",
 151         type => 'string', format => 'dns-name',
 152         maxLength => 255,
 153     },
 154     description => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Container description. Only used on the configuration web interface.",
 158     },
 159     searchdomain => {
 160         optional => 1,
 161         type => 'string', format => 'dns-name-list',
 162         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 163     },
 164     nameserver => {
 165         optional => 1,
 166         type => 'string', format => 'address-list',
 167         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 168     },
 169     rootfs => get_standard_option('pve-ct-rootfs'),
 170     parent => {
 171         optional => 1,
 172         type => 'string', format => 'pve-configid',
 173         maxLength => 40,
 174         description => "Parent snapshot name. This is used internally, and should not be modified.",
 175     },
 176     snaptime => {
 177         optional => 1,
 178         description => "Timestamp for snapshots.",
 179         type => 'integer',
 180         minimum => 0,
 181     },
 182     cmode => {
 183         optional => 1,
 184         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 185         type => 'string',
 186         enum => ['shell', 'console', 'tty'],
 187         default => 'tty',
 188     },
 189     protection => {
 190         optional => 1,
 191         type => 'boolean',
 192         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 193         default => 0,
 194     },
 195     unprivileged => {
 196         optional => 1,
 197         type => 'boolean',
 198         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 199         default => 0,
 200     },
 201 };
 202
 203 my $valid_lxc_conf_keys = {
 204     'lxc.include' => 1,
 205     'lxc.arch' => 1,
 206     'lxc.utsname' => 1,
 207     'lxc.haltsignal' => 1,
 208     'lxc.rebootsignal' => 1,
 209     'lxc.stopsignal' => 1,
 210     'lxc.init_cmd' => 1,
 211     'lxc.network.type' => 1,
 212     'lxc.network.flags' => 1,
 213     'lxc.network.link' => 1,
 214     'lxc.network.mtu' => 1,
 215     'lxc.network.name' => 1,
 216     'lxc.network.hwaddr' => 1,
 217     'lxc.network.ipv4' => 1,
 218     'lxc.network.ipv4.gateway' => 1,
 219     'lxc.network.ipv6' => 1,
 220     'lxc.network.ipv6.gateway' => 1,
 221     'lxc.network.script.up' => 1,
 222     'lxc.network.script.down' => 1,
 223     'lxc.pts' => 1,
 224     'lxc.console.logfile' => 1,
 225     'lxc.console' => 1,
 226     'lxc.tty' => 1,
 227     'lxc.devttydir' => 1,
 228     'lxc.hook.autodev' => 1,
 229     'lxc.autodev' => 1,
 230     'lxc.kmsg' => 1,
 231     'lxc.mount' => 1,
 232     'lxc.mount.entry' => 1,
 233     'lxc.mount.auto' => 1,
 234     'lxc.rootfs' => 1,
 235     'lxc.rootfs.mount' => 1,
 236     'lxc.rootfs.options' => 1,
 237     # lxc.cgroup.*
 238     'lxc.cap.drop' => 1,
 239     'lxc.cap.keep' => 1,
 240     'lxc.aa_profile' => 1,
 241     'lxc.aa_allow_incomplete' => 1,
 242     'lxc.se_context' => 1,
 243     'lxc.seccomp' => 1,
 244     'lxc.id_map' => 1,
 245     'lxc.hook.pre-start' => 1,
 246     'lxc.hook.pre-mount' => 1,
 247     'lxc.hook.mount' => 1,
 248     'lxc.hook.start' => 1,
 249     'lxc.hook.stop' => 1,
 250     'lxc.hook.post-stop' => 1,
 251     'lxc.hook.clone' => 1,
 252     'lxc.hook.destroy' => 1,
 253     'lxc.loglevel' => 1,
 254     'lxc.logfile' => 1,
 255     'lxc.start.auto' => 1,
 256     'lxc.start.delay' => 1,
 257     'lxc.start.order' => 1,
 258     'lxc.group' => 1,
 259     'lxc.environment' => 1,
 260 };
 261
 262 my $netconf_desc = {
 263     type => {
 264         type => 'string',
 265         optional => 1,
 266         description => "Network interface type.",
 267         enum => [qw(veth)],
 268     },
 269     name => {
 270         type => 'string',
 271         format_description => 'String',
 272         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 273         pattern => '[-_.\w\d]+',
 274     },
 275     bridge => {
 276         type => 'string',
 277         format_description => 'vmbr<Number>',
 278         description => 'Bridge to attach the network device to.',
 279         pattern => '[-_.\w\d]+',
 280         optional => 1,
 281     },
 282     hwaddr => {
 283         type => 'string',
 284         format_description => 'MAC',
 285         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 286         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 287         optional => 1,
 288     },
 289     mtu => {
 290         type => 'integer',
 291         format_description => 'Number',
 292         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 293         minimum => 64, # minimum ethernet frame is 64 bytes
 294         optional => 1,
 295     },
 296     ip => {
 297         type => 'string',
 298         format => 'pve-ipv4-config',
 299         format_description => 'IPv4Format/CIDR',
 300         description => 'IPv4 address in CIDR format.',
 301         optional => 1,
 302     },
 303     gw => {
 304         type => 'string',
 305         format => 'ipv4',
 306         format_description => 'GatewayIPv4',
 307         description => 'Default gateway for IPv4 traffic.',
 308         optional => 1,
 309     },
 310     ip6 => {
 311         type => 'string',
 312         format => 'pve-ipv6-config',
 313         format_description => 'IPv6Format/CIDR',
 314         description => 'IPv6 address in CIDR format.',
 315         optional => 1,
 316     },
 317     gw6 => {
 318         type => 'string',
 319         format => 'ipv6',
 320         format_description => 'GatewayIPv6',
 321         description => 'Default gateway for IPv6 traffic.',
 322         optional => 1,
 323     },
 324     firewall => {
 325         type => 'boolean',
 326         format_description => '[1|0]',
 327         description => "Controls whether this interface's firewall rules should be used.",
 328         optional => 1,
 329     },
 330     tag => {
 331         type => 'integer',
 332         format_description => 'VlanNo',
 333         minimum => '2',
 334         maximum => '4094',
 335         description => "VLAN tag for this interface.",
 336         optional => 1,
 337     },
 338     trunks => {
 339         type => 'string',
 340         pattern => qr/\d+(?:;\d+)*/,
 341         format_description => 'vlanid[;vlanid...]',
 342         description => "VLAN ids to pass through the interface",
 343         optional => 1,
 344     },
 345 };
 346 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 347
 348 my $MAX_LXC_NETWORKS = 10;
 349 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 350     $confdesc->{"net$i"} = {
 351         optional => 1,
 352         type => 'string', format => $netconf_desc,
 353         description => "Specifies network interfaces for the container.",
 354     };
 355 }
 356
 357 my $mp_desc = {
 358     %$rootfs_desc,
 359     mp => {
 360         type => 'string',
 361         format_description => 'Path',
 362         description => 'Path to the mountpoint as seen from inside the container.',
 363     },
 364 };
 365 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 366
 367 my $unuseddesc = {
 368     optional => 1,
 369     type => 'string', format => 'pve-volume-id',
 370     description => "Reference to unused volumes.",
 371 };
 372
 373 my $MAX_MOUNT_POINTS = 10;
 374 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 375     $confdesc->{"mp$i"} = {
 376         optional => 1,
 377         type => 'string', format => $mp_desc,
 378         description => "Use volume as container mount point (experimental feature).",
 379         optional => 1,
 380     };
 381 }
 382
 383 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 384 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 385     $confdesc->{"unused$i"} = $unuseddesc;
 386 }
 387
 388 sub write_pct_config {
 389     my ($filename, $conf) = @_;
 390
 391     delete $conf->{snapstate}; # just to be sure
 392
 393     my $generate_raw_config = sub {
 394         my ($conf) = @_;
 395
 396         my $raw = '';
 397
 398         # add description as comment to top of file
 399         my $descr = $conf->{description} || '';
 400         foreach my $cl (split(/\n/, $descr)) {
 401             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 402         }
 403
 404         foreach my $key (sort keys %$conf) {
 405             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 406                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 407             my $value = $conf->{$key};
 408             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 409             $raw .= "$key: $value\n";
 410         }
 411
 412         if (my $lxcconf = $conf->{lxc}) {
 413             foreach my $entry (@$lxcconf) {
 414                 my ($k, $v) = @$entry;
 415                 $raw .= "$k: $v\n";
 416             }
 417         }
 418
 419         return $raw;
 420     };
 421
 422     my $raw = &$generate_raw_config($conf);
 423
 424     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 425         $raw .= "\n[$snapname]\n";
 426         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 427     }
 428
 429     return $raw;
 430 }
 431
 432 sub check_type {
 433     my ($key, $value) = @_;
 434
 435     die "unknown setting '$key'\n" if !$confdesc->{$key};
 436
 437     my $type = $confdesc->{$key}->{type};
 438
 439     if (!defined($value)) {
 440         die "got undefined value\n";
 441     }
 442
 443     if ($value =~ m/[\n\r]/) {
 444         die "property contains a line feed\n";
 445     }
 446
 447     if ($type eq 'boolean') {
 448         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 449         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 450         die "type check ('boolean') failed - got '$value'\n";
 451     } elsif ($type eq 'integer') {
 452         return int($1) if $value =~ m/^(\d+)$/;
 453         die "type check ('integer') failed - got '$value'\n";
 454     } elsif ($type eq 'number') {
 455         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 456         die "type check ('number') failed - got '$value'\n";
 457     } elsif ($type eq 'string') {
 458         if (my $fmt = $confdesc->{$key}->{format}) {
 459             PVE::JSONSchema::check_format($fmt, $value);
 460             return $value;
 461         }
 462         return $value;
 463     } else {
 464         die "internal error"
 465     }
 466 }
 467
 468 sub parse_pct_config {
 469     my ($filename, $raw) = @_;
 470
 471     return undef if !defined($raw);
 472
 473     my $res = {
 474         digest => Digest::SHA::sha1_hex($raw),
 475         snapshots => {},
 476     };
 477
 478     $filename =~ m|/lxc/(\d+).conf$|
 479         || die "got strange filename '$filename'";
 480
 481     my $vmid = $1;
 482
 483     my $conf = $res;
 484     my $descr = '';
 485     my $section = '';
 486
 487     my @lines = split(/\n/, $raw);
 488     foreach my $line (@lines) {
 489         next if $line =~ m/^\s*$/;
 490
 491         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 492             $section = $1;
 493             $conf->{description} = $descr if $descr;
 494             $descr = '';
 495             $conf = $res->{snapshots}->{$section} = {};
 496             next;
 497         }
 498
 499         if ($line =~ m/^\#(.*)\s*$/) {
 500             $descr .= PVE::Tools::decode_text($1) . "\n";
 501             next;
 502         }
 503
 504         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 505             my $key = $1;
 506             my $value = $3;
 507             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 508                 push @{$conf->{lxc}}, [$key, $value];
 509             } else {
 510                 warn "vm $vmid - unable to parse config: $line\n";
 511             }
 512         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 513             $descr .= PVE::Tools::decode_text($2);
 514         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 515             $conf->{snapstate} = $1;
 516         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 517             my $key = $1;
 518             my $value = $2;
 519             eval { $value = check_type($key, $value); };
 520             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 521             $conf->{$key} = $value;
 522         } else {
 523             warn "vm $vmid - unable to parse config: $line\n";
 524         }
 525     }
 526
 527     $conf->{description} = $descr if $descr;
 528
 529     delete $res->{snapstate}; # just to be sure
 530
 531     return $res;
 532 }
 533
 534 sub config_list {
 535     my $vmlist = PVE::Cluster::get_vmlist();
 536     my $res = {};
 537     return $res if !$vmlist || !$vmlist->{ids};
 538     my $ids = $vmlist->{ids};
 539
 540     foreach my $vmid (keys %$ids) {
 541         next if !$vmid; # skip CT0
 542         my $d = $ids->{$vmid};
 543         next if !$d->{node} || $d->{node} ne $nodename;
 544         next if !$d->{type} || $d->{type} ne 'lxc';
 545         $res->{$vmid}->{type} = 'lxc';
 546     }
 547     return $res;
 548 }
 549
 550 sub cfs_config_path {
 551     my ($vmid, $node) = @_;
 552
 553     $node = $nodename if !$node;
 554     return "nodes/$node/lxc/$vmid.conf";
 555 }
 556
 557 sub config_file {
 558     my ($vmid, $node) = @_;
 559
 560     my $cfspath = cfs_config_path($vmid, $node);
 561     return "/etc/pve/$cfspath";
 562 }
 563
 564 sub load_config {
 565     my ($vmid, $node) = @_;
 566
 567     $node = $nodename if !$node;
 568     my $cfspath = cfs_config_path($vmid, $node);
 569
 570     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 571     die "container $vmid does not exist\n" if !defined($conf);
 572
 573     return $conf;
 574 }
 575
 576 sub create_config {
 577     my ($vmid, $conf) = @_;
 578
 579     my $dir = "/etc/pve/nodes/$nodename/lxc";
 580     mkdir $dir;
 581
 582     write_config($vmid, $conf);
 583 }
 584
 585 sub destroy_config {
 586     my ($vmid) = @_;
 587
 588     unlink config_file($vmid, $nodename);
 589 }
 590
 591 sub write_config {
 592     my ($vmid, $conf) = @_;
 593
 594     my $cfspath = cfs_config_path($vmid);
 595
 596     PVE::Cluster::cfs_write_file($cfspath, $conf);
 597 }
 598
 599 # flock: we use one file handle per process, so lock file
 600 # can be called multiple times and will succeed for the same process.
 601
 602 my $lock_handles =  {};
 603 my $lockdir = "/run/lock/lxc";
 604
 605 sub lock_filename {
 606     my ($vmid) = @_;
 607
 608     return "$lockdir/pve-config-${vmid}.lock";
 609 }
 610
 611 sub lock_aquire {
 612     my ($vmid, $timeout) = @_;
 613
 614     $timeout = 10 if !$timeout;
 615     my $mode = LOCK_EX;
 616
 617     my $filename = lock_filename($vmid);
 618
 619     mkdir $lockdir if !-d $lockdir;
 620
 621     my $lock_func = sub {
 622         if (!$lock_handles->{$$}->{$filename}) {
 623             my $fh = new IO::File(">>$filename") ||
 624                 die "can't open file - $!\n";
 625             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 626         }
 627
 628         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 629             print STDERR "trying to aquire lock...";
 630             my $success;
 631             while(1) {
 632                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 633                 # try again on EINTR (see bug #273)
 634                 if ($success || ($! != EINTR)) {
 635                     last;
 636                 }
 637             }
 638             if (!$success) {
 639                 print STDERR " failed\n";
 640                 die "can't aquire lock - $!\n";
 641             }
 642
 643             print STDERR " OK\n";
 644         }
 645
 646         $lock_handles->{$$}->{$filename}->{refcount}++;
 647     };
 648
 649     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 650     my $err = $@;
 651     if ($err) {
 652         die "can't lock file '$filename' - $err";
 653     }
 654 }
 655
 656 sub lock_release {
 657     my ($vmid) = @_;
 658
 659     my $filename = lock_filename($vmid);
 660
 661     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 662         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 663         if ($refcount <= 0) {
 664             $lock_handles->{$$}->{$filename} = undef;
 665             close ($fh);
 666         }
 667     }
 668 }
 669
 670 sub lock_container {
 671     my ($vmid, $timeout, $code, @param) = @_;
 672
 673     my $res;
 674
 675     lock_aquire($vmid, $timeout);
 676     eval { $res = &$code(@param) };
 677     my $err = $@;
 678     lock_release($vmid);
 679
 680     die $err if $err;
 681
 682     return $res;
 683 }
 684
 685 sub option_exists {
 686     my ($name) = @_;
 687
 688     return defined($confdesc->{$name});
 689 }
 690
 691 # add JSON properties for create and set function
 692 sub json_config_properties {
 693     my $prop = shift;
 694
 695     foreach my $opt (keys %$confdesc) {
 696         next if $opt eq 'parent' || $opt eq 'snaptime';
 697         next if $prop->{$opt};
 698         $prop->{$opt} = $confdesc->{$opt};
 699     }
 700
 701     return $prop;
 702 }
 703
 704 sub json_config_properties_no_rootfs {
 705     my $prop = shift;
 706
 707     foreach my $opt (keys %$confdesc) {
 708         next if $prop->{$opt};
 709         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 710         $prop->{$opt} = $confdesc->{$opt};
 711     }
 712
 713     return $prop;
 714 }
 715
 716 # container status helpers
 717
 718 sub list_active_containers {
 719
 720     my $filename = "/proc/net/unix";
 721
 722     # similar test is used by lcxcontainers.c: list_active_containers
 723     my $res = {};
 724
 725     my $fh = IO::File->new ($filename, "r");
 726     return $res if !$fh;
 727
 728     while (defined(my $line = <$fh>)) {
 729         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 730             my $path = $1;
 731             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 732                 $res->{$1} = 1;
 733             }
 734         }
 735     }
 736
 737     close($fh);
 738
 739     return $res;
 740 }
 741
 742 # warning: this is slow
 743 sub check_running {
 744     my ($vmid) = @_;
 745
 746     my $active_hash = list_active_containers();
 747
 748     return 1 if defined($active_hash->{$vmid});
 749
 750     return undef;
 751 }
 752
 753 sub get_container_disk_usage {
 754     my ($vmid, $pid) = @_;
 755
 756     return PVE::Tools::df("/proc/$pid/root/", 1);
 757 }
 758
 759 my $last_proc_vmid_stat;
 760
 761 my $parse_cpuacct_stat = sub {
 762     my ($vmid) = @_;
 763
 764     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 765
 766     my $stat = {};
 767
 768     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 769
 770         $stat->{utime} = $1;
 771         $stat->{stime} = $2;
 772
 773     }
 774
 775     return $stat;
 776 };
 777
 778 sub vmstatus {
 779     my ($opt_vmid) = @_;
 780
 781     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 782
 783     my $active_hash = list_active_containers();
 784
 785     my $cpucount = $cpuinfo->{cpus} || 1;
 786
 787     my $cdtime = gettimeofday;
 788
 789     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 790
 791     foreach my $vmid (keys %$list) {
 792         my $d = $list->{$vmid};
 793
 794         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 795         warn $@ if $@; # ignore errors (consider them stopped)
 796
 797         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 798
 799         my $cfspath = cfs_config_path($vmid);
 800         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 801
 802         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 803         $d->{name} =~ s/[\s]//g;
 804
 805         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 806
 807         if ($d->{pid}) {
 808             my $res = get_container_disk_usage($vmid, $d->{pid});
 809             $d->{disk} = $res->{used};
 810             $d->{maxdisk} = $res->{total};
 811         } else {
 812             $d->{disk} = 0;
 813             # use 4GB by default ??
 814             if (my $rootfs = $conf->{rootfs}) {
 815                 my $rootinfo = parse_ct_rootfs($rootfs);
 816                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 817             } else {
 818                 $d->{maxdisk} = 4*1024*1024*1024;
 819             }
 820         }
 821
 822         $d->{mem} = 0;
 823         $d->{swap} = 0;
 824         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 825         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 826
 827         $d->{uptime} = 0;
 828         $d->{cpu} = 0;
 829
 830         $d->{netout} = 0;
 831         $d->{netin} = 0;
 832
 833         $d->{diskread} = 0;
 834         $d->{diskwrite} = 0;
 835
 836         $d->{template} = is_template($conf);
 837     }
 838
 839     foreach my $vmid (keys %$list) {
 840         my $d = $list->{$vmid};
 841         my $pid = $d->{pid};
 842
 843         next if !$pid; # skip stopped CTs
 844
 845         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 846         $d->{uptime} = time - $ctime; # the method lxcfs uses
 847
 848         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 849         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 850
 851         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 852         my @bytes = split(/\n/, $blkio_bytes);
 853         foreach my $byte (@bytes) {
 854             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 855                 $d->{diskread} = $2 if $key eq 'Read';
 856                 $d->{diskwrite} = $2 if $key eq 'Write';
 857             }
 858         }
 859
 860         my $pstat = &$parse_cpuacct_stat($vmid);
 861
 862         my $used = $pstat->{utime} + $pstat->{stime};
 863
 864         my $old = $last_proc_vmid_stat->{$vmid};
 865         if (!$old) {
 866             $last_proc_vmid_stat->{$vmid} = {
 867                 time => $cdtime,
 868                 used => $used,
 869                 cpu => 0,
 870             };
 871             next;
 872         }
 873
 874         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 875
 876         if ($dtime > 1000) {
 877             my $dutime = $used -  $old->{used};
 878
 879             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 880             $last_proc_vmid_stat->{$vmid} = {
 881                 time => $cdtime,
 882                 used => $used,
 883                 cpu => $d->{cpu},
 884             };
 885         } else {
 886             $d->{cpu} = $old->{cpu};
 887         }
 888     }
 889
 890     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 891
 892     foreach my $dev (keys %$netdev) {
 893         next if $dev !~ m/^veth([1-9]\d*)i/;
 894         my $vmid = $1;
 895         my $d = $list->{$vmid};
 896
 897         next if !$d;
 898
 899         $d->{netout} += $netdev->{$dev}->{receive};
 900         $d->{netin} += $netdev->{$dev}->{transmit};
 901
 902     }
 903
 904     return $list;
 905 }
 906
 907 sub classify_mountpoint {
 908     my ($vol) = @_;
 909     if ($vol =~ m!^/!) {
 910         return 'device' if $vol =~ m!^/dev/!;
 911         return 'bind';
 912     }
 913     return 'volume';
 914 }
 915
 916 my $parse_ct_mountpoint_full = sub {
 917     my ($desc, $data, $noerr) = @_;
 918
 919     $data //= '';
 920
 921     my $res;
 922     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 923     if ($@) {
 924         return undef if $noerr;
 925         die $@;
 926     }
 927
 928     if (defined(my $size = $res->{size})) {
 929         $size = PVE::JSONSchema::parse_size($size);
 930         if (!defined($size)) {
 931             return undef if $noerr;
 932             die "invalid size: $size\n";
 933         }
 934         $res->{size} = $size;
 935     }
 936
 937     $res->{type} = classify_mountpoint($res->{volume});
 938
 939     return $res;
 940 };
 941
 942 sub parse_ct_rootfs {
 943     my ($data, $noerr) = @_;
 944
 945     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 946
 947     $res->{mp} = '/' if defined($res);
 948
 949     return $res;
 950 }
 951
 952 sub parse_ct_mountpoint {
 953     my ($data, $noerr) = @_;
 954
 955     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 956 }
 957
 958 sub print_ct_mountpoint {
 959     my ($info, $nomp) = @_;
 960     my $skip = [ 'type' ];
 961     push @$skip, 'mp' if $nomp;
 962     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 963 }
 964
 965 sub print_lxc_network {
 966     my $net = shift;
 967     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 968 }
 969
 970 sub parse_lxc_network {
 971     my ($data) = @_;
 972
 973     my $res = {};
 974
 975     return $res if !$data;
 976
 977     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 978
 979     $res->{type} = 'veth';
 980     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 981
 982     return $res;
 983 }
 984
 985 sub read_cgroup_value {
 986     my ($group, $vmid, $name, $full) = @_;
 987
 988     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 989
 990     return PVE::Tools::file_get_contents($path) if $full;
 991
 992     return PVE::Tools::file_read_firstline($path);
 993 }
 994
 995 sub write_cgroup_value {
 996    my ($group, $vmid, $name, $value) = @_;
 997
 998    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 999    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
1000
1001 }
1002
1003 sub find_lxc_console_pids {
1004
1005     my $res = {};
1006
1007     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
1008         my ($pid) = @_;
1009
1010         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
1011         return if !$cmdline;
1012
1013         my @args = split(/\0/, $cmdline);
1014
1015         # search for lxc-console -n <vmid>
1016         return if scalar(@args) != 3;
1017         return if $args[1] ne '-n';
1018         return if $args[2] !~ m/^\d+$/;
1019         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1020
1021         my $vmid = $args[2];
1022
1023         push @{$res->{$vmid}}, $pid;
1024     });
1025
1026     return $res;
1027 }
1028
1029 sub find_lxc_pid {
1030     my ($vmid) = @_;
1031
1032     my $pid = undef;
1033     my $parser = sub {
1034         my $line = shift;
1035         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1036     };
1037     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1038
1039     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1040
1041     return $pid;
1042 }
1043
1044 # Note: we cannot use Net:IP, because that only allows strict
1045 # CIDR networks
1046 sub parse_ipv4_cidr {
1047     my ($cidr, $noerr) = @_;
1048
1049     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1050         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1051     }
1052
1053     return undef if $noerr;
1054
1055     die "unable to parse ipv4 address/mask\n";
1056 }
1057
1058 sub check_lock {
1059     my ($conf) = @_;
1060
1061     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1062 }
1063
1064 sub check_protection {
1065     my ($vm_conf, $err_msg) = @_;
1066
1067     if ($vm_conf->{protection}) {
1068         die "$err_msg - protection mode enabled\n";
1069     }
1070 }
1071
1072 sub update_lxc_config {
1073     my ($storage_cfg, $vmid, $conf) = @_;
1074
1075     my $dir = "/var/lib/lxc/$vmid";
1076
1077     if ($conf->{template}) {
1078
1079         unlink "$dir/config";
1080
1081         return;
1082     }
1083
1084     my $raw = '';
1085
1086     die "missing 'arch' - internal error" if !$conf->{arch};
1087     $raw .= "lxc.arch = $conf->{arch}\n";
1088
1089     my $unprivileged = $conf->{unprivileged};
1090     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1091
1092     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1093     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux)$/x) {
1094         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1095         if ($unprivileged || $custom_idmap) {
1096             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1097         }
1098     } else {
1099         die "implement me (ostype $ostype)";
1100     }
1101
1102     $raw .= "lxc.monitor.unshare = 1\n";
1103
1104     # Should we read them from /etc/subuid?
1105     if ($unprivileged && !$custom_idmap) {
1106         $raw .= "lxc.id_map = u 0 100000 65536\n";
1107         $raw .= "lxc.id_map = g 0 100000 65536\n";
1108     }
1109
1110     if (!has_dev_console($conf)) {
1111         $raw .= "lxc.console = none\n";
1112         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1113     }
1114
1115     my $ttycount = get_tty_count($conf);
1116     $raw .= "lxc.tty = $ttycount\n";
1117
1118     # some init scripts expect a linux terminal (turnkey).
1119     $raw .= "lxc.environment = TERM=linux\n";
1120
1121     my $utsname = $conf->{hostname} || "CT$vmid";
1122     $raw .= "lxc.utsname = $utsname\n";
1123
1124     my $memory = $conf->{memory} || 512;
1125     my $swap = $conf->{swap} // 0;
1126
1127     my $lxcmem = int($memory*1024*1024);
1128     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1129
1130     my $lxcswap = int(($memory + $swap)*1024*1024);
1131     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1132
1133     if (my $cpulimit = $conf->{cpulimit}) {
1134         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1135         my $value = int(100000*$cpulimit);
1136         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1137     }
1138
1139     my $shares = $conf->{cpuunits} || 1024;
1140     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1141
1142     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1143
1144     $raw .= "lxc.rootfs = $dir/rootfs\n";
1145
1146     my $netcount = 0;
1147     foreach my $k (keys %$conf) {
1148         next if $k !~ m/^net(\d+)$/;
1149         my $ind = $1;
1150         my $d = parse_lxc_network($conf->{$k});
1151         $netcount++;
1152         $raw .= "lxc.network.type = veth\n";
1153         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1154         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1155         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1156         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1157     }
1158
1159     if (my $lxcconf = $conf->{lxc}) {
1160         foreach my $entry (@$lxcconf) {
1161             my ($k, $v) = @$entry;
1162             $netcount++ if $k eq 'lxc.network.type';
1163             $raw .= "$k = $v\n";
1164         }
1165     }
1166
1167     $raw .= "lxc.network.type = empty\n" if !$netcount;
1168
1169     File::Path::mkpath("$dir/rootfs");
1170
1171     PVE::Tools::file_set_contents("$dir/config", $raw);
1172 }
1173
1174 # verify and cleanup nameserver list (replace \0 with ' ')
1175 sub verify_nameserver_list {
1176     my ($nameserver_list) = @_;
1177
1178     my @list = ();
1179     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1180         PVE::JSONSchema::pve_verify_ip($server);
1181         push @list, $server;
1182     }
1183
1184     return join(' ', @list);
1185 }
1186
1187 sub verify_searchdomain_list {
1188     my ($searchdomain_list) = @_;
1189
1190     my @list = ();
1191     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1192         # todo: should we add checks for valid dns domains?
1193         push @list, $server;
1194     }
1195
1196     return join(' ', @list);
1197 }
1198
1199 sub add_unused_volume {
1200     my ($config, $volid) = @_;
1201
1202     my $key;
1203     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1204         my $test = "unused$ind";
1205         if (my $vid = $config->{$test}) {
1206             return if $vid eq $volid; # do not add duplicates
1207         } else {
1208             $key = $test;
1209         }
1210     }
1211
1212     die "Too many unused volumes - please delete them first.\n" if !$key;
1213
1214     $config->{$key} = $volid;
1215
1216     return $key;
1217 }
1218
1219 sub update_pct_config {
1220     my ($vmid, $conf, $running, $param, $delete) = @_;
1221
1222     my @nohotplug;
1223
1224     my $new_disks = 0;
1225     my @deleted_volumes;
1226
1227     my $rootdir;
1228     if ($running) {
1229         my $pid = find_lxc_pid($vmid);
1230         $rootdir = "/proc/$pid/root";
1231     }
1232
1233     my $hotplug_error = sub {
1234         if ($running) {
1235             push @nohotplug, @_;
1236             return 1;
1237         } else {
1238             return 0;
1239         }
1240     };
1241
1242     if (defined($delete)) {
1243         foreach my $opt (@$delete) {
1244             if (!exists($conf->{$opt})) {
1245                 warn "no such option: $opt\n";
1246                 next;
1247             }
1248
1249             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1250                 die "unable to delete required option '$opt'\n";
1251             } elsif ($opt eq 'swap') {
1252                 delete $conf->{$opt};
1253                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1254             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1255                 delete $conf->{$opt};
1256             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1257                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1258                 next if $hotplug_error->($opt);
1259                 delete $conf->{$opt};
1260             } elsif ($opt =~ m/^net(\d)$/) {
1261                 delete $conf->{$opt};
1262                 next if !$running;
1263                 my $netid = $1;
1264                 PVE::Network::veth_delete("veth${vmid}i$netid");
1265             } elsif ($opt eq 'protection') {
1266                 delete $conf->{$opt};
1267             } elsif ($opt =~ m/^unused(\d+)$/) {
1268                 next if $hotplug_error->($opt);
1269                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1270                 push @deleted_volumes, $conf->{$opt};
1271                 delete $conf->{$opt};
1272             } elsif ($opt =~ m/^mp(\d+)$/) {
1273                 next if $hotplug_error->($opt);
1274                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1275                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1276                 if ($mountpoint->{type} eq 'volume') {
1277                     add_unused_volume($conf, $mountpoint->{volume})
1278                 }
1279                 delete $conf->{$opt};
1280             } elsif ($opt eq 'unprivileged') {
1281                 die "unable to delete read-only option: '$opt'\n";
1282             } else {
1283                 die "implement me (delete: $opt)"
1284             }
1285             write_config($vmid, $conf) if $running;
1286         }
1287     }
1288
1289     # There's no separate swap size to configure, there's memory and "total"
1290     # memory (iow. memory+swap). This means we have to change them together.
1291     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1292     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1293     if (defined($wanted_memory) || defined($wanted_swap)) {
1294
1295         $wanted_memory //= ($conf->{memory} || 512);
1296         $wanted_swap //=  ($conf->{swap} || 0);
1297
1298         my $total = $wanted_memory + $wanted_swap;
1299         if ($running) {
1300             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1301             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1302         }
1303         $conf->{memory} = $wanted_memory;
1304         $conf->{swap} = $wanted_swap;
1305
1306         write_config($vmid, $conf) if $running;
1307     }
1308
1309     foreach my $opt (keys %$param) {
1310         my $value = $param->{$opt};
1311         if ($opt eq 'hostname') {
1312             $conf->{$opt} = $value;
1313         } elsif ($opt eq 'onboot') {
1314             $conf->{$opt} = $value ? 1 : 0;
1315         } elsif ($opt eq 'startup') {
1316             $conf->{$opt} = $value;
1317         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1318             next if $hotplug_error->($opt);
1319             $conf->{$opt} = $value;
1320         } elsif ($opt eq 'nameserver') {
1321             next if $hotplug_error->($opt);
1322             my $list = verify_nameserver_list($value);
1323             $conf->{$opt} = $list;
1324         } elsif ($opt eq 'searchdomain') {
1325             next if $hotplug_error->($opt);
1326             my $list = verify_searchdomain_list($value);
1327             $conf->{$opt} = $list;
1328         } elsif ($opt eq 'cpulimit') {
1329             next if $hotplug_error->($opt); # FIXME: hotplug
1330             $conf->{$opt} = $value;
1331         } elsif ($opt eq 'cpuunits') {
1332             $conf->{$opt} = $value;
1333             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1334         } elsif ($opt eq 'description') {
1335             $conf->{$opt} = PVE::Tools::encode_text($value);
1336         } elsif ($opt =~ m/^net(\d+)$/) {
1337             my $netid = $1;
1338             my $net = parse_lxc_network($value);
1339             if (!$running) {
1340                 $conf->{$opt} = print_lxc_network($net);
1341             } else {
1342                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1343             }
1344         } elsif ($opt eq 'protection') {
1345             $conf->{$opt} = $value ? 1 : 0;
1346         } elsif ($opt =~ m/^mp(\d+)$/) {
1347             next if $hotplug_error->($opt);
1348             check_protection($conf, "can't update CT $vmid drive '$opt'");
1349             $conf->{$opt} = $value;
1350             $new_disks = 1;
1351         } elsif ($opt eq 'rootfs') {
1352             check_protection($conf, "can't update CT $vmid drive '$opt'");
1353             die "implement me: $opt";
1354         } elsif ($opt eq 'unprivileged') {
1355             die "unable to modify read-only option: '$opt'\n";
1356         } else {
1357             die "implement me: $opt";
1358         }
1359         write_config($vmid, $conf) if $running;
1360     }
1361
1362     if (@deleted_volumes) {
1363         my $storage_cfg = PVE::Storage::config();
1364         foreach my $volume (@deleted_volumes) {
1365             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1366         }
1367     }
1368
1369     if ($new_disks) {
1370         my $storage_cfg = PVE::Storage::config();
1371         create_disks($storage_cfg, $vmid, $conf, $conf);
1372     }
1373
1374     # This should be the last thing we do here
1375     if ($running && scalar(@nohotplug)) {
1376         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1377     }
1378 }
1379
1380 sub has_dev_console {
1381     my ($conf) = @_;
1382
1383     return !(defined($conf->{console}) && !$conf->{console});
1384 }
1385
1386 sub get_tty_count {
1387     my ($conf) = @_;
1388
1389     return $conf->{tty} // $confdesc->{tty}->{default};
1390 }
1391
1392 sub get_cmode {
1393     my ($conf) = @_;
1394
1395     return $conf->{cmode} // $confdesc->{cmode}->{default};
1396 }
1397
1398 sub get_console_command {
1399     my ($vmid, $conf) = @_;
1400
1401     my $cmode = get_cmode($conf);
1402
1403     if ($cmode eq 'console') {
1404         return ['lxc-console', '-n',  $vmid, '-t', 0];
1405     } elsif ($cmode eq 'tty') {
1406         return ['lxc-console', '-n',  $vmid];
1407     } elsif ($cmode eq 'shell') {
1408         return ['lxc-attach', '--clear-env', '-n', $vmid];
1409     } else {
1410         die "internal error";
1411     }
1412 }
1413
1414 sub get_primary_ips {
1415     my ($conf) = @_;
1416
1417     # return data from net0
1418
1419     return undef if !defined($conf->{net0});
1420     my $net = parse_lxc_network($conf->{net0});
1421
1422     my $ipv4 = $net->{ip};
1423     if ($ipv4) {
1424         if ($ipv4 =~ /^(dhcp|manual)$/) {
1425             $ipv4 = undef
1426         } else {
1427             $ipv4 =~ s!/\d+$!!;
1428         }
1429     }
1430     my $ipv6 = $net->{ip6};
1431     if ($ipv6) {
1432         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1433             $ipv6 = undef;
1434         } else {
1435             $ipv6 =~ s!/\d+$!!;
1436         }
1437     }
1438
1439     return ($ipv4, $ipv6);
1440 }
1441
1442 sub delete_mountpoint_volume {
1443     my ($storage_cfg, $vmid, $volume) = @_;
1444
1445     return if classify_mountpoint($volume) ne 'volume';
1446
1447     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1448     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1449 }
1450
1451 sub destroy_lxc_container {
1452     my ($storage_cfg, $vmid, $conf) = @_;
1453
1454     foreach_mountpoint($conf, sub {
1455         my ($ms, $mountpoint) = @_;
1456         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1457     });
1458
1459     rmdir "/var/lib/lxc/$vmid/rootfs";
1460     unlink "/var/lib/lxc/$vmid/config";
1461     rmdir "/var/lib/lxc/$vmid";
1462     destroy_config($vmid);
1463
1464     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1465     #PVE::Tools::run_command($cmd);
1466 }
1467
1468 sub vm_stop_cleanup {
1469     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1470
1471     eval {
1472         if (!$keepActive) {
1473
1474             my $vollist = get_vm_volumes($conf);
1475             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1476         }
1477     };
1478     warn $@ if $@; # avoid errors - just warn
1479 }
1480
1481 my $safe_num_ne = sub {
1482     my ($a, $b) = @_;
1483
1484     return 0 if !defined($a) && !defined($b);
1485     return 1 if !defined($a);
1486     return 1 if !defined($b);
1487
1488     return $a != $b;
1489 };
1490
1491 my $safe_string_ne = sub {
1492     my ($a, $b) = @_;
1493
1494     return 0 if !defined($a) && !defined($b);
1495     return 1 if !defined($a);
1496     return 1 if !defined($b);
1497
1498     return $a ne $b;
1499 };
1500
1501 sub update_net {
1502     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1503
1504     if ($newnet->{type} ne 'veth') {
1505         # for when there are physical interfaces
1506         die "cannot update interface of type $newnet->{type}";
1507     }
1508
1509     my $veth = "veth${vmid}i${netid}";
1510     my $eth = $newnet->{name};
1511
1512     if (my $oldnetcfg = $conf->{$opt}) {
1513         my $oldnet = parse_lxc_network($oldnetcfg);
1514
1515         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1516             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1517
1518             PVE::Network::veth_delete($veth);
1519             delete $conf->{$opt};
1520             write_config($vmid, $conf);
1521
1522             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1523
1524         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1525                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1526                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1527
1528                 if ($oldnet->{bridge}) {
1529                     PVE::Network::tap_unplug($veth);
1530                     foreach (qw(bridge tag firewall)) {
1531                         delete $oldnet->{$_};
1532                     }
1533                     $conf->{$opt} = print_lxc_network($oldnet);
1534                     write_config($vmid, $conf);
1535                 }
1536
1537                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1538                 foreach (qw(bridge tag firewall)) {
1539                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1540                 }
1541                 $conf->{$opt} = print_lxc_network($oldnet);
1542                 write_config($vmid, $conf);
1543         }
1544     } else {
1545         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1546     }
1547
1548     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1549 }
1550
1551 sub hotplug_net {
1552     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1553
1554     my $veth = "veth${vmid}i${netid}";
1555     my $vethpeer = $veth . "p";
1556     my $eth = $newnet->{name};
1557
1558     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1559     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1560
1561     # attach peer in container
1562     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1563     PVE::Tools::run_command($cmd);
1564
1565     # link up peer in container
1566     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1567     PVE::Tools::run_command($cmd);
1568
1569     my $done = { type => 'veth' };
1570     foreach (qw(bridge tag firewall hwaddr name)) {
1571         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1572     }
1573     $conf->{$opt} = print_lxc_network($done);
1574
1575     write_config($vmid, $conf);
1576 }
1577
1578 sub update_ipconfig {
1579     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1580
1581     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1582
1583     my $optdata = parse_lxc_network($conf->{$opt});
1584     my $deleted = [];
1585     my $added = [];
1586     my $nscmd = sub {
1587         my $cmdargs = shift;
1588         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1589     };
1590     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1591
1592     my $change_ip_config = sub {
1593         my ($ipversion) = @_;
1594
1595         my $family_opt = "-$ipversion";
1596         my $suffix = $ipversion == 4 ? '' : $ipversion;
1597         my $gw= "gw$suffix";
1598         my $ip= "ip$suffix";
1599
1600         my $newip = $newnet->{$ip};
1601         my $newgw = $newnet->{$gw};
1602         my $oldip = $optdata->{$ip};
1603
1604         my $change_ip = &$safe_string_ne($oldip, $newip);
1605         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1606
1607         return if !$change_ip && !$change_gw;
1608
1609         # step 1: add new IP, if this fails we cancel
1610         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1611         if ($change_ip && $is_real_ip) {
1612             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1613             if (my $err = $@) {
1614                 warn $err;
1615                 return;
1616             }
1617         }
1618
1619         # step 2: replace gateway
1620         #   If this fails we delete the added IP and cancel.
1621         #   If it succeeds we save the config and delete the old IP, ignoring
1622         #   errors. The config is then saved.
1623         # Note: 'ip route replace' can add
1624         if ($change_gw) {
1625             if ($newgw) {
1626                 eval {
1627                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1628                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1629                     }
1630                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1631                 };
1632                 if (my $err = $@) {
1633                     warn $err;
1634                     # the route was not replaced, the old IP is still available
1635                     # rollback (delete new IP) and cancel
1636                     if ($change_ip) {
1637                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1638                         warn $@ if $@; # no need to die here
1639                     }
1640                     return;
1641                 }
1642             } else {
1643                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1644                 # if the route was not deleted, the guest might have deleted it manually
1645                 # warn and continue
1646                 warn $@ if $@;
1647             }
1648         }
1649
1650         # from this point on we save the configuration
1651         # step 3: delete old IP ignoring errors
1652         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1653             # We need to enable promote_secondaries, otherwise our newly added
1654             # address will be removed along with the old one.
1655             my $promote = 0;
1656             eval {
1657                 if ($ipversion == 4) {
1658                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1659                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1660                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1661                 }
1662                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1663             };
1664             warn $@ if $@; # no need to die here
1665
1666             if ($ipversion == 4) {
1667                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1668             }
1669         }
1670
1671         foreach my $property ($ip, $gw) {
1672             if ($newnet->{$property}) {
1673                 $optdata->{$property} = $newnet->{$property};
1674             } else {
1675                 delete $optdata->{$property};
1676             }
1677         }
1678         $conf->{$opt} = print_lxc_network($optdata);
1679         write_config($vmid, $conf);
1680         $lxc_setup->setup_network($conf);
1681     };
1682
1683     &$change_ip_config(4);
1684     &$change_ip_config(6);
1685
1686 }
1687
1688 # Internal snapshots
1689
1690 # NOTE: Snapshot create/delete involves several non-atomic
1691 # actions, and can take a long time.
1692 # So we try to avoid locking the file and use the 'lock' variable
1693 # inside the config file instead.
1694
1695 my $snapshot_copy_config = sub {
1696     my ($source, $dest) = @_;
1697
1698     foreach my $k (keys %$source) {
1699         next if $k eq 'snapshots';
1700         next if $k eq 'snapstate';
1701         next if $k eq 'snaptime';
1702         next if $k eq 'vmstate';
1703         next if $k eq 'lock';
1704         next if $k eq 'digest';
1705         next if $k eq 'description';
1706
1707         $dest->{$k} = $source->{$k};
1708     }
1709 };
1710
1711 my $snapshot_prepare = sub {
1712     my ($vmid, $snapname, $comment) = @_;
1713
1714     my $snap;
1715
1716     my $updatefn =  sub {
1717
1718         my $conf = load_config($vmid);
1719
1720         die "you can't take a snapshot if it's a template\n"
1721             if is_template($conf);
1722
1723         check_lock($conf);
1724
1725         $conf->{lock} = 'snapshot';
1726
1727         die "snapshot name '$snapname' already used\n"
1728             if defined($conf->{snapshots}->{$snapname});
1729
1730         my $storecfg = PVE::Storage::config();
1731         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1732
1733         $snap = $conf->{snapshots}->{$snapname} = {};
1734
1735         &$snapshot_copy_config($conf, $snap);
1736
1737         $snap->{'snapstate'} = "prepare";
1738         $snap->{'snaptime'} = time();
1739         $snap->{'description'} = $comment if $comment;
1740         $conf->{snapshots}->{$snapname} = $snap;
1741
1742         write_config($vmid, $conf);
1743     };
1744
1745     lock_container($vmid, 10, $updatefn);
1746
1747     return $snap;
1748 };
1749
1750 my $snapshot_commit = sub {
1751     my ($vmid, $snapname) = @_;
1752
1753     my $updatefn = sub {
1754
1755         my $conf = load_config($vmid);
1756
1757         die "missing snapshot lock\n"
1758             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1759
1760         die "snapshot '$snapname' does not exist\n"
1761             if !defined($conf->{snapshots}->{$snapname});
1762
1763         die "wrong snapshot state\n"
1764             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1765                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1766
1767         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1768         delete $conf->{lock};
1769         $conf->{parent} = $snapname;
1770
1771         write_config($vmid, $conf);
1772     };
1773
1774     lock_container($vmid, 10 ,$updatefn);
1775 };
1776
1777 sub has_feature {
1778     my ($feature, $conf, $storecfg, $snapname) = @_;
1779
1780     my $err;
1781
1782     foreach_mountpoint($conf, sub {
1783         my ($ms, $mountpoint) = @_;
1784
1785         return if $err; # skip further test
1786
1787         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1788
1789         # TODO: implement support for mountpoints
1790         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1791             if $ms ne 'rootfs';
1792     });
1793
1794     return $err ? 0 : 1;
1795 }
1796
1797 sub snapshot_create {
1798     my ($vmid, $snapname, $comment) = @_;
1799
1800     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1801
1802     my $conf = load_config($vmid);
1803
1804     my $running = check_running($vmid);
1805
1806     my $unfreeze = 0;
1807
1808     eval {
1809         if ($running) {
1810             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1811             $unfreeze = 1;
1812             PVE::Tools::run_command(['/bin/sync']);
1813         };
1814
1815         my $storecfg = PVE::Storage::config();
1816         my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1817         my $volid = $rootinfo->{volume};
1818
1819         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1820         &$snapshot_commit($vmid, $snapname);
1821     };
1822     my $err = $@;
1823
1824     if ($unfreeze) {
1825         eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
1826         warn $@ if $@;
1827     }
1828
1829     if ($err) {
1830         snapshot_delete($vmid, $snapname, 1);
1831         die "$err\n";
1832     }
1833 }
1834
1835 sub snapshot_delete {
1836     my ($vmid, $snapname, $force) = @_;
1837
1838     my $snap;
1839
1840     my $conf;
1841
1842     my $updatefn =  sub {
1843
1844         $conf = load_config($vmid);
1845
1846         die "you can't delete a snapshot if vm is a template\n"
1847             if is_template($conf);
1848
1849         $snap = $conf->{snapshots}->{$snapname};
1850
1851         check_lock($conf);
1852
1853         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1854
1855         $snap->{snapstate} = 'delete';
1856
1857         write_config($vmid, $conf);
1858     };
1859
1860     lock_container($vmid, 10, $updatefn);
1861
1862     my $storecfg = PVE::Storage::config();
1863
1864     my $unlink_parent = sub {
1865
1866         my ($confref, $new_parent) = @_;
1867
1868         if ($confref->{parent} && $confref->{parent} eq $snapname) {
1869             if ($new_parent) {
1870                 $confref->{parent} = $new_parent;
1871             } else {
1872                 delete $confref->{parent};
1873             }
1874         }
1875     };
1876
1877     my $del_snap =  sub {
1878
1879         check_lock($conf);
1880
1881         my $parent = $conf->{snapshots}->{$snapname}->{parent};
1882         foreach my $snapkey (keys %{$conf->{snapshots}}) {
1883             &$unlink_parent($conf->{snapshots}->{$snapkey}, $parent);
1884         }
1885
1886         &$unlink_parent($conf, $parent);
1887
1888         delete $conf->{snapshots}->{$snapname};
1889
1890         write_config($vmid, $conf);
1891     };
1892
1893     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1894     my $rootinfo = parse_ct_rootfs($rootfs);
1895     my $volid = $rootinfo->{volume};
1896
1897     eval {
1898         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1899     };
1900     my $err = $@;
1901
1902     if(!$err || ($err && $force)) {
1903         lock_container($vmid, 10, $del_snap);
1904         if ($err) {
1905             die "Can't delete snapshot: $vmid $snapname $err\n";
1906         }
1907     }
1908 }
1909
1910 sub snapshot_rollback {
1911     my ($vmid, $snapname) = @_;
1912
1913     my $storecfg = PVE::Storage::config();
1914
1915     my $conf = load_config($vmid);
1916
1917     die "you can't rollback if vm is a template\n" if is_template($conf);
1918
1919     my $snap = $conf->{snapshots}->{$snapname};
1920
1921     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1922
1923     my $rootfs = $snap->{rootfs};
1924     my $rootinfo = parse_ct_rootfs($rootfs);
1925     my $volid = $rootinfo->{volume};
1926
1927     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1928
1929     my $updatefn = sub {
1930
1931         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1932             if $snap->{snapstate};
1933
1934         check_lock($conf);
1935
1936         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1937
1938         die "unable to rollback vm $vmid: vm is running\n"
1939             if check_running($vmid);
1940
1941         $conf->{lock} = 'rollback';
1942
1943         my $forcemachine;
1944
1945         # copy snapshot config to current config
1946
1947         my $tmp_conf = $conf;
1948         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1949         $conf->{snapshots} = $tmp_conf->{snapshots};
1950         delete $conf->{snaptime};
1951         delete $conf->{snapname};
1952         $conf->{parent} = $snapname;
1953
1954         write_config($vmid, $conf);
1955     };
1956
1957     my $unlockfn = sub {
1958         delete $conf->{lock};
1959         write_config($vmid, $conf);
1960     };
1961
1962     lock_container($vmid, 10, $updatefn);
1963
1964     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1965
1966     lock_container($vmid, 5, $unlockfn);
1967 }
1968
1969 sub template_create {
1970     my ($vmid, $conf) = @_;
1971
1972     my $storecfg = PVE::Storage::config();
1973
1974     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1975     my $volid = $rootinfo->{volume};
1976
1977     die "Template feature is not available for '$volid'\n"
1978         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1979
1980     PVE::Storage::activate_volumes($storecfg, [$volid]);
1981
1982     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1983     $rootinfo->{volume} = $template_volid;
1984     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1985
1986     write_config($vmid, $conf);
1987 }
1988
1989 sub is_template {
1990     my ($conf) = @_;
1991
1992     return 1 if defined $conf->{template} && $conf->{template} == 1;
1993 }
1994
1995 sub mountpoint_names {
1996     my ($reverse) = @_;
1997
1998     my @names = ('rootfs');
1999
2000     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
2001         push @names, "mp$i";
2002     }
2003
2004     return $reverse ? reverse @names : @names;
2005 }
2006
2007 # The container might have *different* symlinks than the host. realpath/abs_path
2008 # use the actual filesystem to resolve links.
2009 sub sanitize_mountpoint {
2010     my ($mp) = @_;
2011     $mp = '/' . $mp; # we always start with a slash
2012     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
2013     $mp =~ s@/\./@@g; # collapse /./
2014     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
2015     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
2016     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
2017     return $mp;
2018 }
2019
2020 sub foreach_mountpoint_full {
2021     my ($conf, $reverse, $func) = @_;
2022
2023     foreach my $key (mountpoint_names($reverse)) {
2024         my $value = $conf->{$key};
2025         next if !defined($value);
2026         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2027         next if !defined($mountpoint);
2028
2029         $mountpoint->{mp} = sanitize_mountpoint($mountpoint->{mp});
2030
2031         my $path = $mountpoint->{volume};
2032         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
2033
2034         &$func($key, $mountpoint);
2035     }
2036 }
2037
2038 sub foreach_mountpoint {
2039     my ($conf, $func) = @_;
2040
2041     foreach_mountpoint_full($conf, 0, $func);
2042 }
2043
2044 sub foreach_mountpoint_reverse {
2045     my ($conf, $func) = @_;
2046
2047     foreach_mountpoint_full($conf, 1, $func);
2048 }
2049
2050 sub check_ct_modify_config_perm {
2051     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2052
2053     return 1 if $authuser ne 'root@pam';
2054
2055     foreach my $opt (@$key_list) {
2056
2057         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2058             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2059         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2060             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2061         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2062             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2063         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2064                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2065             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2066         } else {
2067             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2068         }
2069     }
2070
2071     return 1;
2072 }
2073
2074 sub umount_all {
2075     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2076
2077     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2078     my $volid_list = get_vm_volumes($conf);
2079
2080     foreach_mountpoint_reverse($conf, sub {
2081         my ($ms, $mountpoint) = @_;
2082
2083         my $volid = $mountpoint->{volume};
2084         my $mount = $mountpoint->{mp};
2085
2086         return if !$volid || !$mount;
2087
2088         my $mount_path = "$rootdir/$mount";
2089         $mount_path =~ s!/+!/!g;
2090
2091         return if !PVE::ProcFSTools::is_mounted($mount_path);
2092
2093         eval {
2094             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2095         };
2096         if (my $err = $@) {
2097             if ($noerr) {
2098                 warn $err;
2099             } else {
2100                 die $err;
2101             }
2102         }
2103     });
2104 }
2105
2106 sub mount_all {
2107     my ($vmid, $storage_cfg, $conf) = @_;
2108
2109     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2110     File::Path::make_path($rootdir);
2111
2112     my $volid_list = get_vm_volumes($conf);
2113     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2114
2115     eval {
2116         foreach_mountpoint($conf, sub {
2117             my ($ms, $mountpoint) = @_;
2118
2119             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2120         });
2121     };
2122     if (my $err = $@) {
2123         warn "mounting container failed\n";
2124         umount_all($vmid, $storage_cfg, $conf, 1);
2125         die $err;
2126     }
2127
2128     return $rootdir;
2129 }
2130
2131
2132 sub mountpoint_mount_path {
2133     my ($mountpoint, $storage_cfg, $snapname) = @_;
2134
2135     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2136 }
2137
2138 my $check_mount_path = sub {
2139     my ($path) = @_;
2140     $path = File::Spec->canonpath($path);
2141     my $real = Cwd::realpath($path);
2142     if ($real ne $path) {
2143         die "mount path modified by symlink: $path != $real";
2144     }
2145 };
2146
2147 sub query_loopdev {
2148     my ($path) = @_;
2149     my $found;
2150     my $parser = sub {
2151         my $line = shift;
2152         if ($line =~ m@^(/dev/loop\d+):@) {
2153             $found = $1;
2154         }
2155     };
2156     my $cmd = ['losetup', '--associated', $path];
2157     PVE::Tools::run_command($cmd, outfunc => $parser);
2158     return $found;
2159 }
2160
2161 # use $rootdir = undef to just return the corresponding mount path
2162 sub mountpoint_mount {
2163     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2164
2165     my $volid = $mountpoint->{volume};
2166     my $mount = $mountpoint->{mp};
2167     my $type = $mountpoint->{type};
2168
2169     return if !$volid || !$mount;
2170
2171     my $mount_path;
2172
2173     if (defined($rootdir)) {
2174         $rootdir =~ s!/+$!!;
2175         $mount_path = "$rootdir/$mount";
2176         $mount_path =~ s!/+!/!g;
2177         &$check_mount_path($mount_path);
2178         File::Path::mkpath($mount_path);
2179     }
2180
2181     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2182
2183     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2184
2185     if ($storage) {
2186
2187         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2188         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2189
2190         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2191             PVE::Storage::parse_volname($storage_cfg, $volid);
2192
2193         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2194
2195         if ($format eq 'subvol') {
2196             if ($mount_path) {
2197                 if ($snapname) {
2198                     if ($scfg->{type} eq 'zfspool') {
2199                         my $path_arg = $path;
2200                         $path_arg =~ s!^/+!!;
2201                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2202                     } else {
2203                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2204                     }
2205                 } else {
2206                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2207                 }
2208             }
2209             return wantarray ? ($path, 0) : $path;
2210         } elsif ($format eq 'raw' || $format eq 'iso') {
2211             my $use_loopdev = 0;
2212             my @extra_opts;
2213             if ($scfg->{path}) {
2214                 push @extra_opts, '-o', 'loop';
2215                 $use_loopdev = 1;
2216             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2217                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2218                 # do nothing
2219             } else {
2220                 die "unsupported storage type '$scfg->{type}'\n";
2221             }
2222             if ($mount_path) {
2223                 if ($format eq 'iso') {
2224                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2225                 } elsif ($isBase || defined($snapname)) {
2226                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2227                 } else {
2228                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2229                 }
2230             }
2231             return wantarray ? ($path, $use_loopdev) : $path;
2232         } else {
2233             die "unsupported image format '$format'\n";
2234         }
2235     } elsif ($type eq 'device') {
2236         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2237         return wantarray ? ($volid, 0) : $volid;
2238     } elsif ($type eq 'bind') {
2239         die "directory '$volid' does not exist\n" if ! -d $volid;
2240         &$check_mount_path($volid);
2241         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2242         return wantarray ? ($volid, 0) : $volid;
2243     }
2244
2245     die "unsupported storage";
2246 }
2247
2248 sub get_vm_volumes {
2249     my ($conf, $excludes) = @_;
2250
2251     my $vollist = [];
2252
2253     foreach_mountpoint($conf, sub {
2254         my ($ms, $mountpoint) = @_;
2255
2256         return if $excludes && $ms eq $excludes;
2257
2258         my $volid = $mountpoint->{volume};
2259
2260         return if !$volid || $mountpoint->{type} ne 'volume';
2261
2262         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2263         return if !$sid;
2264
2265         push @$vollist, $volid;
2266     });
2267
2268     return $vollist;
2269 }
2270
2271 sub mkfs {
2272     my ($dev, $rootuid, $rootgid) = @_;
2273
2274     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2275                              '-E', "root_owner=$rootuid:$rootgid",
2276                              $dev]);
2277 }
2278
2279 sub format_disk {
2280     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2281
2282     if ($volid =~ m!^/dev/.+!) {
2283         mkfs($volid);
2284         return;
2285     }
2286
2287     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2288
2289     die "cannot format volume '$volid' with no storage\n" if !$storage;
2290
2291     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2292
2293     my $path = PVE::Storage::path($storage_cfg, $volid);
2294
2295     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2296         PVE::Storage::parse_volname($storage_cfg, $volid);
2297
2298     die "cannot format volume '$volid' (format == $format)\n"
2299         if $format ne 'raw';
2300
2301     mkfs($path, $rootuid, $rootgid);
2302 }
2303
2304 sub destroy_disks {
2305     my ($storecfg, $vollist) = @_;
2306
2307     foreach my $volid (@$vollist) {
2308         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2309         warn $@ if $@;
2310     }
2311 }
2312
2313 sub create_disks {
2314     my ($storecfg, $vmid, $settings, $conf) = @_;
2315
2316     my $vollist = [];
2317
2318     eval {
2319         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2320         my $chown_vollist = [];
2321
2322         foreach_mountpoint($settings, sub {
2323             my ($ms, $mountpoint) = @_;
2324
2325             my $volid = $mountpoint->{volume};
2326             my $mp = $mountpoint->{mp};
2327
2328             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2329
2330             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2331                 my ($storeid, $size_gb) = ($1, $2);
2332
2333                 my $size_kb = int(${size_gb}*1024) * 1024;
2334
2335                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2336                 # fixme: use better naming ct-$vmid-disk-X.raw?
2337
2338                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2339                     if ($size_kb > 0) {
2340                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2341                                                            undef, $size_kb);
2342                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2343                     } else {
2344                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2345                                                            undef, 0);
2346                         push @$chown_vollist, $volid;
2347                     }
2348                 } elsif ($scfg->{type} eq 'zfspool') {
2349
2350                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2351                                                        undef, $size_kb);
2352                     push @$chown_vollist, $volid;
2353                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2354
2355                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2356                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2357
2358                 } elsif ($scfg->{type} eq 'rbd') {
2359
2360                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2361                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2362                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2363                 } else {
2364                     die "unable to create containers on storage type '$scfg->{type}'\n";
2365                 }
2366                 push @$vollist, $volid;
2367                 $mountpoint->{volume} = $volid;
2368                 $mountpoint->{size} = $size_kb * 1024;
2369                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2370             } else {
2371                 # use specified/existing volid/dir/device
2372                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2373             }
2374         });
2375
2376         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2377         foreach my $volid (@$chown_vollist) {
2378             my $path = PVE::Storage::path($storecfg, $volid, undef);
2379             chown($rootuid, $rootgid, $path);
2380         }
2381         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2382     };
2383     # free allocated images on error
2384     if (my $err = $@) {
2385         destroy_disks($storecfg, $vollist);
2386         die $err;
2387     }
2388     return $vollist;
2389 }
2390
2391 # bash completion helper
2392
2393 sub complete_os_templates {
2394     my ($cmdname, $pname, $cvalue) = @_;
2395
2396     my $cfg = PVE::Storage::config();
2397
2398     my $storeid;
2399
2400     if ($cvalue =~ m/^([^:]+):/) {
2401         $storeid = $1;
2402     }
2403
2404     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2405     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2406
2407     my $res = [];
2408     foreach my $id (keys %$data) {
2409         foreach my $item (@{$data->{$id}}) {
2410             push @$res, $item->{volid} if defined($item->{volid});
2411         }
2412     }
2413
2414     return $res;
2415 }
2416
2417 my $complete_ctid_full = sub {
2418     my ($running) = @_;
2419
2420     my $idlist = vmstatus();
2421
2422     my $active_hash = list_active_containers();
2423
2424     my $res = [];
2425
2426     foreach my $id (keys %$idlist) {
2427         my $d = $idlist->{$id};
2428         if (defined($running)) {
2429             next if $d->{template};
2430             next if $running && !$active_hash->{$id};
2431             next if !$running && $active_hash->{$id};
2432         }
2433         push @$res, $id;
2434
2435     }
2436     return $res;
2437 };
2438
2439 sub complete_ctid {
2440     return &$complete_ctid_full();
2441 }
2442
2443 sub complete_ctid_stopped {
2444     return &$complete_ctid_full(0);
2445 }
2446
2447 sub complete_ctid_running {
2448     return &$complete_ctid_full(1);
2449 }
2450
2451 sub parse_id_maps {
2452     my ($conf) = @_;
2453
2454     my $id_map = [];
2455     my $rootuid = 0;
2456     my $rootgid = 0;
2457
2458     my $lxc = $conf->{lxc};
2459     foreach my $entry (@$lxc) {
2460         my ($key, $value) = @$entry;
2461         next if $key ne 'lxc.id_map';
2462         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2463             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2464             push @$id_map, [$type, $ct, $host, $length];
2465             if ($ct == 0) {
2466                 $rootuid = $host if $type eq 'u';
2467                 $rootgid = $host if $type eq 'g';
2468             }
2469         } else {
2470             die "failed to parse id_map: $value\n";
2471         }
2472     }
2473
2474     if (!@$id_map && $conf->{unprivileged}) {
2475         # Should we read them from /etc/subuid?
2476         $id_map = [ ['u', '0', '100000', '65536'],
2477                     ['g', '0', '100000', '65536'] ];
2478         $rootuid = $rootgid = 100000;
2479     }
2480
2481     return ($id_map, $rootuid, $rootgid);
2482 }
2483
2484 sub userns_command {
2485     my ($id_map) = @_;
2486     if (@$id_map) {
2487         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2488     }
2489     return [];
2490 }
2491
2492 1;