src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format_description => 'volume',
  42         description => 'Volume, device or directory to mount into the container.',
  43     },
  44     backup => {
  45         type => 'boolean',
  46         format_description => '[1|0]',
  47         description => 'Whether to include the mountpoint in backups.',
  48         optional => 1,
  49     },
  50     size => {
  51         type => 'string',
  52         format => 'disk-size',
  53         format_description => 'DiskSize',
  54         description => 'Volume size (read only value).',
  55         optional => 1,
  56     },
  57 };
  58
  59 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  60     type => 'string', format => $rootfs_desc,
  61     description => "Use volume as container root.",
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 my $confdesc = {
  72     lock => {
  73         optional => 1,
  74         type => 'string',
  75         description => "Lock/unlock the VM.",
  76         enum => [qw(migrate backup snapshot rollback)],
  77     },
  78     onboot => {
  79         optional => 1,
  80         type => 'boolean',
  81         description => "Specifies whether a VM will be started during system bootup.",
  82         default => 0,
  83     },
  84     startup => get_standard_option('pve-startup-order'),
  85     template => {
  86         optional => 1,
  87         type => 'boolean',
  88         description => "Enable/disable Template.",
  89         default => 0,
  90     },
  91     arch => {
  92         optional => 1,
  93         type => 'string',
  94         enum => ['amd64', 'i386'],
  95         description => "OS architecture type.",
  96         default => 'amd64',
  97     },
  98     ostype => {
  99         optional => 1,
 100         type => 'string',
 101         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux'],
 102         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 103     },
 104     console => {
 105         optional => 1,
 106         type => 'boolean',
 107         description => "Attach a console device (/dev/console) to the container.",
 108         default => 1,
 109     },
 110     tty => {
 111         optional => 1,
 112         type => 'integer',
 113         description => "Specify the number of tty available to the container",
 114         minimum => 0,
 115         maximum => 6,
 116         default => 2,
 117     },
 118     cpulimit => {
 119         optional => 1,
 120         type => 'number',
 121         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 122         minimum => 0,
 123         maximum => 128,
 124         default => 0,
 125     },
 126     cpuunits => {
 127         optional => 1,
 128         type => 'integer',
 129         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 130         minimum => 0,
 131         maximum => 500000,
 132         default => 1024,
 133     },
 134     memory => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Amount of RAM for the VM in MB.",
 138         minimum => 16,
 139         default => 512,
 140     },
 141     swap => {
 142         optional => 1,
 143         type => 'integer',
 144         description => "Amount of SWAP for the VM in MB.",
 145         minimum => 0,
 146         default => 512,
 147     },
 148     hostname => {
 149         optional => 1,
 150         description => "Set a host name for the container.",
 151         type => 'string', format => 'dns-name',
 152         maxLength => 255,
 153     },
 154     description => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Container description. Only used on the configuration web interface.",
 158     },
 159     searchdomain => {
 160         optional => 1,
 161         type => 'string', format => 'dns-name-list',
 162         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 163     },
 164     nameserver => {
 165         optional => 1,
 166         type => 'string', format => 'address-list',
 167         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 168     },
 169     rootfs => get_standard_option('pve-ct-rootfs'),
 170     parent => {
 171         optional => 1,
 172         type => 'string', format => 'pve-configid',
 173         maxLength => 40,
 174         description => "Parent snapshot name. This is used internally, and should not be modified.",
 175     },
 176     snaptime => {
 177         optional => 1,
 178         description => "Timestamp for snapshots.",
 179         type => 'integer',
 180         minimum => 0,
 181     },
 182     cmode => {
 183         optional => 1,
 184         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 185         type => 'string',
 186         enum => ['shell', 'console', 'tty'],
 187         default => 'tty',
 188     },
 189     protection => {
 190         optional => 1,
 191         type => 'boolean',
 192         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 193         default => 0,
 194     },
 195     unprivileged => {
 196         optional => 1,
 197         type => 'boolean',
 198         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 199         default => 0,
 200     },
 201 };
 202
 203 my $valid_lxc_conf_keys = {
 204     'lxc.include' => 1,
 205     'lxc.arch' => 1,
 206     'lxc.utsname' => 1,
 207     'lxc.haltsignal' => 1,
 208     'lxc.rebootsignal' => 1,
 209     'lxc.stopsignal' => 1,
 210     'lxc.init_cmd' => 1,
 211     'lxc.network.type' => 1,
 212     'lxc.network.flags' => 1,
 213     'lxc.network.link' => 1,
 214     'lxc.network.mtu' => 1,
 215     'lxc.network.name' => 1,
 216     'lxc.network.hwaddr' => 1,
 217     'lxc.network.ipv4' => 1,
 218     'lxc.network.ipv4.gateway' => 1,
 219     'lxc.network.ipv6' => 1,
 220     'lxc.network.ipv6.gateway' => 1,
 221     'lxc.network.script.up' => 1,
 222     'lxc.network.script.down' => 1,
 223     'lxc.pts' => 1,
 224     'lxc.console.logfile' => 1,
 225     'lxc.console' => 1,
 226     'lxc.tty' => 1,
 227     'lxc.devttydir' => 1,
 228     'lxc.hook.autodev' => 1,
 229     'lxc.autodev' => 1,
 230     'lxc.kmsg' => 1,
 231     'lxc.mount' => 1,
 232     'lxc.mount.entry' => 1,
 233     'lxc.mount.auto' => 1,
 234     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 235     'lxc.rootfs.mount' => 1,
 236     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 237                             ', please use mountpoint options in the "rootfs" key',
 238     # lxc.cgroup.*
 239     'lxc.cap.drop' => 1,
 240     'lxc.cap.keep' => 1,
 241     'lxc.aa_profile' => 1,
 242     'lxc.aa_allow_incomplete' => 1,
 243     'lxc.se_context' => 1,
 244     'lxc.seccomp' => 1,
 245     'lxc.id_map' => 1,
 246     'lxc.hook.pre-start' => 1,
 247     'lxc.hook.pre-mount' => 1,
 248     'lxc.hook.mount' => 1,
 249     'lxc.hook.start' => 1,
 250     'lxc.hook.stop' => 1,
 251     'lxc.hook.post-stop' => 1,
 252     'lxc.hook.clone' => 1,
 253     'lxc.hook.destroy' => 1,
 254     'lxc.loglevel' => 1,
 255     'lxc.logfile' => 1,
 256     'lxc.start.auto' => 1,
 257     'lxc.start.delay' => 1,
 258     'lxc.start.order' => 1,
 259     'lxc.group' => 1,
 260     'lxc.environment' => 1,
 261 };
 262
 263 my $netconf_desc = {
 264     type => {
 265         type => 'string',
 266         optional => 1,
 267         description => "Network interface type.",
 268         enum => [qw(veth)],
 269     },
 270     name => {
 271         type => 'string',
 272         format_description => 'String',
 273         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 274         pattern => '[-_.\w\d]+',
 275     },
 276     bridge => {
 277         type => 'string',
 278         format_description => 'vmbr<Number>',
 279         description => 'Bridge to attach the network device to.',
 280         pattern => '[-_.\w\d]+',
 281         optional => 1,
 282     },
 283     hwaddr => {
 284         type => 'string',
 285         format_description => 'MAC',
 286         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 287         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 288         optional => 1,
 289     },
 290     mtu => {
 291         type => 'integer',
 292         format_description => 'Number',
 293         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 294         minimum => 64, # minimum ethernet frame is 64 bytes
 295         optional => 1,
 296     },
 297     ip => {
 298         type => 'string',
 299         format => 'pve-ipv4-config',
 300         format_description => 'IPv4Format/CIDR',
 301         description => 'IPv4 address in CIDR format.',
 302         optional => 1,
 303     },
 304     gw => {
 305         type => 'string',
 306         format => 'ipv4',
 307         format_description => 'GatewayIPv4',
 308         description => 'Default gateway for IPv4 traffic.',
 309         optional => 1,
 310     },
 311     ip6 => {
 312         type => 'string',
 313         format => 'pve-ipv6-config',
 314         format_description => 'IPv6Format/CIDR',
 315         description => 'IPv6 address in CIDR format.',
 316         optional => 1,
 317     },
 318     gw6 => {
 319         type => 'string',
 320         format => 'ipv6',
 321         format_description => 'GatewayIPv6',
 322         description => 'Default gateway for IPv6 traffic.',
 323         optional => 1,
 324     },
 325     firewall => {
 326         type => 'boolean',
 327         format_description => '[1|0]',
 328         description => "Controls whether this interface's firewall rules should be used.",
 329         optional => 1,
 330     },
 331     tag => {
 332         type => 'integer',
 333         format_description => 'VlanNo',
 334         minimum => '2',
 335         maximum => '4094',
 336         description => "VLAN tag for this interface.",
 337         optional => 1,
 338     },
 339     trunks => {
 340         type => 'string',
 341         pattern => qr/\d+(?:;\d+)*/,
 342         format_description => 'vlanid[;vlanid...]',
 343         description => "VLAN ids to pass through the interface",
 344         optional => 1,
 345     },
 346 };
 347 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 348
 349 my $MAX_LXC_NETWORKS = 10;
 350 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 351     $confdesc->{"net$i"} = {
 352         optional => 1,
 353         type => 'string', format => $netconf_desc,
 354         description => "Specifies network interfaces for the container.",
 355     };
 356 }
 357
 358 my $mp_desc = {
 359     %$rootfs_desc,
 360     mp => {
 361         type => 'string',
 362         format_description => 'Path',
 363         description => 'Path to the mountpoint as seen from inside the container.',
 364     },
 365 };
 366 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 367
 368 my $unuseddesc = {
 369     optional => 1,
 370     type => 'string', format => 'pve-volume-id',
 371     description => "Reference to unused volumes.",
 372 };
 373
 374 my $MAX_MOUNT_POINTS = 10;
 375 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 376     $confdesc->{"mp$i"} = {
 377         optional => 1,
 378         type => 'string', format => $mp_desc,
 379         description => "Use volume as container mount point (experimental feature).",
 380         optional => 1,
 381     };
 382 }
 383
 384 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 385 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 386     $confdesc->{"unused$i"} = $unuseddesc;
 387 }
 388
 389 sub write_pct_config {
 390     my ($filename, $conf) = @_;
 391
 392     delete $conf->{snapstate}; # just to be sure
 393
 394     my $generate_raw_config = sub {
 395         my ($conf) = @_;
 396
 397         my $raw = '';
 398
 399         # add description as comment to top of file
 400         my $descr = $conf->{description} || '';
 401         foreach my $cl (split(/\n/, $descr)) {
 402             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 403         }
 404
 405         foreach my $key (sort keys %$conf) {
 406             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 407                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 408             my $value = $conf->{$key};
 409             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 410             $raw .= "$key: $value\n";
 411         }
 412
 413         if (my $lxcconf = $conf->{lxc}) {
 414             foreach my $entry (@$lxcconf) {
 415                 my ($k, $v) = @$entry;
 416                 $raw .= "$k: $v\n";
 417             }
 418         }
 419
 420         return $raw;
 421     };
 422
 423     my $raw = &$generate_raw_config($conf);
 424
 425     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 426         $raw .= "\n[$snapname]\n";
 427         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 428     }
 429
 430     return $raw;
 431 }
 432
 433 sub check_type {
 434     my ($key, $value) = @_;
 435
 436     die "unknown setting '$key'\n" if !$confdesc->{$key};
 437
 438     my $type = $confdesc->{$key}->{type};
 439
 440     if (!defined($value)) {
 441         die "got undefined value\n";
 442     }
 443
 444     if ($value =~ m/[\n\r]/) {
 445         die "property contains a line feed\n";
 446     }
 447
 448     if ($type eq 'boolean') {
 449         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 450         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 451         die "type check ('boolean') failed - got '$value'\n";
 452     } elsif ($type eq 'integer') {
 453         return int($1) if $value =~ m/^(\d+)$/;
 454         die "type check ('integer') failed - got '$value'\n";
 455     } elsif ($type eq 'number') {
 456         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 457         die "type check ('number') failed - got '$value'\n";
 458     } elsif ($type eq 'string') {
 459         if (my $fmt = $confdesc->{$key}->{format}) {
 460             PVE::JSONSchema::check_format($fmt, $value);
 461             return $value;
 462         }
 463         return $value;
 464     } else {
 465         die "internal error"
 466     }
 467 }
 468
 469 sub parse_pct_config {
 470     my ($filename, $raw) = @_;
 471
 472     return undef if !defined($raw);
 473
 474     my $res = {
 475         digest => Digest::SHA::sha1_hex($raw),
 476         snapshots => {},
 477     };
 478
 479     $filename =~ m|/lxc/(\d+).conf$|
 480         || die "got strange filename '$filename'";
 481
 482     my $vmid = $1;
 483
 484     my $conf = $res;
 485     my $descr = '';
 486     my $section = '';
 487
 488     my @lines = split(/\n/, $raw);
 489     foreach my $line (@lines) {
 490         next if $line =~ m/^\s*$/;
 491
 492         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 493             $section = $1;
 494             $conf->{description} = $descr if $descr;
 495             $descr = '';
 496             $conf = $res->{snapshots}->{$section} = {};
 497             next;
 498         }
 499
 500         if ($line =~ m/^\#(.*)\s*$/) {
 501             $descr .= PVE::Tools::decode_text($1) . "\n";
 502             next;
 503         }
 504
 505         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 506             my $key = $1;
 507             my $value = $3;
 508             if ($valid_lxc_conf_keys->{$key} eq 1 || $key =~ m/^lxc\.cgroup\./) {
 509                 push @{$conf->{lxc}}, [$key, $value];
 510             } elsif (my $errmsg = $valid_lxc_conf_keys->{$key}) {
 511                 warn "vm $vmid - $key: $errmsg\n";
 512             } else {
 513                 warn "vm $vmid - unable to parse config: $line\n";
 514             }
 515         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 516             $descr .= PVE::Tools::decode_text($2);
 517         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 518             $conf->{snapstate} = $1;
 519         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 520             my $key = $1;
 521             my $value = $2;
 522             eval { $value = check_type($key, $value); };
 523             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 524             $conf->{$key} = $value;
 525         } else {
 526             warn "vm $vmid - unable to parse config: $line\n";
 527         }
 528     }
 529
 530     $conf->{description} = $descr if $descr;
 531
 532     delete $res->{snapstate}; # just to be sure
 533
 534     return $res;
 535 }
 536
 537 sub config_list {
 538     my $vmlist = PVE::Cluster::get_vmlist();
 539     my $res = {};
 540     return $res if !$vmlist || !$vmlist->{ids};
 541     my $ids = $vmlist->{ids};
 542
 543     foreach my $vmid (keys %$ids) {
 544         next if !$vmid; # skip CT0
 545         my $d = $ids->{$vmid};
 546         next if !$d->{node} || $d->{node} ne $nodename;
 547         next if !$d->{type} || $d->{type} ne 'lxc';
 548         $res->{$vmid}->{type} = 'lxc';
 549     }
 550     return $res;
 551 }
 552
 553 sub cfs_config_path {
 554     my ($vmid, $node) = @_;
 555
 556     $node = $nodename if !$node;
 557     return "nodes/$node/lxc/$vmid.conf";
 558 }
 559
 560 sub config_file {
 561     my ($vmid, $node) = @_;
 562
 563     my $cfspath = cfs_config_path($vmid, $node);
 564     return "/etc/pve/$cfspath";
 565 }
 566
 567 sub load_config {
 568     my ($vmid, $node) = @_;
 569
 570     $node = $nodename if !$node;
 571     my $cfspath = cfs_config_path($vmid, $node);
 572
 573     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 574     die "container $vmid does not exist\n" if !defined($conf);
 575
 576     return $conf;
 577 }
 578
 579 sub create_config {
 580     my ($vmid, $conf) = @_;
 581
 582     my $dir = "/etc/pve/nodes/$nodename/lxc";
 583     mkdir $dir;
 584
 585     write_config($vmid, $conf);
 586 }
 587
 588 sub destroy_config {
 589     my ($vmid) = @_;
 590
 591     unlink config_file($vmid, $nodename);
 592 }
 593
 594 sub write_config {
 595     my ($vmid, $conf) = @_;
 596
 597     my $cfspath = cfs_config_path($vmid);
 598
 599     PVE::Cluster::cfs_write_file($cfspath, $conf);
 600 }
 601
 602 # flock: we use one file handle per process, so lock file
 603 # can be called multiple times and will succeed for the same process.
 604
 605 my $lock_handles =  {};
 606 my $lockdir = "/run/lock/lxc";
 607
 608 sub lock_filename {
 609     my ($vmid) = @_;
 610
 611     return "$lockdir/pve-config-${vmid}.lock";
 612 }
 613
 614 sub lock_aquire {
 615     my ($vmid, $timeout) = @_;
 616
 617     $timeout = 10 if !$timeout;
 618     my $mode = LOCK_EX;
 619
 620     my $filename = lock_filename($vmid);
 621
 622     mkdir $lockdir if !-d $lockdir;
 623
 624     my $lock_func = sub {
 625         if (!$lock_handles->{$$}->{$filename}) {
 626             my $fh = new IO::File(">>$filename") ||
 627                 die "can't open file - $!\n";
 628             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 629         }
 630
 631         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 632             print STDERR "trying to aquire lock...";
 633             my $success;
 634             while(1) {
 635                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 636                 # try again on EINTR (see bug #273)
 637                 if ($success || ($! != EINTR)) {
 638                     last;
 639                 }
 640             }
 641             if (!$success) {
 642                 print STDERR " failed\n";
 643                 die "can't aquire lock - $!\n";
 644             }
 645
 646             print STDERR " OK\n";
 647         }
 648
 649         $lock_handles->{$$}->{$filename}->{refcount}++;
 650     };
 651
 652     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 653     my $err = $@;
 654     if ($err) {
 655         die "can't lock file '$filename' - $err";
 656     }
 657 }
 658
 659 sub lock_release {
 660     my ($vmid) = @_;
 661
 662     my $filename = lock_filename($vmid);
 663
 664     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 665         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 666         if ($refcount <= 0) {
 667             $lock_handles->{$$}->{$filename} = undef;
 668             close ($fh);
 669         }
 670     }
 671 }
 672
 673 sub lock_container {
 674     my ($vmid, $timeout, $code, @param) = @_;
 675
 676     my $res;
 677
 678     lock_aquire($vmid, $timeout);
 679     eval { $res = &$code(@param) };
 680     my $err = $@;
 681     lock_release($vmid);
 682
 683     die $err if $err;
 684
 685     return $res;
 686 }
 687
 688 sub option_exists {
 689     my ($name) = @_;
 690
 691     return defined($confdesc->{$name});
 692 }
 693
 694 # add JSON properties for create and set function
 695 sub json_config_properties {
 696     my $prop = shift;
 697
 698     foreach my $opt (keys %$confdesc) {
 699         next if $opt eq 'parent' || $opt eq 'snaptime';
 700         next if $prop->{$opt};
 701         $prop->{$opt} = $confdesc->{$opt};
 702     }
 703
 704     return $prop;
 705 }
 706
 707 sub json_config_properties_no_rootfs {
 708     my $prop = shift;
 709
 710     foreach my $opt (keys %$confdesc) {
 711         next if $prop->{$opt};
 712         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 713         $prop->{$opt} = $confdesc->{$opt};
 714     }
 715
 716     return $prop;
 717 }
 718
 719 # container status helpers
 720
 721 sub list_active_containers {
 722
 723     my $filename = "/proc/net/unix";
 724
 725     # similar test is used by lcxcontainers.c: list_active_containers
 726     my $res = {};
 727
 728     my $fh = IO::File->new ($filename, "r");
 729     return $res if !$fh;
 730
 731     while (defined(my $line = <$fh>)) {
 732         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 733             my $path = $1;
 734             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 735                 $res->{$1} = 1;
 736             }
 737         }
 738     }
 739
 740     close($fh);
 741
 742     return $res;
 743 }
 744
 745 # warning: this is slow
 746 sub check_running {
 747     my ($vmid) = @_;
 748
 749     my $active_hash = list_active_containers();
 750
 751     return 1 if defined($active_hash->{$vmid});
 752
 753     return undef;
 754 }
 755
 756 sub get_container_disk_usage {
 757     my ($vmid, $pid) = @_;
 758
 759     return PVE::Tools::df("/proc/$pid/root/", 1);
 760 }
 761
 762 my $last_proc_vmid_stat;
 763
 764 my $parse_cpuacct_stat = sub {
 765     my ($vmid) = @_;
 766
 767     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 768
 769     my $stat = {};
 770
 771     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 772
 773         $stat->{utime} = $1;
 774         $stat->{stime} = $2;
 775
 776     }
 777
 778     return $stat;
 779 };
 780
 781 sub vmstatus {
 782     my ($opt_vmid) = @_;
 783
 784     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 785
 786     my $active_hash = list_active_containers();
 787
 788     my $cpucount = $cpuinfo->{cpus} || 1;
 789
 790     my $cdtime = gettimeofday;
 791
 792     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 793
 794     foreach my $vmid (keys %$list) {
 795         my $d = $list->{$vmid};
 796
 797         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 798         warn $@ if $@; # ignore errors (consider them stopped)
 799
 800         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 801
 802         my $cfspath = cfs_config_path($vmid);
 803         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 804
 805         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 806         $d->{name} =~ s/[\s]//g;
 807
 808         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 809
 810         if ($d->{pid}) {
 811             my $res = get_container_disk_usage($vmid, $d->{pid});
 812             $d->{disk} = $res->{used};
 813             $d->{maxdisk} = $res->{total};
 814         } else {
 815             $d->{disk} = 0;
 816             # use 4GB by default ??
 817             if (my $rootfs = $conf->{rootfs}) {
 818                 my $rootinfo = parse_ct_rootfs($rootfs);
 819                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 820             } else {
 821                 $d->{maxdisk} = 4*1024*1024*1024;
 822             }
 823         }
 824
 825         $d->{mem} = 0;
 826         $d->{swap} = 0;
 827         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 828         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 829
 830         $d->{uptime} = 0;
 831         $d->{cpu} = 0;
 832
 833         $d->{netout} = 0;
 834         $d->{netin} = 0;
 835
 836         $d->{diskread} = 0;
 837         $d->{diskwrite} = 0;
 838
 839         $d->{template} = is_template($conf);
 840     }
 841
 842     foreach my $vmid (keys %$list) {
 843         my $d = $list->{$vmid};
 844         my $pid = $d->{pid};
 845
 846         next if !$pid; # skip stopped CTs
 847
 848         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 849         $d->{uptime} = time - $ctime; # the method lxcfs uses
 850
 851         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 852         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 853
 854         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 855         my @bytes = split(/\n/, $blkio_bytes);
 856         foreach my $byte (@bytes) {
 857             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 858                 $d->{diskread} = $2 if $key eq 'Read';
 859                 $d->{diskwrite} = $2 if $key eq 'Write';
 860             }
 861         }
 862
 863         my $pstat = &$parse_cpuacct_stat($vmid);
 864
 865         my $used = $pstat->{utime} + $pstat->{stime};
 866
 867         my $old = $last_proc_vmid_stat->{$vmid};
 868         if (!$old) {
 869             $last_proc_vmid_stat->{$vmid} = {
 870                 time => $cdtime,
 871                 used => $used,
 872                 cpu => 0,
 873             };
 874             next;
 875         }
 876
 877         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 878
 879         if ($dtime > 1000) {
 880             my $dutime = $used -  $old->{used};
 881
 882             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 883             $last_proc_vmid_stat->{$vmid} = {
 884                 time => $cdtime,
 885                 used => $used,
 886                 cpu => $d->{cpu},
 887             };
 888         } else {
 889             $d->{cpu} = $old->{cpu};
 890         }
 891     }
 892
 893     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 894
 895     foreach my $dev (keys %$netdev) {
 896         next if $dev !~ m/^veth([1-9]\d*)i/;
 897         my $vmid = $1;
 898         my $d = $list->{$vmid};
 899
 900         next if !$d;
 901
 902         $d->{netout} += $netdev->{$dev}->{receive};
 903         $d->{netin} += $netdev->{$dev}->{transmit};
 904
 905     }
 906
 907     return $list;
 908 }
 909
 910 sub classify_mountpoint {
 911     my ($vol) = @_;
 912     if ($vol =~ m!^/!) {
 913         return 'device' if $vol =~ m!^/dev/!;
 914         return 'bind';
 915     }
 916     return 'volume';
 917 }
 918
 919 my $parse_ct_mountpoint_full = sub {
 920     my ($desc, $data, $noerr) = @_;
 921
 922     $data //= '';
 923
 924     my $res;
 925     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 926     if ($@) {
 927         return undef if $noerr;
 928         die $@;
 929     }
 930
 931     if (defined(my $size = $res->{size})) {
 932         $size = PVE::JSONSchema::parse_size($size);
 933         if (!defined($size)) {
 934             return undef if $noerr;
 935             die "invalid size: $size\n";
 936         }
 937         $res->{size} = $size;
 938     }
 939
 940     $res->{type} = classify_mountpoint($res->{volume});
 941
 942     return $res;
 943 };
 944
 945 sub parse_ct_rootfs {
 946     my ($data, $noerr) = @_;
 947
 948     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 949
 950     $res->{mp} = '/' if defined($res);
 951
 952     return $res;
 953 }
 954
 955 sub parse_ct_mountpoint {
 956     my ($data, $noerr) = @_;
 957
 958     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 959 }
 960
 961 sub print_ct_mountpoint {
 962     my ($info, $nomp) = @_;
 963     my $skip = [ 'type' ];
 964     push @$skip, 'mp' if $nomp;
 965     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 966 }
 967
 968 sub print_lxc_network {
 969     my $net = shift;
 970     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 971 }
 972
 973 sub parse_lxc_network {
 974     my ($data) = @_;
 975
 976     my $res = {};
 977
 978     return $res if !$data;
 979
 980     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 981
 982     $res->{type} = 'veth';
 983     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 984
 985     return $res;
 986 }
 987
 988 sub read_cgroup_value {
 989     my ($group, $vmid, $name, $full) = @_;
 990
 991     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 992
 993     return PVE::Tools::file_get_contents($path) if $full;
 994
 995     return PVE::Tools::file_read_firstline($path);
 996 }
 997
 998 sub write_cgroup_value {
 999    my ($group, $vmid, $name, $value) = @_;
1000
1001    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
1002    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
1003
1004 }
1005
1006 sub find_lxc_console_pids {
1007
1008     my $res = {};
1009
1010     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
1011         my ($pid) = @_;
1012
1013         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
1014         return if !$cmdline;
1015
1016         my @args = split(/\0/, $cmdline);
1017
1018         # search for lxc-console -n <vmid>
1019         return if scalar(@args) != 3;
1020         return if $args[1] ne '-n';
1021         return if $args[2] !~ m/^\d+$/;
1022         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1023
1024         my $vmid = $args[2];
1025
1026         push @{$res->{$vmid}}, $pid;
1027     });
1028
1029     return $res;
1030 }
1031
1032 sub find_lxc_pid {
1033     my ($vmid) = @_;
1034
1035     my $pid = undef;
1036     my $parser = sub {
1037         my $line = shift;
1038         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1039     };
1040     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1041
1042     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1043
1044     return $pid;
1045 }
1046
1047 # Note: we cannot use Net:IP, because that only allows strict
1048 # CIDR networks
1049 sub parse_ipv4_cidr {
1050     my ($cidr, $noerr) = @_;
1051
1052     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1053         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1054     }
1055
1056     return undef if $noerr;
1057
1058     die "unable to parse ipv4 address/mask\n";
1059 }
1060
1061 sub check_lock {
1062     my ($conf) = @_;
1063
1064     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1065 }
1066
1067 sub check_protection {
1068     my ($vm_conf, $err_msg) = @_;
1069
1070     if ($vm_conf->{protection}) {
1071         die "$err_msg - protection mode enabled\n";
1072     }
1073 }
1074
1075 sub update_lxc_config {
1076     my ($storage_cfg, $vmid, $conf) = @_;
1077
1078     my $dir = "/var/lib/lxc/$vmid";
1079
1080     if ($conf->{template}) {
1081
1082         unlink "$dir/config";
1083
1084         return;
1085     }
1086
1087     my $raw = '';
1088
1089     die "missing 'arch' - internal error" if !$conf->{arch};
1090     $raw .= "lxc.arch = $conf->{arch}\n";
1091
1092     my $unprivileged = $conf->{unprivileged};
1093     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1094
1095     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1096     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux)$/x) {
1097         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1098         if ($unprivileged || $custom_idmap) {
1099             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1100         }
1101     } else {
1102         die "implement me (ostype $ostype)";
1103     }
1104
1105     $raw .= "lxc.monitor.unshare = 1\n";
1106
1107     # Should we read them from /etc/subuid?
1108     if ($unprivileged && !$custom_idmap) {
1109         $raw .= "lxc.id_map = u 0 100000 65536\n";
1110         $raw .= "lxc.id_map = g 0 100000 65536\n";
1111     }
1112
1113     if (!has_dev_console($conf)) {
1114         $raw .= "lxc.console = none\n";
1115         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1116     }
1117
1118     my $ttycount = get_tty_count($conf);
1119     $raw .= "lxc.tty = $ttycount\n";
1120
1121     # some init scripts expect a linux terminal (turnkey).
1122     $raw .= "lxc.environment = TERM=linux\n";
1123
1124     my $utsname = $conf->{hostname} || "CT$vmid";
1125     $raw .= "lxc.utsname = $utsname\n";
1126
1127     my $memory = $conf->{memory} || 512;
1128     my $swap = $conf->{swap} // 0;
1129
1130     my $lxcmem = int($memory*1024*1024);
1131     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1132
1133     my $lxcswap = int(($memory + $swap)*1024*1024);
1134     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1135
1136     if (my $cpulimit = $conf->{cpulimit}) {
1137         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1138         my $value = int(100000*$cpulimit);
1139         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1140     }
1141
1142     my $shares = $conf->{cpuunits} || 1024;
1143     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1144
1145     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1146
1147     $raw .= "lxc.rootfs = $dir/rootfs\n";
1148
1149     my $netcount = 0;
1150     foreach my $k (keys %$conf) {
1151         next if $k !~ m/^net(\d+)$/;
1152         my $ind = $1;
1153         my $d = parse_lxc_network($conf->{$k});
1154         $netcount++;
1155         $raw .= "lxc.network.type = veth\n";
1156         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1157         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1158         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1159         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1160     }
1161
1162     if (my $lxcconf = $conf->{lxc}) {
1163         foreach my $entry (@$lxcconf) {
1164             my ($k, $v) = @$entry;
1165             $netcount++ if $k eq 'lxc.network.type';
1166             $raw .= "$k = $v\n";
1167         }
1168     }
1169
1170     $raw .= "lxc.network.type = empty\n" if !$netcount;
1171
1172     File::Path::mkpath("$dir/rootfs");
1173
1174     PVE::Tools::file_set_contents("$dir/config", $raw);
1175 }
1176
1177 # verify and cleanup nameserver list (replace \0 with ' ')
1178 sub verify_nameserver_list {
1179     my ($nameserver_list) = @_;
1180
1181     my @list = ();
1182     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1183         PVE::JSONSchema::pve_verify_ip($server);
1184         push @list, $server;
1185     }
1186
1187     return join(' ', @list);
1188 }
1189
1190 sub verify_searchdomain_list {
1191     my ($searchdomain_list) = @_;
1192
1193     my @list = ();
1194     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1195         # todo: should we add checks for valid dns domains?
1196         push @list, $server;
1197     }
1198
1199     return join(' ', @list);
1200 }
1201
1202 sub add_unused_volume {
1203     my ($config, $volid) = @_;
1204
1205     my $key;
1206     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1207         my $test = "unused$ind";
1208         if (my $vid = $config->{$test}) {
1209             return if $vid eq $volid; # do not add duplicates
1210         } else {
1211             $key = $test;
1212         }
1213     }
1214
1215     die "Too many unused volumes - please delete them first.\n" if !$key;
1216
1217     $config->{$key} = $volid;
1218
1219     return $key;
1220 }
1221
1222 sub update_pct_config {
1223     my ($vmid, $conf, $running, $param, $delete) = @_;
1224
1225     my @nohotplug;
1226
1227     my $new_disks = 0;
1228     my @deleted_volumes;
1229
1230     my $rootdir;
1231     if ($running) {
1232         my $pid = find_lxc_pid($vmid);
1233         $rootdir = "/proc/$pid/root";
1234     }
1235
1236     my $hotplug_error = sub {
1237         if ($running) {
1238             push @nohotplug, @_;
1239             return 1;
1240         } else {
1241             return 0;
1242         }
1243     };
1244
1245     if (defined($delete)) {
1246         foreach my $opt (@$delete) {
1247             if (!exists($conf->{$opt})) {
1248                 warn "no such option: $opt\n";
1249                 next;
1250             }
1251
1252             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1253                 die "unable to delete required option '$opt'\n";
1254             } elsif ($opt eq 'swap') {
1255                 delete $conf->{$opt};
1256                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1257             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1258                 delete $conf->{$opt};
1259             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1260                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1261                 next if $hotplug_error->($opt);
1262                 delete $conf->{$opt};
1263             } elsif ($opt =~ m/^net(\d)$/) {
1264                 delete $conf->{$opt};
1265                 next if !$running;
1266                 my $netid = $1;
1267                 PVE::Network::veth_delete("veth${vmid}i$netid");
1268             } elsif ($opt eq 'protection') {
1269                 delete $conf->{$opt};
1270             } elsif ($opt =~ m/^unused(\d+)$/) {
1271                 next if $hotplug_error->($opt);
1272                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1273                 push @deleted_volumes, $conf->{$opt};
1274                 delete $conf->{$opt};
1275             } elsif ($opt =~ m/^mp(\d+)$/) {
1276                 next if $hotplug_error->($opt);
1277                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1278                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1279                 if ($mountpoint->{type} eq 'volume') {
1280                     add_unused_volume($conf, $mountpoint->{volume})
1281                 }
1282                 delete $conf->{$opt};
1283             } elsif ($opt eq 'unprivileged') {
1284                 die "unable to delete read-only option: '$opt'\n";
1285             } else {
1286                 die "implement me (delete: $opt)"
1287             }
1288             write_config($vmid, $conf) if $running;
1289         }
1290     }
1291
1292     # There's no separate swap size to configure, there's memory and "total"
1293     # memory (iow. memory+swap). This means we have to change them together.
1294     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1295     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1296     if (defined($wanted_memory) || defined($wanted_swap)) {
1297
1298         $wanted_memory //= ($conf->{memory} || 512);
1299         $wanted_swap //=  ($conf->{swap} || 0);
1300
1301         my $total = $wanted_memory + $wanted_swap;
1302         if ($running) {
1303             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1304             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1305         }
1306         $conf->{memory} = $wanted_memory;
1307         $conf->{swap} = $wanted_swap;
1308
1309         write_config($vmid, $conf) if $running;
1310     }
1311
1312     foreach my $opt (keys %$param) {
1313         my $value = $param->{$opt};
1314         if ($opt eq 'hostname') {
1315             $conf->{$opt} = $value;
1316         } elsif ($opt eq 'onboot') {
1317             $conf->{$opt} = $value ? 1 : 0;
1318         } elsif ($opt eq 'startup') {
1319             $conf->{$opt} = $value;
1320         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1321             next if $hotplug_error->($opt);
1322             $conf->{$opt} = $value;
1323         } elsif ($opt eq 'nameserver') {
1324             next if $hotplug_error->($opt);
1325             my $list = verify_nameserver_list($value);
1326             $conf->{$opt} = $list;
1327         } elsif ($opt eq 'searchdomain') {
1328             next if $hotplug_error->($opt);
1329             my $list = verify_searchdomain_list($value);
1330             $conf->{$opt} = $list;
1331         } elsif ($opt eq 'cpulimit') {
1332             next if $hotplug_error->($opt); # FIXME: hotplug
1333             $conf->{$opt} = $value;
1334         } elsif ($opt eq 'cpuunits') {
1335             $conf->{$opt} = $value;
1336             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1337         } elsif ($opt eq 'description') {
1338             $conf->{$opt} = PVE::Tools::encode_text($value);
1339         } elsif ($opt =~ m/^net(\d+)$/) {
1340             my $netid = $1;
1341             my $net = parse_lxc_network($value);
1342             if (!$running) {
1343                 $conf->{$opt} = print_lxc_network($net);
1344             } else {
1345                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1346             }
1347         } elsif ($opt eq 'protection') {
1348             $conf->{$opt} = $value ? 1 : 0;
1349         } elsif ($opt =~ m/^mp(\d+)$/) {
1350             next if $hotplug_error->($opt);
1351             check_protection($conf, "can't update CT $vmid drive '$opt'");
1352             $conf->{$opt} = $value;
1353             $new_disks = 1;
1354         } elsif ($opt eq 'rootfs') {
1355             check_protection($conf, "can't update CT $vmid drive '$opt'");
1356             die "implement me: $opt";
1357         } elsif ($opt eq 'unprivileged') {
1358             die "unable to modify read-only option: '$opt'\n";
1359         } else {
1360             die "implement me: $opt";
1361         }
1362         write_config($vmid, $conf) if $running;
1363     }
1364
1365     if (@deleted_volumes) {
1366         my $storage_cfg = PVE::Storage::config();
1367         foreach my $volume (@deleted_volumes) {
1368             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1369         }
1370     }
1371
1372     if ($new_disks) {
1373         my $storage_cfg = PVE::Storage::config();
1374         create_disks($storage_cfg, $vmid, $conf, $conf);
1375     }
1376
1377     # This should be the last thing we do here
1378     if ($running && scalar(@nohotplug)) {
1379         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1380     }
1381 }
1382
1383 sub has_dev_console {
1384     my ($conf) = @_;
1385
1386     return !(defined($conf->{console}) && !$conf->{console});
1387 }
1388
1389 sub get_tty_count {
1390     my ($conf) = @_;
1391
1392     return $conf->{tty} // $confdesc->{tty}->{default};
1393 }
1394
1395 sub get_cmode {
1396     my ($conf) = @_;
1397
1398     return $conf->{cmode} // $confdesc->{cmode}->{default};
1399 }
1400
1401 sub get_console_command {
1402     my ($vmid, $conf) = @_;
1403
1404     my $cmode = get_cmode($conf);
1405
1406     if ($cmode eq 'console') {
1407         return ['lxc-console', '-n',  $vmid, '-t', 0];
1408     } elsif ($cmode eq 'tty') {
1409         return ['lxc-console', '-n',  $vmid];
1410     } elsif ($cmode eq 'shell') {
1411         return ['lxc-attach', '--clear-env', '-n', $vmid];
1412     } else {
1413         die "internal error";
1414     }
1415 }
1416
1417 sub get_primary_ips {
1418     my ($conf) = @_;
1419
1420     # return data from net0
1421
1422     return undef if !defined($conf->{net0});
1423     my $net = parse_lxc_network($conf->{net0});
1424
1425     my $ipv4 = $net->{ip};
1426     if ($ipv4) {
1427         if ($ipv4 =~ /^(dhcp|manual)$/) {
1428             $ipv4 = undef
1429         } else {
1430             $ipv4 =~ s!/\d+$!!;
1431         }
1432     }
1433     my $ipv6 = $net->{ip6};
1434     if ($ipv6) {
1435         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1436             $ipv6 = undef;
1437         } else {
1438             $ipv6 =~ s!/\d+$!!;
1439         }
1440     }
1441
1442     return ($ipv4, $ipv6);
1443 }
1444
1445 sub delete_mountpoint_volume {
1446     my ($storage_cfg, $vmid, $volume) = @_;
1447
1448     return if classify_mountpoint($volume) ne 'volume';
1449
1450     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1451     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1452 }
1453
1454 sub destroy_lxc_container {
1455     my ($storage_cfg, $vmid, $conf) = @_;
1456
1457     foreach_mountpoint($conf, sub {
1458         my ($ms, $mountpoint) = @_;
1459         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1460     });
1461
1462     rmdir "/var/lib/lxc/$vmid/rootfs";
1463     unlink "/var/lib/lxc/$vmid/config";
1464     rmdir "/var/lib/lxc/$vmid";
1465     destroy_config($vmid);
1466
1467     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1468     #PVE::Tools::run_command($cmd);
1469 }
1470
1471 sub vm_stop_cleanup {
1472     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1473
1474     eval {
1475         if (!$keepActive) {
1476
1477             my $vollist = get_vm_volumes($conf);
1478             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1479         }
1480     };
1481     warn $@ if $@; # avoid errors - just warn
1482 }
1483
1484 my $safe_num_ne = sub {
1485     my ($a, $b) = @_;
1486
1487     return 0 if !defined($a) && !defined($b);
1488     return 1 if !defined($a);
1489     return 1 if !defined($b);
1490
1491     return $a != $b;
1492 };
1493
1494 my $safe_string_ne = sub {
1495     my ($a, $b) = @_;
1496
1497     return 0 if !defined($a) && !defined($b);
1498     return 1 if !defined($a);
1499     return 1 if !defined($b);
1500
1501     return $a ne $b;
1502 };
1503
1504 sub update_net {
1505     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1506
1507     if ($newnet->{type} ne 'veth') {
1508         # for when there are physical interfaces
1509         die "cannot update interface of type $newnet->{type}";
1510     }
1511
1512     my $veth = "veth${vmid}i${netid}";
1513     my $eth = $newnet->{name};
1514
1515     if (my $oldnetcfg = $conf->{$opt}) {
1516         my $oldnet = parse_lxc_network($oldnetcfg);
1517
1518         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1519             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1520
1521             PVE::Network::veth_delete($veth);
1522             delete $conf->{$opt};
1523             write_config($vmid, $conf);
1524
1525             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1526
1527         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1528                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1529                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1530
1531                 if ($oldnet->{bridge}) {
1532                     PVE::Network::tap_unplug($veth);
1533                     foreach (qw(bridge tag firewall)) {
1534                         delete $oldnet->{$_};
1535                     }
1536                     $conf->{$opt} = print_lxc_network($oldnet);
1537                     write_config($vmid, $conf);
1538                 }
1539
1540                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1541                 foreach (qw(bridge tag firewall)) {
1542                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1543                 }
1544                 $conf->{$opt} = print_lxc_network($oldnet);
1545                 write_config($vmid, $conf);
1546         }
1547     } else {
1548         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1549     }
1550
1551     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1552 }
1553
1554 sub hotplug_net {
1555     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1556
1557     my $veth = "veth${vmid}i${netid}";
1558     my $vethpeer = $veth . "p";
1559     my $eth = $newnet->{name};
1560
1561     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1562     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1563
1564     # attach peer in container
1565     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1566     PVE::Tools::run_command($cmd);
1567
1568     # link up peer in container
1569     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1570     PVE::Tools::run_command($cmd);
1571
1572     my $done = { type => 'veth' };
1573     foreach (qw(bridge tag firewall hwaddr name)) {
1574         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1575     }
1576     $conf->{$opt} = print_lxc_network($done);
1577
1578     write_config($vmid, $conf);
1579 }
1580
1581 sub update_ipconfig {
1582     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1583
1584     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1585
1586     my $optdata = parse_lxc_network($conf->{$opt});
1587     my $deleted = [];
1588     my $added = [];
1589     my $nscmd = sub {
1590         my $cmdargs = shift;
1591         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1592     };
1593     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1594
1595     my $change_ip_config = sub {
1596         my ($ipversion) = @_;
1597
1598         my $family_opt = "-$ipversion";
1599         my $suffix = $ipversion == 4 ? '' : $ipversion;
1600         my $gw= "gw$suffix";
1601         my $ip= "ip$suffix";
1602
1603         my $newip = $newnet->{$ip};
1604         my $newgw = $newnet->{$gw};
1605         my $oldip = $optdata->{$ip};
1606
1607         my $change_ip = &$safe_string_ne($oldip, $newip);
1608         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1609
1610         return if !$change_ip && !$change_gw;
1611
1612         # step 1: add new IP, if this fails we cancel
1613         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1614         if ($change_ip && $is_real_ip) {
1615             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1616             if (my $err = $@) {
1617                 warn $err;
1618                 return;
1619             }
1620         }
1621
1622         # step 2: replace gateway
1623         #   If this fails we delete the added IP and cancel.
1624         #   If it succeeds we save the config and delete the old IP, ignoring
1625         #   errors. The config is then saved.
1626         # Note: 'ip route replace' can add
1627         if ($change_gw) {
1628             if ($newgw) {
1629                 eval {
1630                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1631                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1632                     }
1633                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1634                 };
1635                 if (my $err = $@) {
1636                     warn $err;
1637                     # the route was not replaced, the old IP is still available
1638                     # rollback (delete new IP) and cancel
1639                     if ($change_ip) {
1640                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1641                         warn $@ if $@; # no need to die here
1642                     }
1643                     return;
1644                 }
1645             } else {
1646                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1647                 # if the route was not deleted, the guest might have deleted it manually
1648                 # warn and continue
1649                 warn $@ if $@;
1650             }
1651         }
1652
1653         # from this point on we save the configuration
1654         # step 3: delete old IP ignoring errors
1655         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1656             # We need to enable promote_secondaries, otherwise our newly added
1657             # address will be removed along with the old one.
1658             my $promote = 0;
1659             eval {
1660                 if ($ipversion == 4) {
1661                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1662                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1663                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1664                 }
1665                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1666             };
1667             warn $@ if $@; # no need to die here
1668
1669             if ($ipversion == 4) {
1670                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1671             }
1672         }
1673
1674         foreach my $property ($ip, $gw) {
1675             if ($newnet->{$property}) {
1676                 $optdata->{$property} = $newnet->{$property};
1677             } else {
1678                 delete $optdata->{$property};
1679             }
1680         }
1681         $conf->{$opt} = print_lxc_network($optdata);
1682         write_config($vmid, $conf);
1683         $lxc_setup->setup_network($conf);
1684     };
1685
1686     &$change_ip_config(4);
1687     &$change_ip_config(6);
1688
1689 }
1690
1691 # Internal snapshots
1692
1693 # NOTE: Snapshot create/delete involves several non-atomic
1694 # actions, and can take a long time.
1695 # So we try to avoid locking the file and use the 'lock' variable
1696 # inside the config file instead.
1697
1698 my $snapshot_copy_config = sub {
1699     my ($source, $dest) = @_;
1700
1701     foreach my $k (keys %$source) {
1702         next if $k eq 'snapshots';
1703         next if $k eq 'snapstate';
1704         next if $k eq 'snaptime';
1705         next if $k eq 'vmstate';
1706         next if $k eq 'lock';
1707         next if $k eq 'digest';
1708         next if $k eq 'description';
1709
1710         $dest->{$k} = $source->{$k};
1711     }
1712 };
1713
1714 my $snapshot_prepare = sub {
1715     my ($vmid, $snapname, $comment) = @_;
1716
1717     my $snap;
1718
1719     my $updatefn =  sub {
1720
1721         my $conf = load_config($vmid);
1722
1723         die "you can't take a snapshot if it's a template\n"
1724             if is_template($conf);
1725
1726         check_lock($conf);
1727
1728         $conf->{lock} = 'snapshot';
1729
1730         die "snapshot name '$snapname' already used\n"
1731             if defined($conf->{snapshots}->{$snapname});
1732
1733         my $storecfg = PVE::Storage::config();
1734         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1735
1736         $snap = $conf->{snapshots}->{$snapname} = {};
1737
1738         &$snapshot_copy_config($conf, $snap);
1739
1740         $snap->{'snapstate'} = "prepare";
1741         $snap->{'snaptime'} = time();
1742         $snap->{'description'} = $comment if $comment;
1743         $conf->{snapshots}->{$snapname} = $snap;
1744
1745         write_config($vmid, $conf);
1746     };
1747
1748     lock_container($vmid, 10, $updatefn);
1749
1750     return $snap;
1751 };
1752
1753 my $snapshot_commit = sub {
1754     my ($vmid, $snapname) = @_;
1755
1756     my $updatefn = sub {
1757
1758         my $conf = load_config($vmid);
1759
1760         die "missing snapshot lock\n"
1761             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1762
1763         die "snapshot '$snapname' does not exist\n"
1764             if !defined($conf->{snapshots}->{$snapname});
1765
1766         die "wrong snapshot state\n"
1767             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1768                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1769
1770         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1771         delete $conf->{lock};
1772         $conf->{parent} = $snapname;
1773
1774         write_config($vmid, $conf);
1775     };
1776
1777     lock_container($vmid, 10 ,$updatefn);
1778 };
1779
1780 sub has_feature {
1781     my ($feature, $conf, $storecfg, $snapname) = @_;
1782
1783     my $err;
1784
1785     foreach_mountpoint($conf, sub {
1786         my ($ms, $mountpoint) = @_;
1787
1788         return if $err; # skip further test
1789
1790         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1791
1792         # TODO: implement support for mountpoints
1793         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1794             if $ms ne 'rootfs';
1795     });
1796
1797     return $err ? 0 : 1;
1798 }
1799
1800 sub snapshot_create {
1801     my ($vmid, $snapname, $comment) = @_;
1802
1803     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1804
1805     my $conf = load_config($vmid);
1806
1807     my $running = check_running($vmid);
1808
1809     my $unfreeze = 0;
1810
1811     eval {
1812         if ($running) {
1813             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1814             $unfreeze = 1;
1815             PVE::Tools::run_command(['/bin/sync']);
1816         };
1817
1818         my $storecfg = PVE::Storage::config();
1819         my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1820         my $volid = $rootinfo->{volume};
1821
1822         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1823         &$snapshot_commit($vmid, $snapname);
1824     };
1825     my $err = $@;
1826
1827     if ($unfreeze) {
1828         eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
1829         warn $@ if $@;
1830     }
1831
1832     if ($err) {
1833         snapshot_delete($vmid, $snapname, 1);
1834         die "$err\n";
1835     }
1836 }
1837
1838 sub snapshot_delete {
1839     my ($vmid, $snapname, $force) = @_;
1840
1841     my $snap;
1842
1843     my $conf;
1844
1845     my $updatefn =  sub {
1846
1847         $conf = load_config($vmid);
1848
1849         die "you can't delete a snapshot if vm is a template\n"
1850             if is_template($conf);
1851
1852         $snap = $conf->{snapshots}->{$snapname};
1853
1854         check_lock($conf);
1855
1856         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1857
1858         $snap->{snapstate} = 'delete';
1859
1860         write_config($vmid, $conf);
1861     };
1862
1863     lock_container($vmid, 10, $updatefn);
1864
1865     my $storecfg = PVE::Storage::config();
1866
1867     my $unlink_parent = sub {
1868
1869         my ($confref, $new_parent) = @_;
1870
1871         if ($confref->{parent} && $confref->{parent} eq $snapname) {
1872             if ($new_parent) {
1873                 $confref->{parent} = $new_parent;
1874             } else {
1875                 delete $confref->{parent};
1876             }
1877         }
1878     };
1879
1880     my $del_snap =  sub {
1881
1882         check_lock($conf);
1883
1884         my $parent = $conf->{snapshots}->{$snapname}->{parent};
1885         foreach my $snapkey (keys %{$conf->{snapshots}}) {
1886             &$unlink_parent($conf->{snapshots}->{$snapkey}, $parent);
1887         }
1888
1889         &$unlink_parent($conf, $parent);
1890
1891         delete $conf->{snapshots}->{$snapname};
1892
1893         write_config($vmid, $conf);
1894     };
1895
1896     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1897     my $rootinfo = parse_ct_rootfs($rootfs);
1898     my $volid = $rootinfo->{volume};
1899
1900     eval {
1901         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1902     };
1903     my $err = $@;
1904
1905     if(!$err || ($err && $force)) {
1906         lock_container($vmid, 10, $del_snap);
1907         if ($err) {
1908             die "Can't delete snapshot: $vmid $snapname $err\n";
1909         }
1910     }
1911 }
1912
1913 sub snapshot_rollback {
1914     my ($vmid, $snapname) = @_;
1915
1916     my $storecfg = PVE::Storage::config();
1917
1918     my $conf = load_config($vmid);
1919
1920     die "you can't rollback if vm is a template\n" if is_template($conf);
1921
1922     my $snap = $conf->{snapshots}->{$snapname};
1923
1924     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1925
1926     my $rootfs = $snap->{rootfs};
1927     my $rootinfo = parse_ct_rootfs($rootfs);
1928     my $volid = $rootinfo->{volume};
1929
1930     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1931
1932     my $updatefn = sub {
1933
1934         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1935             if $snap->{snapstate};
1936
1937         check_lock($conf);
1938
1939         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1940
1941         die "unable to rollback vm $vmid: vm is running\n"
1942             if check_running($vmid);
1943
1944         $conf->{lock} = 'rollback';
1945
1946         my $forcemachine;
1947
1948         # copy snapshot config to current config
1949
1950         my $tmp_conf = $conf;
1951         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1952         $conf->{snapshots} = $tmp_conf->{snapshots};
1953         delete $conf->{snaptime};
1954         delete $conf->{snapname};
1955         $conf->{parent} = $snapname;
1956
1957         write_config($vmid, $conf);
1958     };
1959
1960     my $unlockfn = sub {
1961         delete $conf->{lock};
1962         write_config($vmid, $conf);
1963     };
1964
1965     lock_container($vmid, 10, $updatefn);
1966
1967     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1968
1969     lock_container($vmid, 5, $unlockfn);
1970 }
1971
1972 sub template_create {
1973     my ($vmid, $conf) = @_;
1974
1975     my $storecfg = PVE::Storage::config();
1976
1977     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1978     my $volid = $rootinfo->{volume};
1979
1980     die "Template feature is not available for '$volid'\n"
1981         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1982
1983     PVE::Storage::activate_volumes($storecfg, [$volid]);
1984
1985     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1986     $rootinfo->{volume} = $template_volid;
1987     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1988
1989     write_config($vmid, $conf);
1990 }
1991
1992 sub is_template {
1993     my ($conf) = @_;
1994
1995     return 1 if defined $conf->{template} && $conf->{template} == 1;
1996 }
1997
1998 sub mountpoint_names {
1999     my ($reverse) = @_;
2000
2001     my @names = ('rootfs');
2002
2003     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
2004         push @names, "mp$i";
2005     }
2006
2007     return $reverse ? reverse @names : @names;
2008 }
2009
2010 # The container might have *different* symlinks than the host. realpath/abs_path
2011 # use the actual filesystem to resolve links.
2012 sub sanitize_mountpoint {
2013     my ($mp) = @_;
2014     $mp = '/' . $mp; # we always start with a slash
2015     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
2016     $mp =~ s@/\./@@g; # collapse /./
2017     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
2018     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
2019     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
2020     return $mp;
2021 }
2022
2023 sub foreach_mountpoint_full {
2024     my ($conf, $reverse, $func) = @_;
2025
2026     foreach my $key (mountpoint_names($reverse)) {
2027         my $value = $conf->{$key};
2028         next if !defined($value);
2029         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2030         next if !defined($mountpoint);
2031
2032         $mountpoint->{mp} = sanitize_mountpoint($mountpoint->{mp});
2033
2034         my $path = $mountpoint->{volume};
2035         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
2036
2037         &$func($key, $mountpoint);
2038     }
2039 }
2040
2041 sub foreach_mountpoint {
2042     my ($conf, $func) = @_;
2043
2044     foreach_mountpoint_full($conf, 0, $func);
2045 }
2046
2047 sub foreach_mountpoint_reverse {
2048     my ($conf, $func) = @_;
2049
2050     foreach_mountpoint_full($conf, 1, $func);
2051 }
2052
2053 sub check_ct_modify_config_perm {
2054     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2055
2056     return 1 if $authuser ne 'root@pam';
2057
2058     foreach my $opt (@$key_list) {
2059
2060         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2061             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2062         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2063             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2064         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2065             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2066         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2067                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2068             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2069         } else {
2070             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2071         }
2072     }
2073
2074     return 1;
2075 }
2076
2077 sub umount_all {
2078     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2079
2080     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2081     my $volid_list = get_vm_volumes($conf);
2082
2083     foreach_mountpoint_reverse($conf, sub {
2084         my ($ms, $mountpoint) = @_;
2085
2086         my $volid = $mountpoint->{volume};
2087         my $mount = $mountpoint->{mp};
2088
2089         return if !$volid || !$mount;
2090
2091         my $mount_path = "$rootdir/$mount";
2092         $mount_path =~ s!/+!/!g;
2093
2094         return if !PVE::ProcFSTools::is_mounted($mount_path);
2095
2096         eval {
2097             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2098         };
2099         if (my $err = $@) {
2100             if ($noerr) {
2101                 warn $err;
2102             } else {
2103                 die $err;
2104             }
2105         }
2106     });
2107 }
2108
2109 sub mount_all {
2110     my ($vmid, $storage_cfg, $conf) = @_;
2111
2112     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2113     File::Path::make_path($rootdir);
2114
2115     my $volid_list = get_vm_volumes($conf);
2116     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2117
2118     eval {
2119         foreach_mountpoint($conf, sub {
2120             my ($ms, $mountpoint) = @_;
2121
2122             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2123         });
2124     };
2125     if (my $err = $@) {
2126         warn "mounting container failed\n";
2127         umount_all($vmid, $storage_cfg, $conf, 1);
2128         die $err;
2129     }
2130
2131     return $rootdir;
2132 }
2133
2134
2135 sub mountpoint_mount_path {
2136     my ($mountpoint, $storage_cfg, $snapname) = @_;
2137
2138     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2139 }
2140
2141 my $check_mount_path = sub {
2142     my ($path) = @_;
2143     $path = File::Spec->canonpath($path);
2144     my $real = Cwd::realpath($path);
2145     if ($real ne $path) {
2146         die "mount path modified by symlink: $path != $real";
2147     }
2148 };
2149
2150 sub query_loopdev {
2151     my ($path) = @_;
2152     my $found;
2153     my $parser = sub {
2154         my $line = shift;
2155         if ($line =~ m@^(/dev/loop\d+):@) {
2156             $found = $1;
2157         }
2158     };
2159     my $cmd = ['losetup', '--associated', $path];
2160     PVE::Tools::run_command($cmd, outfunc => $parser);
2161     return $found;
2162 }
2163
2164 # use $rootdir = undef to just return the corresponding mount path
2165 sub mountpoint_mount {
2166     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2167
2168     my $volid = $mountpoint->{volume};
2169     my $mount = $mountpoint->{mp};
2170     my $type = $mountpoint->{type};
2171
2172     return if !$volid || !$mount;
2173
2174     my $mount_path;
2175
2176     if (defined($rootdir)) {
2177         $rootdir =~ s!/+$!!;
2178         $mount_path = "$rootdir/$mount";
2179         $mount_path =~ s!/+!/!g;
2180         &$check_mount_path($mount_path);
2181         File::Path::mkpath($mount_path);
2182     }
2183
2184     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2185
2186     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2187
2188     if ($storage) {
2189
2190         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2191         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2192
2193         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2194             PVE::Storage::parse_volname($storage_cfg, $volid);
2195
2196         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2197
2198         if ($format eq 'subvol') {
2199             if ($mount_path) {
2200                 if ($snapname) {
2201                     if ($scfg->{type} eq 'zfspool') {
2202                         my $path_arg = $path;
2203                         $path_arg =~ s!^/+!!;
2204                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2205                     } else {
2206                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2207                     }
2208                 } else {
2209                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2210                 }
2211             }
2212             return wantarray ? ($path, 0) : $path;
2213         } elsif ($format eq 'raw' || $format eq 'iso') {
2214             my $use_loopdev = 0;
2215             my @extra_opts;
2216             if ($scfg->{path}) {
2217                 push @extra_opts, '-o', 'loop';
2218                 $use_loopdev = 1;
2219             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2220                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2221                 # do nothing
2222             } else {
2223                 die "unsupported storage type '$scfg->{type}'\n";
2224             }
2225             if ($mount_path) {
2226                 if ($format eq 'iso') {
2227                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2228                 } elsif ($isBase || defined($snapname)) {
2229                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2230                 } else {
2231                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2232                 }
2233             }
2234             return wantarray ? ($path, $use_loopdev) : $path;
2235         } else {
2236             die "unsupported image format '$format'\n";
2237         }
2238     } elsif ($type eq 'device') {
2239         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2240         return wantarray ? ($volid, 0) : $volid;
2241     } elsif ($type eq 'bind') {
2242         die "directory '$volid' does not exist\n" if ! -d $volid;
2243         &$check_mount_path($volid);
2244         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2245         return wantarray ? ($volid, 0) : $volid;
2246     }
2247
2248     die "unsupported storage";
2249 }
2250
2251 sub get_vm_volumes {
2252     my ($conf, $excludes) = @_;
2253
2254     my $vollist = [];
2255
2256     foreach_mountpoint($conf, sub {
2257         my ($ms, $mountpoint) = @_;
2258
2259         return if $excludes && $ms eq $excludes;
2260
2261         my $volid = $mountpoint->{volume};
2262
2263         return if !$volid || $mountpoint->{type} ne 'volume';
2264
2265         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2266         return if !$sid;
2267
2268         push @$vollist, $volid;
2269     });
2270
2271     return $vollist;
2272 }
2273
2274 sub mkfs {
2275     my ($dev, $rootuid, $rootgid) = @_;
2276
2277     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2278                              '-E', "root_owner=$rootuid:$rootgid",
2279                              $dev]);
2280 }
2281
2282 sub format_disk {
2283     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2284
2285     if ($volid =~ m!^/dev/.+!) {
2286         mkfs($volid);
2287         return;
2288     }
2289
2290     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2291
2292     die "cannot format volume '$volid' with no storage\n" if !$storage;
2293
2294     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2295
2296     my $path = PVE::Storage::path($storage_cfg, $volid);
2297
2298     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2299         PVE::Storage::parse_volname($storage_cfg, $volid);
2300
2301     die "cannot format volume '$volid' (format == $format)\n"
2302         if $format ne 'raw';
2303
2304     mkfs($path, $rootuid, $rootgid);
2305 }
2306
2307 sub destroy_disks {
2308     my ($storecfg, $vollist) = @_;
2309
2310     foreach my $volid (@$vollist) {
2311         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2312         warn $@ if $@;
2313     }
2314 }
2315
2316 sub create_disks {
2317     my ($storecfg, $vmid, $settings, $conf) = @_;
2318
2319     my $vollist = [];
2320
2321     eval {
2322         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2323         my $chown_vollist = [];
2324
2325         foreach_mountpoint($settings, sub {
2326             my ($ms, $mountpoint) = @_;
2327
2328             my $volid = $mountpoint->{volume};
2329             my $mp = $mountpoint->{mp};
2330
2331             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2332
2333             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2334                 my ($storeid, $size_gb) = ($1, $2);
2335
2336                 my $size_kb = int(${size_gb}*1024) * 1024;
2337
2338                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2339                 # fixme: use better naming ct-$vmid-disk-X.raw?
2340
2341                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2342                     if ($size_kb > 0) {
2343                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2344                                                            undef, $size_kb);
2345                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2346                     } else {
2347                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2348                                                            undef, 0);
2349                         push @$chown_vollist, $volid;
2350                     }
2351                 } elsif ($scfg->{type} eq 'zfspool') {
2352
2353                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2354                                                        undef, $size_kb);
2355                     push @$chown_vollist, $volid;
2356                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2357
2358                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2359                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2360
2361                 } elsif ($scfg->{type} eq 'rbd') {
2362
2363                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2364                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2365                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2366                 } else {
2367                     die "unable to create containers on storage type '$scfg->{type}'\n";
2368                 }
2369                 push @$vollist, $volid;
2370                 $mountpoint->{volume} = $volid;
2371                 $mountpoint->{size} = $size_kb * 1024;
2372                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2373             } else {
2374                 # use specified/existing volid/dir/device
2375                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2376             }
2377         });
2378
2379         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2380         foreach my $volid (@$chown_vollist) {
2381             my $path = PVE::Storage::path($storecfg, $volid, undef);
2382             chown($rootuid, $rootgid, $path);
2383         }
2384         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2385     };
2386     # free allocated images on error
2387     if (my $err = $@) {
2388         destroy_disks($storecfg, $vollist);
2389         die $err;
2390     }
2391     return $vollist;
2392 }
2393
2394 # bash completion helper
2395
2396 sub complete_os_templates {
2397     my ($cmdname, $pname, $cvalue) = @_;
2398
2399     my $cfg = PVE::Storage::config();
2400
2401     my $storeid;
2402
2403     if ($cvalue =~ m/^([^:]+):/) {
2404         $storeid = $1;
2405     }
2406
2407     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2408     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2409
2410     my $res = [];
2411     foreach my $id (keys %$data) {
2412         foreach my $item (@{$data->{$id}}) {
2413             push @$res, $item->{volid} if defined($item->{volid});
2414         }
2415     }
2416
2417     return $res;
2418 }
2419
2420 my $complete_ctid_full = sub {
2421     my ($running) = @_;
2422
2423     my $idlist = vmstatus();
2424
2425     my $active_hash = list_active_containers();
2426
2427     my $res = [];
2428
2429     foreach my $id (keys %$idlist) {
2430         my $d = $idlist->{$id};
2431         if (defined($running)) {
2432             next if $d->{template};
2433             next if $running && !$active_hash->{$id};
2434             next if !$running && $active_hash->{$id};
2435         }
2436         push @$res, $id;
2437
2438     }
2439     return $res;
2440 };
2441
2442 sub complete_ctid {
2443     return &$complete_ctid_full();
2444 }
2445
2446 sub complete_ctid_stopped {
2447     return &$complete_ctid_full(0);
2448 }
2449
2450 sub complete_ctid_running {
2451     return &$complete_ctid_full(1);
2452 }
2453
2454 sub parse_id_maps {
2455     my ($conf) = @_;
2456
2457     my $id_map = [];
2458     my $rootuid = 0;
2459     my $rootgid = 0;
2460
2461     my $lxc = $conf->{lxc};
2462     foreach my $entry (@$lxc) {
2463         my ($key, $value) = @$entry;
2464         next if $key ne 'lxc.id_map';
2465         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2466             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2467             push @$id_map, [$type, $ct, $host, $length];
2468             if ($ct == 0) {
2469                 $rootuid = $host if $type eq 'u';
2470                 $rootgid = $host if $type eq 'g';
2471             }
2472         } else {
2473             die "failed to parse id_map: $value\n";
2474         }
2475     }
2476
2477     if (!@$id_map && $conf->{unprivileged}) {
2478         # Should we read them from /etc/subuid?
2479         $id_map = [ ['u', '0', '100000', '65536'],
2480                     ['g', '0', '100000', '65536'] ];
2481         $rootuid = $rootgid = 100000;
2482     }
2483
2484     return ($id_map, $rootuid, $rootgid);
2485 }
2486
2487 sub userns_command {
2488     my ($id_map) = @_;
2489     if (@$id_map) {
2490         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2491     }
2492     return [];
2493 }
2494
2495 1;