src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format => 'pve-lxc-mp-string',
  42         format_description => 'volume',
  43         description => 'Volume, device or directory to mount into the container.',
  44     },
  45     backup => {
  46         type => 'boolean',
  47         format_description => '[1|0]',
  48         description => 'Whether to include the mountpoint in backups.',
  49         optional => 1,
  50     },
  51     size => {
  52         type => 'string',
  53         format => 'disk-size',
  54         format_description => 'DiskSize',
  55         description => 'Volume size (read only value).',
  56         optional => 1,
  57     },
  58     acl => {
  59         type => 'boolean',
  60         format_description => 'acl',
  61         description => 'Explicitly enable or disable ACL support.',
  62         optional => 1,
  63     },
  64     ro => {
  65         type => 'boolean',
  66         format_description => 'ro',
  67         description => 'Read-only mountpoint (not supported with bind mounts)',
  68         optional => 1,
  69     },
  70 };
  71
  72 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  73     type => 'string', format => $rootfs_desc,
  74     description => "Use volume as container root.",
  75     optional => 1,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  79     description => "The name of the snapshot.",
  80     type => 'string', format => 'pve-configid',
  81     maxLength => 40,
  82 });
  83
  84 my $confdesc = {
  85     lock => {
  86         optional => 1,
  87         type => 'string',
  88         description => "Lock/unlock the VM.",
  89         enum => [qw(migrate backup snapshot rollback)],
  90     },
  91     onboot => {
  92         optional => 1,
  93         type => 'boolean',
  94         description => "Specifies whether a VM will be started during system bootup.",
  95         default => 0,
  96     },
  97     startup => get_standard_option('pve-startup-order'),
  98     template => {
  99         optional => 1,
 100         type => 'boolean',
 101         description => "Enable/disable Template.",
 102         default => 0,
 103     },
 104     arch => {
 105         optional => 1,
 106         type => 'string',
 107         enum => ['amd64', 'i386'],
 108         description => "OS architecture type.",
 109         default => 'amd64',
 110     },
 111     ostype => {
 112         optional => 1,
 113         type => 'string',
 114         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux'],
 115         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 116     },
 117     console => {
 118         optional => 1,
 119         type => 'boolean',
 120         description => "Attach a console device (/dev/console) to the container.",
 121         default => 1,
 122     },
 123     tty => {
 124         optional => 1,
 125         type => 'integer',
 126         description => "Specify the number of tty available to the container",
 127         minimum => 0,
 128         maximum => 6,
 129         default => 2,
 130     },
 131     cpulimit => {
 132         optional => 1,
 133         type => 'number',
 134         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 135         minimum => 0,
 136         maximum => 128,
 137         default => 0,
 138     },
 139     cpuunits => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 143         minimum => 0,
 144         maximum => 500000,
 145         default => 1024,
 146     },
 147     memory => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of RAM for the VM in MB.",
 151         minimum => 16,
 152         default => 512,
 153     },
 154     swap => {
 155         optional => 1,
 156         type => 'integer',
 157         description => "Amount of SWAP for the VM in MB.",
 158         minimum => 0,
 159         default => 512,
 160     },
 161     hostname => {
 162         optional => 1,
 163         description => "Set a host name for the container.",
 164         type => 'string', format => 'dns-name',
 165         maxLength => 255,
 166     },
 167     description => {
 168         optional => 1,
 169         type => 'string',
 170         description => "Container description. Only used on the configuration web interface.",
 171     },
 172     searchdomain => {
 173         optional => 1,
 174         type => 'string', format => 'dns-name-list',
 175         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 176     },
 177     nameserver => {
 178         optional => 1,
 179         type => 'string', format => 'address-list',
 180         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 181     },
 182     rootfs => get_standard_option('pve-ct-rootfs'),
 183     parent => {
 184         optional => 1,
 185         type => 'string', format => 'pve-configid',
 186         maxLength => 40,
 187         description => "Parent snapshot name. This is used internally, and should not be modified.",
 188     },
 189     snaptime => {
 190         optional => 1,
 191         description => "Timestamp for snapshots.",
 192         type => 'integer',
 193         minimum => 0,
 194     },
 195     cmode => {
 196         optional => 1,
 197         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 198         type => 'string',
 199         enum => ['shell', 'console', 'tty'],
 200         default => 'tty',
 201     },
 202     protection => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 206         default => 0,
 207     },
 208     unprivileged => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 212         default => 0,
 213     },
 214 };
 215
 216 my $valid_lxc_conf_keys = {
 217     'lxc.include' => 1,
 218     'lxc.arch' => 1,
 219     'lxc.utsname' => 1,
 220     'lxc.haltsignal' => 1,
 221     'lxc.rebootsignal' => 1,
 222     'lxc.stopsignal' => 1,
 223     'lxc.init_cmd' => 1,
 224     'lxc.network.type' => 1,
 225     'lxc.network.flags' => 1,
 226     'lxc.network.link' => 1,
 227     'lxc.network.mtu' => 1,
 228     'lxc.network.name' => 1,
 229     'lxc.network.hwaddr' => 1,
 230     'lxc.network.ipv4' => 1,
 231     'lxc.network.ipv4.gateway' => 1,
 232     'lxc.network.ipv6' => 1,
 233     'lxc.network.ipv6.gateway' => 1,
 234     'lxc.network.script.up' => 1,
 235     'lxc.network.script.down' => 1,
 236     'lxc.pts' => 1,
 237     'lxc.console.logfile' => 1,
 238     'lxc.console' => 1,
 239     'lxc.tty' => 1,
 240     'lxc.devttydir' => 1,
 241     'lxc.hook.autodev' => 1,
 242     'lxc.autodev' => 1,
 243     'lxc.kmsg' => 1,
 244     'lxc.mount' => 1,
 245     'lxc.mount.entry' => 1,
 246     'lxc.mount.auto' => 1,
 247     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 248     'lxc.rootfs.mount' => 1,
 249     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 250                             ', please use mountpoint options in the "rootfs" key',
 251     # lxc.cgroup.*
 252     'lxc.cap.drop' => 1,
 253     'lxc.cap.keep' => 1,
 254     'lxc.aa_profile' => 1,
 255     'lxc.aa_allow_incomplete' => 1,
 256     'lxc.se_context' => 1,
 257     'lxc.seccomp' => 1,
 258     'lxc.id_map' => 1,
 259     'lxc.hook.pre-start' => 1,
 260     'lxc.hook.pre-mount' => 1,
 261     'lxc.hook.mount' => 1,
 262     'lxc.hook.start' => 1,
 263     'lxc.hook.stop' => 1,
 264     'lxc.hook.post-stop' => 1,
 265     'lxc.hook.clone' => 1,
 266     'lxc.hook.destroy' => 1,
 267     'lxc.loglevel' => 1,
 268     'lxc.logfile' => 1,
 269     'lxc.start.auto' => 1,
 270     'lxc.start.delay' => 1,
 271     'lxc.start.order' => 1,
 272     'lxc.group' => 1,
 273     'lxc.environment' => 1,
 274 };
 275
 276 my $netconf_desc = {
 277     type => {
 278         type => 'string',
 279         optional => 1,
 280         description => "Network interface type.",
 281         enum => [qw(veth)],
 282     },
 283     name => {
 284         type => 'string',
 285         format_description => 'String',
 286         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 287         pattern => '[-_.\w\d]+',
 288     },
 289     bridge => {
 290         type => 'string',
 291         format_description => 'vmbr<Number>',
 292         description => 'Bridge to attach the network device to.',
 293         pattern => '[-_.\w\d]+',
 294         optional => 1,
 295     },
 296     hwaddr => {
 297         type => 'string',
 298         format_description => 'MAC',
 299         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 300         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 301         optional => 1,
 302     },
 303     mtu => {
 304         type => 'integer',
 305         format_description => 'Number',
 306         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 307         minimum => 64, # minimum ethernet frame is 64 bytes
 308         optional => 1,
 309     },
 310     ip => {
 311         type => 'string',
 312         format => 'pve-ipv4-config',
 313         format_description => 'IPv4Format/CIDR',
 314         description => 'IPv4 address in CIDR format.',
 315         optional => 1,
 316     },
 317     gw => {
 318         type => 'string',
 319         format => 'ipv4',
 320         format_description => 'GatewayIPv4',
 321         description => 'Default gateway for IPv4 traffic.',
 322         optional => 1,
 323     },
 324     ip6 => {
 325         type => 'string',
 326         format => 'pve-ipv6-config',
 327         format_description => 'IPv6Format/CIDR',
 328         description => 'IPv6 address in CIDR format.',
 329         optional => 1,
 330     },
 331     gw6 => {
 332         type => 'string',
 333         format => 'ipv6',
 334         format_description => 'GatewayIPv6',
 335         description => 'Default gateway for IPv6 traffic.',
 336         optional => 1,
 337     },
 338     firewall => {
 339         type => 'boolean',
 340         format_description => '[1|0]',
 341         description => "Controls whether this interface's firewall rules should be used.",
 342         optional => 1,
 343     },
 344     tag => {
 345         type => 'integer',
 346         format_description => 'VlanNo',
 347         minimum => '2',
 348         maximum => '4094',
 349         description => "VLAN tag for this interface.",
 350         optional => 1,
 351     },
 352     trunks => {
 353         type => 'string',
 354         pattern => qr/\d+(?:;\d+)*/,
 355         format_description => 'vlanid[;vlanid...]',
 356         description => "VLAN ids to pass through the interface",
 357         optional => 1,
 358     },
 359 };
 360 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 361
 362 my $MAX_LXC_NETWORKS = 10;
 363 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 364     $confdesc->{"net$i"} = {
 365         optional => 1,
 366         type => 'string', format => $netconf_desc,
 367         description => "Specifies network interfaces for the container.",
 368     };
 369 }
 370
 371 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 372 sub verify_lxc_mp_string{
 373     my ($mp, $noerr) = @_;
 374
 375     # do not allow:
 376     # /./ or /../
 377     # /. or /.. at the end
 378     # ../ at the beginning
 379
 380     if($mp =~ m@/\.\.?/@ ||
 381        $mp =~ m@/\.\.?$@ ||
 382        $mp =~ m@^\.\./@){
 383         return undef if $noerr;
 384         die "$mp contains illegal character sequences\n";
 385     }
 386     return $mp;
 387 }
 388
 389 my $mp_desc = {
 390     %$rootfs_desc,
 391     mp => {
 392         type => 'string',
 393         format => 'pve-lxc-mp-string',
 394         format_description => 'Path',
 395         description => 'Path to the mountpoint as seen from inside the container.',
 396     },
 397 };
 398 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 399
 400 my $unuseddesc = {
 401     optional => 1,
 402     type => 'string', format => 'pve-volume-id',
 403     description => "Reference to unused volumes.",
 404 };
 405
 406 my $MAX_MOUNT_POINTS = 10;
 407 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 408     $confdesc->{"mp$i"} = {
 409         optional => 1,
 410         type => 'string', format => $mp_desc,
 411         description => "Use volume as container mount point (experimental feature).",
 412         optional => 1,
 413     };
 414 }
 415
 416 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 417 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 418     $confdesc->{"unused$i"} = $unuseddesc;
 419 }
 420
 421 sub write_pct_config {
 422     my ($filename, $conf) = @_;
 423
 424     delete $conf->{snapstate}; # just to be sure
 425
 426     my $generate_raw_config = sub {
 427         my ($conf) = @_;
 428
 429         my $raw = '';
 430
 431         # add description as comment to top of file
 432         my $descr = $conf->{description} || '';
 433         foreach my $cl (split(/\n/, $descr)) {
 434             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 435         }
 436
 437         foreach my $key (sort keys %$conf) {
 438             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 439                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 440             my $value = $conf->{$key};
 441             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 442             $raw .= "$key: $value\n";
 443         }
 444
 445         if (my $lxcconf = $conf->{lxc}) {
 446             foreach my $entry (@$lxcconf) {
 447                 my ($k, $v) = @$entry;
 448                 $raw .= "$k: $v\n";
 449             }
 450         }
 451
 452         return $raw;
 453     };
 454
 455     my $raw = &$generate_raw_config($conf);
 456
 457     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 458         $raw .= "\n[$snapname]\n";
 459         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 460     }
 461
 462     return $raw;
 463 }
 464
 465 sub check_type {
 466     my ($key, $value) = @_;
 467
 468     die "unknown setting '$key'\n" if !$confdesc->{$key};
 469
 470     my $type = $confdesc->{$key}->{type};
 471
 472     if (!defined($value)) {
 473         die "got undefined value\n";
 474     }
 475
 476     if ($value =~ m/[\n\r]/) {
 477         die "property contains a line feed\n";
 478     }
 479
 480     if ($type eq 'boolean') {
 481         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 482         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 483         die "type check ('boolean') failed - got '$value'\n";
 484     } elsif ($type eq 'integer') {
 485         return int($1) if $value =~ m/^(\d+)$/;
 486         die "type check ('integer') failed - got '$value'\n";
 487     } elsif ($type eq 'number') {
 488         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 489         die "type check ('number') failed - got '$value'\n";
 490     } elsif ($type eq 'string') {
 491         if (my $fmt = $confdesc->{$key}->{format}) {
 492             PVE::JSONSchema::check_format($fmt, $value);
 493             return $value;
 494         }
 495         return $value;
 496     } else {
 497         die "internal error"
 498     }
 499 }
 500
 501 sub parse_pct_config {
 502     my ($filename, $raw) = @_;
 503
 504     return undef if !defined($raw);
 505
 506     my $res = {
 507         digest => Digest::SHA::sha1_hex($raw),
 508         snapshots => {},
 509     };
 510
 511     $filename =~ m|/lxc/(\d+).conf$|
 512         || die "got strange filename '$filename'";
 513
 514     my $vmid = $1;
 515
 516     my $conf = $res;
 517     my $descr = '';
 518     my $section = '';
 519
 520     my @lines = split(/\n/, $raw);
 521     foreach my $line (@lines) {
 522         next if $line =~ m/^\s*$/;
 523
 524         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 525             $section = $1;
 526             $conf->{description} = $descr if $descr;
 527             $descr = '';
 528             $conf = $res->{snapshots}->{$section} = {};
 529             next;
 530         }
 531
 532         if ($line =~ m/^\#(.*)\s*$/) {
 533             $descr .= PVE::Tools::decode_text($1) . "\n";
 534             next;
 535         }
 536
 537         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 538             my $key = $1;
 539             my $value = $3;
 540             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 541             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 542                 push @{$conf->{lxc}}, [$key, $value];
 543             } elsif (my $errmsg = $validity) {
 544                 warn "vm $vmid - $key: $errmsg\n";
 545             } else {
 546                 warn "vm $vmid - unable to parse config: $line\n";
 547             }
 548         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 549             $descr .= PVE::Tools::decode_text($2);
 550         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 551             $conf->{snapstate} = $1;
 552         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 553             my $key = $1;
 554             my $value = $2;
 555             eval { $value = check_type($key, $value); };
 556             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 557             $conf->{$key} = $value;
 558         } else {
 559             warn "vm $vmid - unable to parse config: $line\n";
 560         }
 561     }
 562
 563     $conf->{description} = $descr if $descr;
 564
 565     delete $res->{snapstate}; # just to be sure
 566
 567     return $res;
 568 }
 569
 570 sub config_list {
 571     my $vmlist = PVE::Cluster::get_vmlist();
 572     my $res = {};
 573     return $res if !$vmlist || !$vmlist->{ids};
 574     my $ids = $vmlist->{ids};
 575
 576     foreach my $vmid (keys %$ids) {
 577         next if !$vmid; # skip CT0
 578         my $d = $ids->{$vmid};
 579         next if !$d->{node} || $d->{node} ne $nodename;
 580         next if !$d->{type} || $d->{type} ne 'lxc';
 581         $res->{$vmid}->{type} = 'lxc';
 582     }
 583     return $res;
 584 }
 585
 586 sub cfs_config_path {
 587     my ($vmid, $node) = @_;
 588
 589     $node = $nodename if !$node;
 590     return "nodes/$node/lxc/$vmid.conf";
 591 }
 592
 593 sub config_file {
 594     my ($vmid, $node) = @_;
 595
 596     my $cfspath = cfs_config_path($vmid, $node);
 597     return "/etc/pve/$cfspath";
 598 }
 599
 600 sub load_config {
 601     my ($vmid, $node) = @_;
 602
 603     $node = $nodename if !$node;
 604     my $cfspath = cfs_config_path($vmid, $node);
 605
 606     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 607     die "container $vmid does not exist\n" if !defined($conf);
 608
 609     return $conf;
 610 }
 611
 612 sub create_config {
 613     my ($vmid, $conf) = @_;
 614
 615     my $dir = "/etc/pve/nodes/$nodename/lxc";
 616     mkdir $dir;
 617
 618     write_config($vmid, $conf);
 619 }
 620
 621 sub destroy_config {
 622     my ($vmid) = @_;
 623
 624     unlink config_file($vmid, $nodename);
 625 }
 626
 627 sub write_config {
 628     my ($vmid, $conf) = @_;
 629
 630     my $cfspath = cfs_config_path($vmid);
 631
 632     PVE::Cluster::cfs_write_file($cfspath, $conf);
 633 }
 634
 635 # flock: we use one file handle per process, so lock file
 636 # can be called multiple times and will succeed for the same process.
 637
 638 my $lock_handles =  {};
 639 my $lockdir = "/run/lock/lxc";
 640
 641 sub lock_filename {
 642     my ($vmid) = @_;
 643
 644     return "$lockdir/pve-config-${vmid}.lock";
 645 }
 646
 647 sub lock_container {
 648     my ($vmid, $timeout, $code, @param) = @_;
 649
 650     $timeout = 10 if !$timeout;
 651
 652     my $filename = lock_filename($vmid);
 653
 654     mkdir $lockdir if !-d $lockdir;
 655
 656     my $res = PVE::Tools::lock_file_full($filename, $timeout, 0, $code, @param);
 657
 658     die $@ if $@;
 659
 660     return $res;
 661 }
 662
 663 sub option_exists {
 664     my ($name) = @_;
 665
 666     return defined($confdesc->{$name});
 667 }
 668
 669 # add JSON properties for create and set function
 670 sub json_config_properties {
 671     my $prop = shift;
 672
 673     foreach my $opt (keys %$confdesc) {
 674         next if $opt eq 'parent' || $opt eq 'snaptime';
 675         next if $prop->{$opt};
 676         $prop->{$opt} = $confdesc->{$opt};
 677     }
 678
 679     return $prop;
 680 }
 681
 682 sub json_config_properties_no_rootfs {
 683     my $prop = shift;
 684
 685     foreach my $opt (keys %$confdesc) {
 686         next if $prop->{$opt};
 687         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 688         $prop->{$opt} = $confdesc->{$opt};
 689     }
 690
 691     return $prop;
 692 }
 693
 694 # container status helpers
 695
 696 sub list_active_containers {
 697
 698     my $filename = "/proc/net/unix";
 699
 700     # similar test is used by lcxcontainers.c: list_active_containers
 701     my $res = {};
 702
 703     my $fh = IO::File->new ($filename, "r");
 704     return $res if !$fh;
 705
 706     while (defined(my $line = <$fh>)) {
 707         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 708             my $path = $1;
 709             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 710                 $res->{$1} = 1;
 711             }
 712         }
 713     }
 714
 715     close($fh);
 716
 717     return $res;
 718 }
 719
 720 # warning: this is slow
 721 sub check_running {
 722     my ($vmid) = @_;
 723
 724     my $active_hash = list_active_containers();
 725
 726     return 1 if defined($active_hash->{$vmid});
 727
 728     return undef;
 729 }
 730
 731 sub get_container_disk_usage {
 732     my ($vmid, $pid) = @_;
 733
 734     return PVE::Tools::df("/proc/$pid/root/", 1);
 735 }
 736
 737 my $last_proc_vmid_stat;
 738
 739 my $parse_cpuacct_stat = sub {
 740     my ($vmid) = @_;
 741
 742     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 743
 744     my $stat = {};
 745
 746     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 747
 748         $stat->{utime} = $1;
 749         $stat->{stime} = $2;
 750
 751     }
 752
 753     return $stat;
 754 };
 755
 756 sub vmstatus {
 757     my ($opt_vmid) = @_;
 758
 759     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 760
 761     my $active_hash = list_active_containers();
 762
 763     my $cpucount = $cpuinfo->{cpus} || 1;
 764
 765     my $cdtime = gettimeofday;
 766
 767     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 768
 769     foreach my $vmid (keys %$list) {
 770         my $d = $list->{$vmid};
 771
 772         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 773         warn $@ if $@; # ignore errors (consider them stopped)
 774
 775         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 776
 777         my $cfspath = cfs_config_path($vmid);
 778         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 779
 780         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 781         $d->{name} =~ s/[\s]//g;
 782
 783         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 784
 785         if ($d->{pid}) {
 786             my $res = get_container_disk_usage($vmid, $d->{pid});
 787             $d->{disk} = $res->{used};
 788             $d->{maxdisk} = $res->{total};
 789         } else {
 790             $d->{disk} = 0;
 791             # use 4GB by default ??
 792             if (my $rootfs = $conf->{rootfs}) {
 793                 my $rootinfo = parse_ct_rootfs($rootfs);
 794                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 795             } else {
 796                 $d->{maxdisk} = 4*1024*1024*1024;
 797             }
 798         }
 799
 800         $d->{mem} = 0;
 801         $d->{swap} = 0;
 802         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 803         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 804
 805         $d->{uptime} = 0;
 806         $d->{cpu} = 0;
 807
 808         $d->{netout} = 0;
 809         $d->{netin} = 0;
 810
 811         $d->{diskread} = 0;
 812         $d->{diskwrite} = 0;
 813
 814         $d->{template} = is_template($conf);
 815     }
 816
 817     foreach my $vmid (keys %$list) {
 818         my $d = $list->{$vmid};
 819         my $pid = $d->{pid};
 820
 821         next if !$pid; # skip stopped CTs
 822
 823         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 824         $d->{uptime} = time - $ctime; # the method lxcfs uses
 825
 826         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 827         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 828
 829         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 830         my @bytes = split(/\n/, $blkio_bytes);
 831         foreach my $byte (@bytes) {
 832             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 833                 $d->{diskread} = $2 if $key eq 'Read';
 834                 $d->{diskwrite} = $2 if $key eq 'Write';
 835             }
 836         }
 837
 838         my $pstat = &$parse_cpuacct_stat($vmid);
 839
 840         my $used = $pstat->{utime} + $pstat->{stime};
 841
 842         my $old = $last_proc_vmid_stat->{$vmid};
 843         if (!$old) {
 844             $last_proc_vmid_stat->{$vmid} = {
 845                 time => $cdtime,
 846                 used => $used,
 847                 cpu => 0,
 848             };
 849             next;
 850         }
 851
 852         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 853
 854         if ($dtime > 1000) {
 855             my $dutime = $used -  $old->{used};
 856
 857             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 858             $last_proc_vmid_stat->{$vmid} = {
 859                 time => $cdtime,
 860                 used => $used,
 861                 cpu => $d->{cpu},
 862             };
 863         } else {
 864             $d->{cpu} = $old->{cpu};
 865         }
 866     }
 867
 868     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 869
 870     foreach my $dev (keys %$netdev) {
 871         next if $dev !~ m/^veth([1-9]\d*)i/;
 872         my $vmid = $1;
 873         my $d = $list->{$vmid};
 874
 875         next if !$d;
 876
 877         $d->{netout} += $netdev->{$dev}->{receive};
 878         $d->{netin} += $netdev->{$dev}->{transmit};
 879
 880     }
 881
 882     return $list;
 883 }
 884
 885 sub classify_mountpoint {
 886     my ($vol) = @_;
 887     if ($vol =~ m!^/!) {
 888         return 'device' if $vol =~ m!^/dev/!;
 889         return 'bind';
 890     }
 891     return 'volume';
 892 }
 893
 894 my $parse_ct_mountpoint_full = sub {
 895     my ($desc, $data, $noerr) = @_;
 896
 897     $data //= '';
 898
 899     my $res;
 900     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 901     if ($@) {
 902         return undef if $noerr;
 903         die $@;
 904     }
 905
 906     if (defined(my $size = $res->{size})) {
 907         $size = PVE::JSONSchema::parse_size($size);
 908         if (!defined($size)) {
 909             return undef if $noerr;
 910             die "invalid size: $size\n";
 911         }
 912         $res->{size} = $size;
 913     }
 914
 915     $res->{type} = classify_mountpoint($res->{volume});
 916
 917     return $res;
 918 };
 919
 920 sub parse_ct_rootfs {
 921     my ($data, $noerr) = @_;
 922
 923     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 924
 925     $res->{mp} = '/' if defined($res);
 926
 927     return $res;
 928 }
 929
 930 sub parse_ct_mountpoint {
 931     my ($data, $noerr) = @_;
 932
 933     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 934 }
 935
 936 sub print_ct_mountpoint {
 937     my ($info, $nomp) = @_;
 938     my $skip = [ 'type' ];
 939     push @$skip, 'mp' if $nomp;
 940     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 941 }
 942
 943 sub print_lxc_network {
 944     my $net = shift;
 945     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 946 }
 947
 948 sub parse_lxc_network {
 949     my ($data) = @_;
 950
 951     my $res = {};
 952
 953     return $res if !$data;
 954
 955     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 956
 957     $res->{type} = 'veth';
 958     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 959
 960     return $res;
 961 }
 962
 963 sub read_cgroup_value {
 964     my ($group, $vmid, $name, $full) = @_;
 965
 966     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 967
 968     return PVE::Tools::file_get_contents($path) if $full;
 969
 970     return PVE::Tools::file_read_firstline($path);
 971 }
 972
 973 sub write_cgroup_value {
 974    my ($group, $vmid, $name, $value) = @_;
 975
 976    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 977    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 978
 979 }
 980
 981 sub find_lxc_console_pids {
 982
 983     my $res = {};
 984
 985     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 986         my ($pid) = @_;
 987
 988         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 989         return if !$cmdline;
 990
 991         my @args = split(/\0/, $cmdline);
 992
 993         # search for lxc-console -n <vmid>
 994         return if scalar(@args) != 3;
 995         return if $args[1] ne '-n';
 996         return if $args[2] !~ m/^\d+$/;
 997         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 998
 999         my $vmid = $args[2];
1000
1001         push @{$res->{$vmid}}, $pid;
1002     });
1003
1004     return $res;
1005 }
1006
1007 sub find_lxc_pid {
1008     my ($vmid) = @_;
1009
1010     my $pid = undef;
1011     my $parser = sub {
1012         my $line = shift;
1013         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1014     };
1015     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1016
1017     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1018
1019     return $pid;
1020 }
1021
1022 # Note: we cannot use Net:IP, because that only allows strict
1023 # CIDR networks
1024 sub parse_ipv4_cidr {
1025     my ($cidr, $noerr) = @_;
1026
1027     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1028         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1029     }
1030
1031     return undef if $noerr;
1032
1033     die "unable to parse ipv4 address/mask\n";
1034 }
1035
1036 sub check_lock {
1037     my ($conf) = @_;
1038
1039     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1040 }
1041
1042 sub check_protection {
1043     my ($vm_conf, $err_msg) = @_;
1044
1045     if ($vm_conf->{protection}) {
1046         die "$err_msg - protection mode enabled\n";
1047     }
1048 }
1049
1050 sub update_lxc_config {
1051     my ($storage_cfg, $vmid, $conf) = @_;
1052
1053     my $dir = "/var/lib/lxc/$vmid";
1054
1055     if ($conf->{template}) {
1056
1057         unlink "$dir/config";
1058
1059         return;
1060     }
1061
1062     my $raw = '';
1063
1064     die "missing 'arch' - internal error" if !$conf->{arch};
1065     $raw .= "lxc.arch = $conf->{arch}\n";
1066
1067     my $unprivileged = $conf->{unprivileged};
1068     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1069
1070     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1071     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux)$/x) {
1072         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1073         if ($unprivileged || $custom_idmap) {
1074             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1075         }
1076     } else {
1077         die "implement me (ostype $ostype)";
1078     }
1079
1080     $raw .= "lxc.monitor.unshare = 1\n";
1081
1082     # Should we read them from /etc/subuid?
1083     if ($unprivileged && !$custom_idmap) {
1084         $raw .= "lxc.id_map = u 0 100000 65536\n";
1085         $raw .= "lxc.id_map = g 0 100000 65536\n";
1086     }
1087
1088     if (!has_dev_console($conf)) {
1089         $raw .= "lxc.console = none\n";
1090         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1091     }
1092
1093     my $ttycount = get_tty_count($conf);
1094     $raw .= "lxc.tty = $ttycount\n";
1095
1096     # some init scripts expect a linux terminal (turnkey).
1097     $raw .= "lxc.environment = TERM=linux\n";
1098
1099     my $utsname = $conf->{hostname} || "CT$vmid";
1100     $raw .= "lxc.utsname = $utsname\n";
1101
1102     my $memory = $conf->{memory} || 512;
1103     my $swap = $conf->{swap} // 0;
1104
1105     my $lxcmem = int($memory*1024*1024);
1106     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1107
1108     my $lxcswap = int(($memory + $swap)*1024*1024);
1109     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1110
1111     if (my $cpulimit = $conf->{cpulimit}) {
1112         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1113         my $value = int(100000*$cpulimit);
1114         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1115     }
1116
1117     my $shares = $conf->{cpuunits} || 1024;
1118     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1119
1120     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1121
1122     $raw .= "lxc.rootfs = $dir/rootfs\n";
1123
1124     my $netcount = 0;
1125     foreach my $k (keys %$conf) {
1126         next if $k !~ m/^net(\d+)$/;
1127         my $ind = $1;
1128         my $d = parse_lxc_network($conf->{$k});
1129         $netcount++;
1130         $raw .= "lxc.network.type = veth\n";
1131         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1132         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1133         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1134         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1135     }
1136
1137     if (my $lxcconf = $conf->{lxc}) {
1138         foreach my $entry (@$lxcconf) {
1139             my ($k, $v) = @$entry;
1140             $netcount++ if $k eq 'lxc.network.type';
1141             $raw .= "$k = $v\n";
1142         }
1143     }
1144
1145     $raw .= "lxc.network.type = empty\n" if !$netcount;
1146
1147     File::Path::mkpath("$dir/rootfs");
1148
1149     PVE::Tools::file_set_contents("$dir/config", $raw);
1150 }
1151
1152 # verify and cleanup nameserver list (replace \0 with ' ')
1153 sub verify_nameserver_list {
1154     my ($nameserver_list) = @_;
1155
1156     my @list = ();
1157     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1158         PVE::JSONSchema::pve_verify_ip($server);
1159         push @list, $server;
1160     }
1161
1162     return join(' ', @list);
1163 }
1164
1165 sub verify_searchdomain_list {
1166     my ($searchdomain_list) = @_;
1167
1168     my @list = ();
1169     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1170         # todo: should we add checks for valid dns domains?
1171         push @list, $server;
1172     }
1173
1174     return join(' ', @list);
1175 }
1176
1177 sub add_unused_volume {
1178     my ($config, $volid) = @_;
1179
1180     my $key;
1181     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1182         my $test = "unused$ind";
1183         if (my $vid = $config->{$test}) {
1184             return if $vid eq $volid; # do not add duplicates
1185         } else {
1186             $key = $test;
1187         }
1188     }
1189
1190     die "Too many unused volumes - please delete them first.\n" if !$key;
1191
1192     $config->{$key} = $volid;
1193
1194     return $key;
1195 }
1196
1197 sub update_pct_config {
1198     my ($vmid, $conf, $running, $param, $delete) = @_;
1199
1200     my @nohotplug;
1201
1202     my $new_disks = 0;
1203     my @deleted_volumes;
1204
1205     my $rootdir;
1206     if ($running) {
1207         my $pid = find_lxc_pid($vmid);
1208         $rootdir = "/proc/$pid/root";
1209     }
1210
1211     my $hotplug_error = sub {
1212         if ($running) {
1213             push @nohotplug, @_;
1214             return 1;
1215         } else {
1216             return 0;
1217         }
1218     };
1219
1220     if (defined($delete)) {
1221         foreach my $opt (@$delete) {
1222             if (!exists($conf->{$opt})) {
1223                 warn "no such option: $opt\n";
1224                 next;
1225             }
1226
1227             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1228                 die "unable to delete required option '$opt'\n";
1229             } elsif ($opt eq 'swap') {
1230                 delete $conf->{$opt};
1231                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1232             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1233                 delete $conf->{$opt};
1234             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1235                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1236                 next if $hotplug_error->($opt);
1237                 delete $conf->{$opt};
1238             } elsif ($opt =~ m/^net(\d)$/) {
1239                 delete $conf->{$opt};
1240                 next if !$running;
1241                 my $netid = $1;
1242                 PVE::Network::veth_delete("veth${vmid}i$netid");
1243             } elsif ($opt eq 'protection') {
1244                 delete $conf->{$opt};
1245             } elsif ($opt =~ m/^unused(\d+)$/) {
1246                 next if $hotplug_error->($opt);
1247                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1248                 push @deleted_volumes, $conf->{$opt};
1249                 delete $conf->{$opt};
1250             } elsif ($opt =~ m/^mp(\d+)$/) {
1251                 next if $hotplug_error->($opt);
1252                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1253                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1254                 if ($mountpoint->{type} eq 'volume') {
1255                     add_unused_volume($conf, $mountpoint->{volume})
1256                 }
1257                 delete $conf->{$opt};
1258             } elsif ($opt eq 'unprivileged') {
1259                 die "unable to delete read-only option: '$opt'\n";
1260             } else {
1261                 die "implement me (delete: $opt)"
1262             }
1263             write_config($vmid, $conf) if $running;
1264         }
1265     }
1266
1267     # There's no separate swap size to configure, there's memory and "total"
1268     # memory (iow. memory+swap). This means we have to change them together.
1269     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1270     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1271     if (defined($wanted_memory) || defined($wanted_swap)) {
1272
1273         my $old_memory = ($conf->{memory} || 512);
1274         my $old_swap = ($conf->{swap} || 0);
1275
1276         $wanted_memory //= $old_memory;
1277         $wanted_swap //= $old_swap;
1278
1279         my $total = $wanted_memory + $wanted_swap;
1280         if ($running) {
1281             my $old_total = $old_memory + $old_swap;
1282             if ($total > $old_total) {
1283                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1284                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1285             } else {
1286                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1287                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1288             }
1289         }
1290         $conf->{memory} = $wanted_memory;
1291         $conf->{swap} = $wanted_swap;
1292
1293         write_config($vmid, $conf) if $running;
1294     }
1295
1296     foreach my $opt (keys %$param) {
1297         my $value = $param->{$opt};
1298         if ($opt eq 'hostname') {
1299             $conf->{$opt} = $value;
1300         } elsif ($opt eq 'onboot') {
1301             $conf->{$opt} = $value ? 1 : 0;
1302         } elsif ($opt eq 'startup') {
1303             $conf->{$opt} = $value;
1304         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1305             next if $hotplug_error->($opt);
1306             $conf->{$opt} = $value;
1307         } elsif ($opt eq 'nameserver') {
1308             next if $hotplug_error->($opt);
1309             my $list = verify_nameserver_list($value);
1310             $conf->{$opt} = $list;
1311         } elsif ($opt eq 'searchdomain') {
1312             next if $hotplug_error->($opt);
1313             my $list = verify_searchdomain_list($value);
1314             $conf->{$opt} = $list;
1315         } elsif ($opt eq 'cpulimit') {
1316             next if $hotplug_error->($opt); # FIXME: hotplug
1317             $conf->{$opt} = $value;
1318         } elsif ($opt eq 'cpuunits') {
1319             $conf->{$opt} = $value;
1320             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1321         } elsif ($opt eq 'description') {
1322             $conf->{$opt} = PVE::Tools::encode_text($value);
1323         } elsif ($opt =~ m/^net(\d+)$/) {
1324             my $netid = $1;
1325             my $net = parse_lxc_network($value);
1326             if (!$running) {
1327                 $conf->{$opt} = print_lxc_network($net);
1328             } else {
1329                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1330             }
1331         } elsif ($opt eq 'protection') {
1332             $conf->{$opt} = $value ? 1 : 0;
1333         } elsif ($opt =~ m/^mp(\d+)$/) {
1334             next if $hotplug_error->($opt);
1335             check_protection($conf, "can't update CT $vmid drive '$opt'");
1336             $conf->{$opt} = $value;
1337             $new_disks = 1;
1338         } elsif ($opt eq 'rootfs') {
1339             check_protection($conf, "can't update CT $vmid drive '$opt'");
1340             die "implement me: $opt";
1341         } elsif ($opt eq 'unprivileged') {
1342             die "unable to modify read-only option: '$opt'\n";
1343         } else {
1344             die "implement me: $opt";
1345         }
1346         write_config($vmid, $conf) if $running;
1347     }
1348
1349     if (@deleted_volumes) {
1350         my $storage_cfg = PVE::Storage::config();
1351         foreach my $volume (@deleted_volumes) {
1352             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1353         }
1354     }
1355
1356     if ($new_disks) {
1357         my $storage_cfg = PVE::Storage::config();
1358         create_disks($storage_cfg, $vmid, $conf, $conf);
1359     }
1360
1361     # This should be the last thing we do here
1362     if ($running && scalar(@nohotplug)) {
1363         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1364     }
1365 }
1366
1367 sub has_dev_console {
1368     my ($conf) = @_;
1369
1370     return !(defined($conf->{console}) && !$conf->{console});
1371 }
1372
1373 sub get_tty_count {
1374     my ($conf) = @_;
1375
1376     return $conf->{tty} // $confdesc->{tty}->{default};
1377 }
1378
1379 sub get_cmode {
1380     my ($conf) = @_;
1381
1382     return $conf->{cmode} // $confdesc->{cmode}->{default};
1383 }
1384
1385 sub get_console_command {
1386     my ($vmid, $conf) = @_;
1387
1388     my $cmode = get_cmode($conf);
1389
1390     if ($cmode eq 'console') {
1391         return ['lxc-console', '-n',  $vmid, '-t', 0];
1392     } elsif ($cmode eq 'tty') {
1393         return ['lxc-console', '-n',  $vmid];
1394     } elsif ($cmode eq 'shell') {
1395         return ['lxc-attach', '--clear-env', '-n', $vmid];
1396     } else {
1397         die "internal error";
1398     }
1399 }
1400
1401 sub get_primary_ips {
1402     my ($conf) = @_;
1403
1404     # return data from net0
1405
1406     return undef if !defined($conf->{net0});
1407     my $net = parse_lxc_network($conf->{net0});
1408
1409     my $ipv4 = $net->{ip};
1410     if ($ipv4) {
1411         if ($ipv4 =~ /^(dhcp|manual)$/) {
1412             $ipv4 = undef
1413         } else {
1414             $ipv4 =~ s!/\d+$!!;
1415         }
1416     }
1417     my $ipv6 = $net->{ip6};
1418     if ($ipv6) {
1419         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1420             $ipv6 = undef;
1421         } else {
1422             $ipv6 =~ s!/\d+$!!;
1423         }
1424     }
1425
1426     return ($ipv4, $ipv6);
1427 }
1428
1429 sub delete_mountpoint_volume {
1430     my ($storage_cfg, $vmid, $volume) = @_;
1431
1432     return if classify_mountpoint($volume) ne 'volume';
1433
1434     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1435     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1436 }
1437
1438 sub destroy_lxc_container {
1439     my ($storage_cfg, $vmid, $conf) = @_;
1440
1441     foreach_mountpoint($conf, sub {
1442         my ($ms, $mountpoint) = @_;
1443         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1444     });
1445
1446     rmdir "/var/lib/lxc/$vmid/rootfs";
1447     unlink "/var/lib/lxc/$vmid/config";
1448     rmdir "/var/lib/lxc/$vmid";
1449     destroy_config($vmid);
1450
1451     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1452     #PVE::Tools::run_command($cmd);
1453 }
1454
1455 sub vm_stop_cleanup {
1456     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1457
1458     eval {
1459         if (!$keepActive) {
1460
1461             my $vollist = get_vm_volumes($conf);
1462             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1463         }
1464     };
1465     warn $@ if $@; # avoid errors - just warn
1466 }
1467
1468 my $safe_num_ne = sub {
1469     my ($a, $b) = @_;
1470
1471     return 0 if !defined($a) && !defined($b);
1472     return 1 if !defined($a);
1473     return 1 if !defined($b);
1474
1475     return $a != $b;
1476 };
1477
1478 my $safe_string_ne = sub {
1479     my ($a, $b) = @_;
1480
1481     return 0 if !defined($a) && !defined($b);
1482     return 1 if !defined($a);
1483     return 1 if !defined($b);
1484
1485     return $a ne $b;
1486 };
1487
1488 sub update_net {
1489     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1490
1491     if ($newnet->{type} ne 'veth') {
1492         # for when there are physical interfaces
1493         die "cannot update interface of type $newnet->{type}";
1494     }
1495
1496     my $veth = "veth${vmid}i${netid}";
1497     my $eth = $newnet->{name};
1498
1499     if (my $oldnetcfg = $conf->{$opt}) {
1500         my $oldnet = parse_lxc_network($oldnetcfg);
1501
1502         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1503             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1504
1505             PVE::Network::veth_delete($veth);
1506             delete $conf->{$opt};
1507             write_config($vmid, $conf);
1508
1509             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1510
1511         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1512                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1513                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1514
1515                 if ($oldnet->{bridge}) {
1516                     PVE::Network::tap_unplug($veth);
1517                     foreach (qw(bridge tag firewall)) {
1518                         delete $oldnet->{$_};
1519                     }
1520                     $conf->{$opt} = print_lxc_network($oldnet);
1521                     write_config($vmid, $conf);
1522                 }
1523
1524                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1525                 foreach (qw(bridge tag firewall)) {
1526                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1527                 }
1528                 $conf->{$opt} = print_lxc_network($oldnet);
1529                 write_config($vmid, $conf);
1530         }
1531     } else {
1532         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1533     }
1534
1535     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1536 }
1537
1538 sub hotplug_net {
1539     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1540
1541     my $veth = "veth${vmid}i${netid}";
1542     my $vethpeer = $veth . "p";
1543     my $eth = $newnet->{name};
1544
1545     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1546     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1547
1548     # attach peer in container
1549     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1550     PVE::Tools::run_command($cmd);
1551
1552     # link up peer in container
1553     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1554     PVE::Tools::run_command($cmd);
1555
1556     my $done = { type => 'veth' };
1557     foreach (qw(bridge tag firewall hwaddr name)) {
1558         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1559     }
1560     $conf->{$opt} = print_lxc_network($done);
1561
1562     write_config($vmid, $conf);
1563 }
1564
1565 sub update_ipconfig {
1566     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1567
1568     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1569
1570     my $optdata = parse_lxc_network($conf->{$opt});
1571     my $deleted = [];
1572     my $added = [];
1573     my $nscmd = sub {
1574         my $cmdargs = shift;
1575         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1576     };
1577     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1578
1579     my $change_ip_config = sub {
1580         my ($ipversion) = @_;
1581
1582         my $family_opt = "-$ipversion";
1583         my $suffix = $ipversion == 4 ? '' : $ipversion;
1584         my $gw= "gw$suffix";
1585         my $ip= "ip$suffix";
1586
1587         my $newip = $newnet->{$ip};
1588         my $newgw = $newnet->{$gw};
1589         my $oldip = $optdata->{$ip};
1590
1591         my $change_ip = &$safe_string_ne($oldip, $newip);
1592         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1593
1594         return if !$change_ip && !$change_gw;
1595
1596         # step 1: add new IP, if this fails we cancel
1597         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1598         if ($change_ip && $is_real_ip) {
1599             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1600             if (my $err = $@) {
1601                 warn $err;
1602                 return;
1603             }
1604         }
1605
1606         # step 2: replace gateway
1607         #   If this fails we delete the added IP and cancel.
1608         #   If it succeeds we save the config and delete the old IP, ignoring
1609         #   errors. The config is then saved.
1610         # Note: 'ip route replace' can add
1611         if ($change_gw) {
1612             if ($newgw) {
1613                 eval {
1614                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1615                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1616                     }
1617                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1618                 };
1619                 if (my $err = $@) {
1620                     warn $err;
1621                     # the route was not replaced, the old IP is still available
1622                     # rollback (delete new IP) and cancel
1623                     if ($change_ip) {
1624                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1625                         warn $@ if $@; # no need to die here
1626                     }
1627                     return;
1628                 }
1629             } else {
1630                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1631                 # if the route was not deleted, the guest might have deleted it manually
1632                 # warn and continue
1633                 warn $@ if $@;
1634             }
1635         }
1636
1637         # from this point on we save the configuration
1638         # step 3: delete old IP ignoring errors
1639         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1640             # We need to enable promote_secondaries, otherwise our newly added
1641             # address will be removed along with the old one.
1642             my $promote = 0;
1643             eval {
1644                 if ($ipversion == 4) {
1645                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1646                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1647                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1648                 }
1649                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1650             };
1651             warn $@ if $@; # no need to die here
1652
1653             if ($ipversion == 4) {
1654                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1655             }
1656         }
1657
1658         foreach my $property ($ip, $gw) {
1659             if ($newnet->{$property}) {
1660                 $optdata->{$property} = $newnet->{$property};
1661             } else {
1662                 delete $optdata->{$property};
1663             }
1664         }
1665         $conf->{$opt} = print_lxc_network($optdata);
1666         write_config($vmid, $conf);
1667         $lxc_setup->setup_network($conf);
1668     };
1669
1670     &$change_ip_config(4);
1671     &$change_ip_config(6);
1672
1673 }
1674
1675 # Internal snapshots
1676
1677 # NOTE: Snapshot create/delete involves several non-atomic
1678 # actions, and can take a long time.
1679 # So we try to avoid locking the file and use the 'lock' variable
1680 # inside the config file instead.
1681
1682 my $snapshot_copy_config = sub {
1683     my ($source, $dest) = @_;
1684
1685     foreach my $k (keys %$source) {
1686         next if $k eq 'snapshots';
1687         next if $k eq 'snapstate';
1688         next if $k eq 'snaptime';
1689         next if $k eq 'vmstate';
1690         next if $k eq 'lock';
1691         next if $k eq 'digest';
1692         next if $k eq 'description';
1693
1694         $dest->{$k} = $source->{$k};
1695     }
1696 };
1697
1698 my $snapshot_prepare = sub {
1699     my ($vmid, $snapname, $comment) = @_;
1700
1701     my $snap;
1702
1703     my $updatefn =  sub {
1704
1705         my $conf = load_config($vmid);
1706
1707         die "you can't take a snapshot if it's a template\n"
1708             if is_template($conf);
1709
1710         check_lock($conf);
1711
1712         $conf->{lock} = 'snapshot';
1713
1714         die "snapshot name '$snapname' already used\n"
1715             if defined($conf->{snapshots}->{$snapname});
1716
1717         my $storecfg = PVE::Storage::config();
1718         my $feature = $snapname eq 'vzdump' ? 'vzdump' : 'snapshot';
1719         die "snapshot feature is not available\n" if !has_feature($feature, $conf, $storecfg);
1720
1721         $snap = $conf->{snapshots}->{$snapname} = {};
1722
1723         &$snapshot_copy_config($conf, $snap);
1724
1725         $snap->{'snapstate'} = "prepare";
1726         $snap->{'snaptime'} = time();
1727         $snap->{'description'} = $comment if $comment;
1728         $conf->{snapshots}->{$snapname} = $snap;
1729
1730         write_config($vmid, $conf);
1731     };
1732
1733     lock_container($vmid, 10, $updatefn);
1734
1735     return $snap;
1736 };
1737
1738 my $snapshot_commit = sub {
1739     my ($vmid, $snapname) = @_;
1740
1741     my $updatefn = sub {
1742
1743         my $conf = load_config($vmid);
1744
1745         die "missing snapshot lock\n"
1746             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1747
1748         die "snapshot '$snapname' does not exist\n"
1749             if !defined($conf->{snapshots}->{$snapname});
1750
1751         die "wrong snapshot state\n"
1752             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1753                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1754
1755         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1756         delete $conf->{lock};
1757         $conf->{parent} = $snapname;
1758
1759         write_config($vmid, $conf);
1760     };
1761
1762     lock_container($vmid, 10 ,$updatefn);
1763 };
1764
1765 sub has_feature {
1766     my ($feature, $conf, $storecfg, $snapname) = @_;
1767
1768     my $err;
1769     my $vzdump = $feature eq 'vzdump';
1770     $feature = 'snapshot' if $vzdump;
1771
1772     foreach_mountpoint($conf, sub {
1773         my ($ms, $mountpoint) = @_;
1774
1775         return if $err; # skip further test
1776         return if $vzdump && $ms ne 'rootfs' && !$mountpoint->{backup};
1777
1778         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1779
1780         # TODO: implement support for mountpoints
1781         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1782             if $ms ne 'rootfs';
1783     });
1784
1785     return $err ? 0 : 1;
1786 }
1787
1788 sub snapshot_create {
1789     my ($vmid, $snapname, $comment) = @_;
1790
1791     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1792
1793     my $conf = load_config($vmid);
1794
1795     my $running = check_running($vmid);
1796
1797     my $unfreeze = 0;
1798
1799     my $drivehash = {};
1800
1801     eval {
1802         if ($running) {
1803             $unfreeze = 1;
1804             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1805             PVE::Tools::run_command(['/bin/sync']);
1806         };
1807
1808         my $storecfg = PVE::Storage::config();
1809         my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1810         my $volid = $rootinfo->{volume};
1811
1812         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1813         $drivehash->{rootfs} = 1;
1814     };
1815     my $err = $@;
1816
1817     if ($unfreeze) {
1818         eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
1819         warn $@ if $@;
1820     }
1821
1822     if ($err) {
1823         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
1824         warn "$@\n" if $@;
1825         die "$err\n";
1826     }
1827
1828     &$snapshot_commit($vmid, $snapname);
1829 }
1830
1831 # Note: $drivehash is only set when called from snapshot_create.
1832 sub snapshot_delete {
1833     my ($vmid, $snapname, $force, $drivehash) = @_;
1834
1835     my $snap;
1836
1837     my $conf;
1838
1839     my $updatefn =  sub {
1840
1841         $conf = load_config($vmid);
1842
1843         die "you can't delete a snapshot if vm is a template\n"
1844             if is_template($conf);
1845
1846         $snap = $conf->{snapshots}->{$snapname};
1847
1848         if (!$drivehash) {
1849             check_lock($conf);
1850         }
1851
1852         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1853
1854         $snap->{snapstate} = 'delete';
1855
1856         write_config($vmid, $conf);
1857     };
1858
1859     lock_container($vmid, 10, $updatefn);
1860
1861     my $storecfg = PVE::Storage::config();
1862
1863     my $unlink_parent = sub {
1864
1865         my ($confref, $new_parent) = @_;
1866
1867         if ($confref->{parent} && $confref->{parent} eq $snapname) {
1868             if ($new_parent) {
1869                 $confref->{parent} = $new_parent;
1870             } else {
1871                 delete $confref->{parent};
1872             }
1873         }
1874     };
1875
1876     my $del_snap =  sub {
1877
1878         $conf = load_config($vmid);
1879
1880         if ($drivehash) {
1881             delete $conf->{lock};
1882         } else {
1883             check_lock($conf);
1884         }
1885
1886         my $parent = $conf->{snapshots}->{$snapname}->{parent};
1887         foreach my $snapkey (keys %{$conf->{snapshots}}) {
1888             &$unlink_parent($conf->{snapshots}->{$snapkey}, $parent);
1889         }
1890
1891         &$unlink_parent($conf, $parent);
1892
1893         delete $conf->{snapshots}->{$snapname};
1894
1895         write_config($vmid, $conf);
1896     };
1897
1898     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1899     my $rootinfo = parse_ct_rootfs($rootfs);
1900     my $volid = $rootinfo->{volume};
1901
1902     eval {
1903         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1904     };
1905     my $err = $@;
1906
1907     if(!$err || ($err && $force)) {
1908         lock_container($vmid, 10, $del_snap);
1909         if ($err) {
1910             die "Can't delete snapshot: $vmid $snapname $err\n";
1911         }
1912     }
1913 }
1914
1915 sub snapshot_rollback {
1916     my ($vmid, $snapname) = @_;
1917
1918     my $storecfg = PVE::Storage::config();
1919
1920     my $conf = load_config($vmid);
1921
1922     die "you can't rollback if vm is a template\n" if is_template($conf);
1923
1924     my $snap = $conf->{snapshots}->{$snapname};
1925
1926     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1927
1928     my $rootfs = $snap->{rootfs};
1929     my $rootinfo = parse_ct_rootfs($rootfs);
1930     my $volid = $rootinfo->{volume};
1931
1932     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1933
1934     my $updatefn = sub {
1935
1936         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1937             if $snap->{snapstate};
1938
1939         check_lock($conf);
1940
1941         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1942
1943         die "unable to rollback vm $vmid: vm is running\n"
1944             if check_running($vmid);
1945
1946         $conf->{lock} = 'rollback';
1947
1948         my $forcemachine;
1949
1950         # copy snapshot config to current config
1951
1952         my $tmp_conf = $conf;
1953         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1954         $conf->{snapshots} = $tmp_conf->{snapshots};
1955         delete $conf->{snaptime};
1956         delete $conf->{snapname};
1957         $conf->{parent} = $snapname;
1958
1959         write_config($vmid, $conf);
1960     };
1961
1962     my $unlockfn = sub {
1963         delete $conf->{lock};
1964         write_config($vmid, $conf);
1965     };
1966
1967     lock_container($vmid, 10, $updatefn);
1968
1969     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1970
1971     lock_container($vmid, 5, $unlockfn);
1972 }
1973
1974 sub template_create {
1975     my ($vmid, $conf) = @_;
1976
1977     my $storecfg = PVE::Storage::config();
1978
1979     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1980     my $volid = $rootinfo->{volume};
1981
1982     die "Template feature is not available for '$volid'\n"
1983         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1984
1985     PVE::Storage::activate_volumes($storecfg, [$volid]);
1986
1987     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1988     $rootinfo->{volume} = $template_volid;
1989     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1990
1991     write_config($vmid, $conf);
1992 }
1993
1994 sub is_template {
1995     my ($conf) = @_;
1996
1997     return 1 if defined $conf->{template} && $conf->{template} == 1;
1998 }
1999
2000 sub mountpoint_names {
2001     my ($reverse) = @_;
2002
2003     my @names = ('rootfs');
2004
2005     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
2006         push @names, "mp$i";
2007     }
2008
2009     return $reverse ? reverse @names : @names;
2010 }
2011
2012
2013 sub foreach_mountpoint_full {
2014     my ($conf, $reverse, $func) = @_;
2015
2016     foreach my $key (mountpoint_names($reverse)) {
2017         my $value = $conf->{$key};
2018         next if !defined($value);
2019         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2020         next if !defined($mountpoint);
2021
2022         &$func($key, $mountpoint);
2023     }
2024 }
2025
2026 sub foreach_mountpoint {
2027     my ($conf, $func) = @_;
2028
2029     foreach_mountpoint_full($conf, 0, $func);
2030 }
2031
2032 sub foreach_mountpoint_reverse {
2033     my ($conf, $func) = @_;
2034
2035     foreach_mountpoint_full($conf, 1, $func);
2036 }
2037
2038 sub check_ct_modify_config_perm {
2039     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2040
2041     return 1 if $authuser ne 'root@pam';
2042
2043     foreach my $opt (@$key_list) {
2044
2045         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2046             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2047         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2048             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2049         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2050             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2051         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2052                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2053             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2054         } else {
2055             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2056         }
2057     }
2058
2059     return 1;
2060 }
2061
2062 sub umount_all {
2063     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2064
2065     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2066     my $volid_list = get_vm_volumes($conf);
2067
2068     foreach_mountpoint_reverse($conf, sub {
2069         my ($ms, $mountpoint) = @_;
2070
2071         my $volid = $mountpoint->{volume};
2072         my $mount = $mountpoint->{mp};
2073
2074         return if !$volid || !$mount;
2075
2076         my $mount_path = "$rootdir/$mount";
2077         $mount_path =~ s!/+!/!g;
2078
2079         return if !PVE::ProcFSTools::is_mounted($mount_path);
2080
2081         eval {
2082             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2083         };
2084         if (my $err = $@) {
2085             if ($noerr) {
2086                 warn $err;
2087             } else {
2088                 die $err;
2089             }
2090         }
2091     });
2092 }
2093
2094 sub mount_all {
2095     my ($vmid, $storage_cfg, $conf) = @_;
2096
2097     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2098     File::Path::make_path($rootdir);
2099
2100     my $volid_list = get_vm_volumes($conf);
2101     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2102
2103     eval {
2104         foreach_mountpoint($conf, sub {
2105             my ($ms, $mountpoint) = @_;
2106
2107             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2108         });
2109     };
2110     if (my $err = $@) {
2111         warn "mounting container failed\n";
2112         umount_all($vmid, $storage_cfg, $conf, 1);
2113         die $err;
2114     }
2115
2116     return $rootdir;
2117 }
2118
2119
2120 sub mountpoint_mount_path {
2121     my ($mountpoint, $storage_cfg, $snapname) = @_;
2122
2123     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2124 }
2125
2126 my $check_mount_path = sub {
2127     my ($path) = @_;
2128     $path = File::Spec->canonpath($path);
2129     my $real = Cwd::realpath($path);
2130     if ($real ne $path) {
2131         die "mount path modified by symlink: $path != $real";
2132     }
2133 };
2134
2135 sub query_loopdev {
2136     my ($path) = @_;
2137     my $found;
2138     my $parser = sub {
2139         my $line = shift;
2140         if ($line =~ m@^(/dev/loop\d+):@) {
2141             $found = $1;
2142         }
2143     };
2144     my $cmd = ['losetup', '--associated', $path];
2145     PVE::Tools::run_command($cmd, outfunc => $parser);
2146     return $found;
2147 }
2148
2149 # use $rootdir = undef to just return the corresponding mount path
2150 sub mountpoint_mount {
2151     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2152
2153     my $volid = $mountpoint->{volume};
2154     my $mount = $mountpoint->{mp};
2155     my $type = $mountpoint->{type};
2156
2157     return if !$volid || !$mount;
2158
2159     my $mount_path;
2160
2161     if (defined($rootdir)) {
2162         $rootdir =~ s!/+$!!;
2163         $mount_path = "$rootdir/$mount";
2164         $mount_path =~ s!/+!/!g;
2165         &$check_mount_path($mount_path);
2166         File::Path::mkpath($mount_path);
2167     }
2168
2169     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2170
2171     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2172
2173     my $optstring = '';
2174     if (defined($mountpoint->{acl})) {
2175         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
2176     }
2177     if ($mountpoint->{ro}) {
2178         $optstring .= ',' if $optstring;
2179         $optstring .= 'ro';
2180     }
2181
2182     my @extra_opts = ('-o', $optstring);
2183
2184     if ($storage) {
2185
2186         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2187         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2188
2189         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2190             PVE::Storage::parse_volname($storage_cfg, $volid);
2191
2192         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2193
2194         if ($format eq 'subvol') {
2195             if ($mount_path) {
2196                 if ($snapname) {
2197                     if ($scfg->{type} eq 'zfspool') {
2198                         my $path_arg = $path;
2199                         $path_arg =~ s!^/+!!;
2200                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
2201                     } else {
2202                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2203                     }
2204                 } else {
2205                     if ($mountpoint->{ro}) {
2206                         die "read-only bind mounts not supported\n";
2207                     }
2208                     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $path, $mount_path]);
2209                 }
2210             }
2211             return wantarray ? ($path, 0) : $path;
2212         } elsif ($format eq 'raw' || $format eq 'iso') {
2213             my $use_loopdev = 0;
2214             if ($scfg->{path}) {
2215                 push @extra_opts, '-o', 'loop';
2216                 $use_loopdev = 1;
2217             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2218                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2219                 # do nothing
2220             } else {
2221                 die "unsupported storage type '$scfg->{type}'\n";
2222             }
2223             if ($mount_path) {
2224                 if ($format eq 'iso') {
2225                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2226                 } elsif ($isBase || defined($snapname)) {
2227                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2228                 } else {
2229                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2230                 }
2231             }
2232             return wantarray ? ($path, $use_loopdev) : $path;
2233         } else {
2234             die "unsupported image format '$format'\n";
2235         }
2236     } elsif ($type eq 'device') {
2237         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
2238         return wantarray ? ($volid, 0) : $volid;
2239     } elsif ($type eq 'bind') {
2240         if ($mountpoint->{ro}) {
2241             die "read-only bind mounts not supported\n";
2242             # Theoretically we'd have to execute both:
2243             # mount -o bind $a $b
2244             # mount -o bind,remount,ro $a $b
2245         }
2246         die "directory '$volid' does not exist\n" if ! -d $volid;
2247         &$check_mount_path($volid);
2248         PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $volid, $mount_path]) if $mount_path;
2249         return wantarray ? ($volid, 0) : $volid;
2250     }
2251
2252     die "unsupported storage";
2253 }
2254
2255 sub get_vm_volumes {
2256     my ($conf, $excludes) = @_;
2257
2258     my $vollist = [];
2259
2260     foreach_mountpoint($conf, sub {
2261         my ($ms, $mountpoint) = @_;
2262
2263         return if $excludes && $ms eq $excludes;
2264
2265         my $volid = $mountpoint->{volume};
2266
2267         return if !$volid || $mountpoint->{type} ne 'volume';
2268
2269         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2270         return if !$sid;
2271
2272         push @$vollist, $volid;
2273     });
2274
2275     return $vollist;
2276 }
2277
2278 sub mkfs {
2279     my ($dev, $rootuid, $rootgid) = @_;
2280
2281     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2282                              '-E', "root_owner=$rootuid:$rootgid",
2283                              $dev]);
2284 }
2285
2286 sub format_disk {
2287     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2288
2289     if ($volid =~ m!^/dev/.+!) {
2290         mkfs($volid);
2291         return;
2292     }
2293
2294     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2295
2296     die "cannot format volume '$volid' with no storage\n" if !$storage;
2297
2298     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2299
2300     my $path = PVE::Storage::path($storage_cfg, $volid);
2301
2302     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2303         PVE::Storage::parse_volname($storage_cfg, $volid);
2304
2305     die "cannot format volume '$volid' (format == $format)\n"
2306         if $format ne 'raw';
2307
2308     mkfs($path, $rootuid, $rootgid);
2309 }
2310
2311 sub destroy_disks {
2312     my ($storecfg, $vollist) = @_;
2313
2314     foreach my $volid (@$vollist) {
2315         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2316         warn $@ if $@;
2317     }
2318 }
2319
2320 sub create_disks {
2321     my ($storecfg, $vmid, $settings, $conf) = @_;
2322
2323     my $vollist = [];
2324
2325     eval {
2326         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2327         my $chown_vollist = [];
2328
2329         foreach_mountpoint($settings, sub {
2330             my ($ms, $mountpoint) = @_;
2331
2332             my $volid = $mountpoint->{volume};
2333             my $mp = $mountpoint->{mp};
2334
2335             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2336
2337             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2338                 my ($storeid, $size_gb) = ($1, $2);
2339
2340                 my $size_kb = int(${size_gb}*1024) * 1024;
2341
2342                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2343                 # fixme: use better naming ct-$vmid-disk-X.raw?
2344
2345                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2346                     if ($size_kb > 0) {
2347                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2348                                                            undef, $size_kb);
2349                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2350                     } else {
2351                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2352                                                            undef, 0);
2353                         push @$chown_vollist, $volid;
2354                     }
2355                 } elsif ($scfg->{type} eq 'zfspool') {
2356
2357                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2358                                                        undef, $size_kb);
2359                     push @$chown_vollist, $volid;
2360                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2361
2362                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2363                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2364
2365                 } elsif ($scfg->{type} eq 'rbd') {
2366
2367                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2368                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2369                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2370                 } else {
2371                     die "unable to create containers on storage type '$scfg->{type}'\n";
2372                 }
2373                 push @$vollist, $volid;
2374                 $mountpoint->{volume} = $volid;
2375                 $mountpoint->{size} = $size_kb * 1024;
2376                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2377             } else {
2378                 # use specified/existing volid/dir/device
2379                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2380             }
2381         });
2382
2383         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2384         foreach my $volid (@$chown_vollist) {
2385             my $path = PVE::Storage::path($storecfg, $volid, undef);
2386             chown($rootuid, $rootgid, $path);
2387         }
2388         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2389     };
2390     # free allocated images on error
2391     if (my $err = $@) {
2392         destroy_disks($storecfg, $vollist);
2393         die $err;
2394     }
2395     return $vollist;
2396 }
2397
2398 # bash completion helper
2399
2400 sub complete_os_templates {
2401     my ($cmdname, $pname, $cvalue) = @_;
2402
2403     my $cfg = PVE::Storage::config();
2404
2405     my $storeid;
2406
2407     if ($cvalue =~ m/^([^:]+):/) {
2408         $storeid = $1;
2409     }
2410
2411     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2412     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2413
2414     my $res = [];
2415     foreach my $id (keys %$data) {
2416         foreach my $item (@{$data->{$id}}) {
2417             push @$res, $item->{volid} if defined($item->{volid});
2418         }
2419     }
2420
2421     return $res;
2422 }
2423
2424 my $complete_ctid_full = sub {
2425     my ($running) = @_;
2426
2427     my $idlist = vmstatus();
2428
2429     my $active_hash = list_active_containers();
2430
2431     my $res = [];
2432
2433     foreach my $id (keys %$idlist) {
2434         my $d = $idlist->{$id};
2435         if (defined($running)) {
2436             next if $d->{template};
2437             next if $running && !$active_hash->{$id};
2438             next if !$running && $active_hash->{$id};
2439         }
2440         push @$res, $id;
2441
2442     }
2443     return $res;
2444 };
2445
2446 sub complete_ctid {
2447     return &$complete_ctid_full();
2448 }
2449
2450 sub complete_ctid_stopped {
2451     return &$complete_ctid_full(0);
2452 }
2453
2454 sub complete_ctid_running {
2455     return &$complete_ctid_full(1);
2456 }
2457
2458 sub parse_id_maps {
2459     my ($conf) = @_;
2460
2461     my $id_map = [];
2462     my $rootuid = 0;
2463     my $rootgid = 0;
2464
2465     my $lxc = $conf->{lxc};
2466     foreach my $entry (@$lxc) {
2467         my ($key, $value) = @$entry;
2468         next if $key ne 'lxc.id_map';
2469         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2470             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2471             push @$id_map, [$type, $ct, $host, $length];
2472             if ($ct == 0) {
2473                 $rootuid = $host if $type eq 'u';
2474                 $rootgid = $host if $type eq 'g';
2475             }
2476         } else {
2477             die "failed to parse id_map: $value\n";
2478         }
2479     }
2480
2481     if (!@$id_map && $conf->{unprivileged}) {
2482         # Should we read them from /etc/subuid?
2483         $id_map = [ ['u', '0', '100000', '65536'],
2484                     ['g', '0', '100000', '65536'] ];
2485         $rootuid = $rootgid = 100000;
2486     }
2487
2488     return ($id_map, $rootuid, $rootgid);
2489 }
2490
2491 sub userns_command {
2492     my ($id_map) = @_;
2493     if (@$id_map) {
2494         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2495     }
2496     return [];
2497 }
2498
2499 1;