src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format => 'pve-lxc-mp-string',
  42         format_description => 'volume',
  43         description => 'Volume, device or directory to mount into the container.',
  44     },
  45     backup => {
  46         type => 'boolean',
  47         format_description => '[1|0]',
  48         description => 'Whether to include the mountpoint in backups.',
  49         optional => 1,
  50     },
  51     size => {
  52         type => 'string',
  53         format => 'disk-size',
  54         format_description => 'DiskSize',
  55         description => 'Volume size (read only value).',
  56         optional => 1,
  57     },
  58     acl => {
  59         type => 'boolean',
  60         format_description => 'acl',
  61         description => 'Explicitly enable or disable ACL support.',
  62         optional => 1,
  63     },
  64     ro => {
  65         type => 'boolean',
  66         format_description => 'ro',
  67         description => 'Read-only mountpoint (not supported with bind mounts)',
  68         optional => 1,
  69     },
  70 };
  71
  72 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  73     type => 'string', format => $rootfs_desc,
  74     description => "Use volume as container root.",
  75     optional => 1,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  79     description => "The name of the snapshot.",
  80     type => 'string', format => 'pve-configid',
  81     maxLength => 40,
  82 });
  83
  84 my $confdesc = {
  85     lock => {
  86         optional => 1,
  87         type => 'string',
  88         description => "Lock/unlock the VM.",
  89         enum => [qw(migrate backup snapshot rollback)],
  90     },
  91     onboot => {
  92         optional => 1,
  93         type => 'boolean',
  94         description => "Specifies whether a VM will be started during system bootup.",
  95         default => 0,
  96     },
  97     startup => get_standard_option('pve-startup-order'),
  98     template => {
  99         optional => 1,
 100         type => 'boolean',
 101         description => "Enable/disable Template.",
 102         default => 0,
 103     },
 104     arch => {
 105         optional => 1,
 106         type => 'string',
 107         enum => ['amd64', 'i386'],
 108         description => "OS architecture type.",
 109         default => 'amd64',
 110     },
 111     ostype => {
 112         optional => 1,
 113         type => 'string',
 114         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux'],
 115         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 116     },
 117     console => {
 118         optional => 1,
 119         type => 'boolean',
 120         description => "Attach a console device (/dev/console) to the container.",
 121         default => 1,
 122     },
 123     tty => {
 124         optional => 1,
 125         type => 'integer',
 126         description => "Specify the number of tty available to the container",
 127         minimum => 0,
 128         maximum => 6,
 129         default => 2,
 130     },
 131     cpulimit => {
 132         optional => 1,
 133         type => 'number',
 134         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 135         minimum => 0,
 136         maximum => 128,
 137         default => 0,
 138     },
 139     cpuunits => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 143         minimum => 0,
 144         maximum => 500000,
 145         default => 1024,
 146     },
 147     memory => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of RAM for the VM in MB.",
 151         minimum => 16,
 152         default => 512,
 153     },
 154     swap => {
 155         optional => 1,
 156         type => 'integer',
 157         description => "Amount of SWAP for the VM in MB.",
 158         minimum => 0,
 159         default => 512,
 160     },
 161     hostname => {
 162         optional => 1,
 163         description => "Set a host name for the container.",
 164         type => 'string', format => 'dns-name',
 165         maxLength => 255,
 166     },
 167     description => {
 168         optional => 1,
 169         type => 'string',
 170         description => "Container description. Only used on the configuration web interface.",
 171     },
 172     searchdomain => {
 173         optional => 1,
 174         type => 'string', format => 'dns-name-list',
 175         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 176     },
 177     nameserver => {
 178         optional => 1,
 179         type => 'string', format => 'address-list',
 180         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 181     },
 182     rootfs => get_standard_option('pve-ct-rootfs'),
 183     parent => {
 184         optional => 1,
 185         type => 'string', format => 'pve-configid',
 186         maxLength => 40,
 187         description => "Parent snapshot name. This is used internally, and should not be modified.",
 188     },
 189     snaptime => {
 190         optional => 1,
 191         description => "Timestamp for snapshots.",
 192         type => 'integer',
 193         minimum => 0,
 194     },
 195     cmode => {
 196         optional => 1,
 197         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 198         type => 'string',
 199         enum => ['shell', 'console', 'tty'],
 200         default => 'tty',
 201     },
 202     protection => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 206         default => 0,
 207     },
 208     unprivileged => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 212         default => 0,
 213     },
 214 };
 215
 216 my $valid_lxc_conf_keys = {
 217     'lxc.include' => 1,
 218     'lxc.arch' => 1,
 219     'lxc.utsname' => 1,
 220     'lxc.haltsignal' => 1,
 221     'lxc.rebootsignal' => 1,
 222     'lxc.stopsignal' => 1,
 223     'lxc.init_cmd' => 1,
 224     'lxc.network.type' => 1,
 225     'lxc.network.flags' => 1,
 226     'lxc.network.link' => 1,
 227     'lxc.network.mtu' => 1,
 228     'lxc.network.name' => 1,
 229     'lxc.network.hwaddr' => 1,
 230     'lxc.network.ipv4' => 1,
 231     'lxc.network.ipv4.gateway' => 1,
 232     'lxc.network.ipv6' => 1,
 233     'lxc.network.ipv6.gateway' => 1,
 234     'lxc.network.script.up' => 1,
 235     'lxc.network.script.down' => 1,
 236     'lxc.pts' => 1,
 237     'lxc.console.logfile' => 1,
 238     'lxc.console' => 1,
 239     'lxc.tty' => 1,
 240     'lxc.devttydir' => 1,
 241     'lxc.hook.autodev' => 1,
 242     'lxc.autodev' => 1,
 243     'lxc.kmsg' => 1,
 244     'lxc.mount' => 1,
 245     'lxc.mount.entry' => 1,
 246     'lxc.mount.auto' => 1,
 247     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 248     'lxc.rootfs.mount' => 1,
 249     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 250                             ', please use mountpoint options in the "rootfs" key',
 251     # lxc.cgroup.*
 252     'lxc.cap.drop' => 1,
 253     'lxc.cap.keep' => 1,
 254     'lxc.aa_profile' => 1,
 255     'lxc.aa_allow_incomplete' => 1,
 256     'lxc.se_context' => 1,
 257     'lxc.seccomp' => 1,
 258     'lxc.id_map' => 1,
 259     'lxc.hook.pre-start' => 1,
 260     'lxc.hook.pre-mount' => 1,
 261     'lxc.hook.mount' => 1,
 262     'lxc.hook.start' => 1,
 263     'lxc.hook.stop' => 1,
 264     'lxc.hook.post-stop' => 1,
 265     'lxc.hook.clone' => 1,
 266     'lxc.hook.destroy' => 1,
 267     'lxc.loglevel' => 1,
 268     'lxc.logfile' => 1,
 269     'lxc.start.auto' => 1,
 270     'lxc.start.delay' => 1,
 271     'lxc.start.order' => 1,
 272     'lxc.group' => 1,
 273     'lxc.environment' => 1,
 274 };
 275
 276 my $netconf_desc = {
 277     type => {
 278         type => 'string',
 279         optional => 1,
 280         description => "Network interface type.",
 281         enum => [qw(veth)],
 282     },
 283     name => {
 284         type => 'string',
 285         format_description => 'String',
 286         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 287         pattern => '[-_.\w\d]+',
 288     },
 289     bridge => {
 290         type => 'string',
 291         format_description => 'vmbr<Number>',
 292         description => 'Bridge to attach the network device to.',
 293         pattern => '[-_.\w\d]+',
 294         optional => 1,
 295     },
 296     hwaddr => {
 297         type => 'string',
 298         format_description => 'MAC',
 299         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 300         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 301         optional => 1,
 302     },
 303     mtu => {
 304         type => 'integer',
 305         format_description => 'Number',
 306         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 307         minimum => 64, # minimum ethernet frame is 64 bytes
 308         optional => 1,
 309     },
 310     ip => {
 311         type => 'string',
 312         format => 'pve-ipv4-config',
 313         format_description => 'IPv4Format/CIDR',
 314         description => 'IPv4 address in CIDR format.',
 315         optional => 1,
 316     },
 317     gw => {
 318         type => 'string',
 319         format => 'ipv4',
 320         format_description => 'GatewayIPv4',
 321         description => 'Default gateway for IPv4 traffic.',
 322         optional => 1,
 323     },
 324     ip6 => {
 325         type => 'string',
 326         format => 'pve-ipv6-config',
 327         format_description => 'IPv6Format/CIDR',
 328         description => 'IPv6 address in CIDR format.',
 329         optional => 1,
 330     },
 331     gw6 => {
 332         type => 'string',
 333         format => 'ipv6',
 334         format_description => 'GatewayIPv6',
 335         description => 'Default gateway for IPv6 traffic.',
 336         optional => 1,
 337     },
 338     firewall => {
 339         type => 'boolean',
 340         format_description => '[1|0]',
 341         description => "Controls whether this interface's firewall rules should be used.",
 342         optional => 1,
 343     },
 344     tag => {
 345         type => 'integer',
 346         format_description => 'VlanNo',
 347         minimum => '2',
 348         maximum => '4094',
 349         description => "VLAN tag for this interface.",
 350         optional => 1,
 351     },
 352     trunks => {
 353         type => 'string',
 354         pattern => qr/\d+(?:;\d+)*/,
 355         format_description => 'vlanid[;vlanid...]',
 356         description => "VLAN ids to pass through the interface",
 357         optional => 1,
 358     },
 359 };
 360 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 361
 362 my $MAX_LXC_NETWORKS = 10;
 363 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 364     $confdesc->{"net$i"} = {
 365         optional => 1,
 366         type => 'string', format => $netconf_desc,
 367         description => "Specifies network interfaces for the container.",
 368     };
 369 }
 370
 371 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 372 sub verify_lxc_mp_string{
 373     my ($mp, $noerr) = @_;
 374
 375     # do not allow:
 376     # /./ or /../
 377     # /. or /.. at the end
 378     # ../ at the beginning
 379
 380     if($mp =~ m@/\.\.?/@ ||
 381        $mp =~ m@/\.\.?$@ ||
 382        $mp =~ m@^\.\./@){
 383         return undef if $noerr;
 384         die "$mp contains illegal character sequences\n";
 385     }
 386     return $mp;
 387 }
 388
 389 my $mp_desc = {
 390     %$rootfs_desc,
 391     mp => {
 392         type => 'string',
 393         format => 'pve-lxc-mp-string',
 394         format_description => 'Path',
 395         description => 'Path to the mountpoint as seen from inside the container.',
 396     },
 397 };
 398 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 399
 400 my $unuseddesc = {
 401     optional => 1,
 402     type => 'string', format => 'pve-volume-id',
 403     description => "Reference to unused volumes.",
 404 };
 405
 406 my $MAX_MOUNT_POINTS = 10;
 407 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 408     $confdesc->{"mp$i"} = {
 409         optional => 1,
 410         type => 'string', format => $mp_desc,
 411         description => "Use volume as container mount point (experimental feature).",
 412         optional => 1,
 413     };
 414 }
 415
 416 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 417 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 418     $confdesc->{"unused$i"} = $unuseddesc;
 419 }
 420
 421 sub write_pct_config {
 422     my ($filename, $conf) = @_;
 423
 424     delete $conf->{snapstate}; # just to be sure
 425
 426     my $generate_raw_config = sub {
 427         my ($conf) = @_;
 428
 429         my $raw = '';
 430
 431         # add description as comment to top of file
 432         my $descr = $conf->{description} || '';
 433         foreach my $cl (split(/\n/, $descr)) {
 434             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 435         }
 436
 437         foreach my $key (sort keys %$conf) {
 438             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 439                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 440             my $value = $conf->{$key};
 441             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 442             $raw .= "$key: $value\n";
 443         }
 444
 445         if (my $lxcconf = $conf->{lxc}) {
 446             foreach my $entry (@$lxcconf) {
 447                 my ($k, $v) = @$entry;
 448                 $raw .= "$k: $v\n";
 449             }
 450         }
 451
 452         return $raw;
 453     };
 454
 455     my $raw = &$generate_raw_config($conf);
 456
 457     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 458         $raw .= "\n[$snapname]\n";
 459         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 460     }
 461
 462     return $raw;
 463 }
 464
 465 sub check_type {
 466     my ($key, $value) = @_;
 467
 468     die "unknown setting '$key'\n" if !$confdesc->{$key};
 469
 470     my $type = $confdesc->{$key}->{type};
 471
 472     if (!defined($value)) {
 473         die "got undefined value\n";
 474     }
 475
 476     if ($value =~ m/[\n\r]/) {
 477         die "property contains a line feed\n";
 478     }
 479
 480     if ($type eq 'boolean') {
 481         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 482         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 483         die "type check ('boolean') failed - got '$value'\n";
 484     } elsif ($type eq 'integer') {
 485         return int($1) if $value =~ m/^(\d+)$/;
 486         die "type check ('integer') failed - got '$value'\n";
 487     } elsif ($type eq 'number') {
 488         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 489         die "type check ('number') failed - got '$value'\n";
 490     } elsif ($type eq 'string') {
 491         if (my $fmt = $confdesc->{$key}->{format}) {
 492             PVE::JSONSchema::check_format($fmt, $value);
 493             return $value;
 494         }
 495         return $value;
 496     } else {
 497         die "internal error"
 498     }
 499 }
 500
 501 sub parse_pct_config {
 502     my ($filename, $raw) = @_;
 503
 504     return undef if !defined($raw);
 505
 506     my $res = {
 507         digest => Digest::SHA::sha1_hex($raw),
 508         snapshots => {},
 509     };
 510
 511     $filename =~ m|/lxc/(\d+).conf$|
 512         || die "got strange filename '$filename'";
 513
 514     my $vmid = $1;
 515
 516     my $conf = $res;
 517     my $descr = '';
 518     my $section = '';
 519
 520     my @lines = split(/\n/, $raw);
 521     foreach my $line (@lines) {
 522         next if $line =~ m/^\s*$/;
 523
 524         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 525             $section = $1;
 526             $conf->{description} = $descr if $descr;
 527             $descr = '';
 528             $conf = $res->{snapshots}->{$section} = {};
 529             next;
 530         }
 531
 532         if ($line =~ m/^\#(.*)\s*$/) {
 533             $descr .= PVE::Tools::decode_text($1) . "\n";
 534             next;
 535         }
 536
 537         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 538             my $key = $1;
 539             my $value = $3;
 540             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 541             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 542                 push @{$conf->{lxc}}, [$key, $value];
 543             } elsif (my $errmsg = $validity) {
 544                 warn "vm $vmid - $key: $errmsg\n";
 545             } else {
 546                 warn "vm $vmid - unable to parse config: $line\n";
 547             }
 548         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 549             $descr .= PVE::Tools::decode_text($2);
 550         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 551             $conf->{snapstate} = $1;
 552         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 553             my $key = $1;
 554             my $value = $2;
 555             eval { $value = check_type($key, $value); };
 556             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 557             $conf->{$key} = $value;
 558         } else {
 559             warn "vm $vmid - unable to parse config: $line\n";
 560         }
 561     }
 562
 563     $conf->{description} = $descr if $descr;
 564
 565     delete $res->{snapstate}; # just to be sure
 566
 567     return $res;
 568 }
 569
 570 sub config_list {
 571     my $vmlist = PVE::Cluster::get_vmlist();
 572     my $res = {};
 573     return $res if !$vmlist || !$vmlist->{ids};
 574     my $ids = $vmlist->{ids};
 575
 576     foreach my $vmid (keys %$ids) {
 577         next if !$vmid; # skip CT0
 578         my $d = $ids->{$vmid};
 579         next if !$d->{node} || $d->{node} ne $nodename;
 580         next if !$d->{type} || $d->{type} ne 'lxc';
 581         $res->{$vmid}->{type} = 'lxc';
 582     }
 583     return $res;
 584 }
 585
 586 sub cfs_config_path {
 587     my ($vmid, $node) = @_;
 588
 589     $node = $nodename if !$node;
 590     return "nodes/$node/lxc/$vmid.conf";
 591 }
 592
 593 sub config_file {
 594     my ($vmid, $node) = @_;
 595
 596     my $cfspath = cfs_config_path($vmid, $node);
 597     return "/etc/pve/$cfspath";
 598 }
 599
 600 sub load_config {
 601     my ($vmid, $node) = @_;
 602
 603     $node = $nodename if !$node;
 604     my $cfspath = cfs_config_path($vmid, $node);
 605
 606     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 607     die "container $vmid does not exist\n" if !defined($conf);
 608
 609     return $conf;
 610 }
 611
 612 sub create_config {
 613     my ($vmid, $conf) = @_;
 614
 615     my $dir = "/etc/pve/nodes/$nodename/lxc";
 616     mkdir $dir;
 617
 618     write_config($vmid, $conf);
 619 }
 620
 621 sub destroy_config {
 622     my ($vmid) = @_;
 623
 624     unlink config_file($vmid, $nodename);
 625 }
 626
 627 sub write_config {
 628     my ($vmid, $conf) = @_;
 629
 630     my $cfspath = cfs_config_path($vmid);
 631
 632     PVE::Cluster::cfs_write_file($cfspath, $conf);
 633 }
 634
 635 # flock: we use one file handle per process, so lock file
 636 # can be called multiple times and will succeed for the same process.
 637
 638 my $lock_handles =  {};
 639 my $lockdir = "/run/lock/lxc";
 640
 641 sub lock_filename {
 642     my ($vmid) = @_;
 643
 644     return "$lockdir/pve-config-${vmid}.lock";
 645 }
 646
 647 sub lock_container {
 648     my ($vmid, $timeout, $code, @param) = @_;
 649
 650     $timeout = 10 if !$timeout;
 651
 652     my $filename = lock_filename($vmid);
 653
 654     mkdir $lockdir if !-d $lockdir;
 655
 656     my $res = PVE::Tools::lock_file_full($filename, $timeout, 0, $code, @param);
 657
 658     die $@ if $@;
 659
 660     return $res;
 661 }
 662
 663 sub option_exists {
 664     my ($name) = @_;
 665
 666     return defined($confdesc->{$name});
 667 }
 668
 669 # add JSON properties for create and set function
 670 sub json_config_properties {
 671     my $prop = shift;
 672
 673     foreach my $opt (keys %$confdesc) {
 674         next if $opt eq 'parent' || $opt eq 'snaptime';
 675         next if $prop->{$opt};
 676         $prop->{$opt} = $confdesc->{$opt};
 677     }
 678
 679     return $prop;
 680 }
 681
 682 sub json_config_properties_no_rootfs {
 683     my $prop = shift;
 684
 685     foreach my $opt (keys %$confdesc) {
 686         next if $prop->{$opt};
 687         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 688         $prop->{$opt} = $confdesc->{$opt};
 689     }
 690
 691     return $prop;
 692 }
 693
 694 # container status helpers
 695
 696 sub list_active_containers {
 697
 698     my $filename = "/proc/net/unix";
 699
 700     # similar test is used by lcxcontainers.c: list_active_containers
 701     my $res = {};
 702
 703     my $fh = IO::File->new ($filename, "r");
 704     return $res if !$fh;
 705
 706     while (defined(my $line = <$fh>)) {
 707         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 708             my $path = $1;
 709             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 710                 $res->{$1} = 1;
 711             }
 712         }
 713     }
 714
 715     close($fh);
 716
 717     return $res;
 718 }
 719
 720 # warning: this is slow
 721 sub check_running {
 722     my ($vmid) = @_;
 723
 724     my $active_hash = list_active_containers();
 725
 726     return 1 if defined($active_hash->{$vmid});
 727
 728     return undef;
 729 }
 730
 731 sub get_container_disk_usage {
 732     my ($vmid, $pid) = @_;
 733
 734     return PVE::Tools::df("/proc/$pid/root/", 1);
 735 }
 736
 737 my $last_proc_vmid_stat;
 738
 739 my $parse_cpuacct_stat = sub {
 740     my ($vmid) = @_;
 741
 742     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 743
 744     my $stat = {};
 745
 746     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 747
 748         $stat->{utime} = $1;
 749         $stat->{stime} = $2;
 750
 751     }
 752
 753     return $stat;
 754 };
 755
 756 sub vmstatus {
 757     my ($opt_vmid) = @_;
 758
 759     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 760
 761     my $active_hash = list_active_containers();
 762
 763     my $cpucount = $cpuinfo->{cpus} || 1;
 764
 765     my $cdtime = gettimeofday;
 766
 767     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 768
 769     foreach my $vmid (keys %$list) {
 770         my $d = $list->{$vmid};
 771
 772         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 773         warn $@ if $@; # ignore errors (consider them stopped)
 774
 775         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 776
 777         my $cfspath = cfs_config_path($vmid);
 778         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 779
 780         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 781         $d->{name} =~ s/[\s]//g;
 782
 783         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 784
 785         if ($d->{pid}) {
 786             my $res = get_container_disk_usage($vmid, $d->{pid});
 787             $d->{disk} = $res->{used};
 788             $d->{maxdisk} = $res->{total};
 789         } else {
 790             $d->{disk} = 0;
 791             # use 4GB by default ??
 792             if (my $rootfs = $conf->{rootfs}) {
 793                 my $rootinfo = parse_ct_rootfs($rootfs);
 794                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 795             } else {
 796                 $d->{maxdisk} = 4*1024*1024*1024;
 797             }
 798         }
 799
 800         $d->{mem} = 0;
 801         $d->{swap} = 0;
 802         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 803         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 804
 805         $d->{uptime} = 0;
 806         $d->{cpu} = 0;
 807
 808         $d->{netout} = 0;
 809         $d->{netin} = 0;
 810
 811         $d->{diskread} = 0;
 812         $d->{diskwrite} = 0;
 813
 814         $d->{template} = is_template($conf);
 815     }
 816
 817     foreach my $vmid (keys %$list) {
 818         my $d = $list->{$vmid};
 819         my $pid = $d->{pid};
 820
 821         next if !$pid; # skip stopped CTs
 822
 823         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 824         $d->{uptime} = time - $ctime; # the method lxcfs uses
 825
 826         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 827         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 828
 829         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 830         my @bytes = split(/\n/, $blkio_bytes);
 831         foreach my $byte (@bytes) {
 832             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 833                 $d->{diskread} = $2 if $key eq 'Read';
 834                 $d->{diskwrite} = $2 if $key eq 'Write';
 835             }
 836         }
 837
 838         my $pstat = &$parse_cpuacct_stat($vmid);
 839
 840         my $used = $pstat->{utime} + $pstat->{stime};
 841
 842         my $old = $last_proc_vmid_stat->{$vmid};
 843         if (!$old) {
 844             $last_proc_vmid_stat->{$vmid} = {
 845                 time => $cdtime,
 846                 used => $used,
 847                 cpu => 0,
 848             };
 849             next;
 850         }
 851
 852         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 853
 854         if ($dtime > 1000) {
 855             my $dutime = $used -  $old->{used};
 856
 857             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 858             $last_proc_vmid_stat->{$vmid} = {
 859                 time => $cdtime,
 860                 used => $used,
 861                 cpu => $d->{cpu},
 862             };
 863         } else {
 864             $d->{cpu} = $old->{cpu};
 865         }
 866     }
 867
 868     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 869
 870     foreach my $dev (keys %$netdev) {
 871         next if $dev !~ m/^veth([1-9]\d*)i/;
 872         my $vmid = $1;
 873         my $d = $list->{$vmid};
 874
 875         next if !$d;
 876
 877         $d->{netout} += $netdev->{$dev}->{receive};
 878         $d->{netin} += $netdev->{$dev}->{transmit};
 879
 880     }
 881
 882     return $list;
 883 }
 884
 885 sub classify_mountpoint {
 886     my ($vol) = @_;
 887     if ($vol =~ m!^/!) {
 888         return 'device' if $vol =~ m!^/dev/!;
 889         return 'bind';
 890     }
 891     return 'volume';
 892 }
 893
 894 my $parse_ct_mountpoint_full = sub {
 895     my ($desc, $data, $noerr) = @_;
 896
 897     $data //= '';
 898
 899     my $res;
 900     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 901     if ($@) {
 902         return undef if $noerr;
 903         die $@;
 904     }
 905
 906     if (defined(my $size = $res->{size})) {
 907         $size = PVE::JSONSchema::parse_size($size);
 908         if (!defined($size)) {
 909             return undef if $noerr;
 910             die "invalid size: $size\n";
 911         }
 912         $res->{size} = $size;
 913     }
 914
 915     $res->{type} = classify_mountpoint($res->{volume});
 916
 917     return $res;
 918 };
 919
 920 sub parse_ct_rootfs {
 921     my ($data, $noerr) = @_;
 922
 923     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 924
 925     $res->{mp} = '/' if defined($res);
 926
 927     return $res;
 928 }
 929
 930 sub parse_ct_mountpoint {
 931     my ($data, $noerr) = @_;
 932
 933     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 934 }
 935
 936 sub print_ct_mountpoint {
 937     my ($info, $nomp) = @_;
 938     my $skip = [ 'type' ];
 939     push @$skip, 'mp' if $nomp;
 940     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 941 }
 942
 943 sub print_lxc_network {
 944     my $net = shift;
 945     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 946 }
 947
 948 sub parse_lxc_network {
 949     my ($data) = @_;
 950
 951     my $res = {};
 952
 953     return $res if !$data;
 954
 955     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 956
 957     $res->{type} = 'veth';
 958     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 959
 960     return $res;
 961 }
 962
 963 sub read_cgroup_value {
 964     my ($group, $vmid, $name, $full) = @_;
 965
 966     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 967
 968     return PVE::Tools::file_get_contents($path) if $full;
 969
 970     return PVE::Tools::file_read_firstline($path);
 971 }
 972
 973 sub write_cgroup_value {
 974    my ($group, $vmid, $name, $value) = @_;
 975
 976    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 977    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 978
 979 }
 980
 981 sub find_lxc_console_pids {
 982
 983     my $res = {};
 984
 985     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 986         my ($pid) = @_;
 987
 988         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 989         return if !$cmdline;
 990
 991         my @args = split(/\0/, $cmdline);
 992
 993         # search for lxc-console -n <vmid>
 994         return if scalar(@args) != 3;
 995         return if $args[1] ne '-n';
 996         return if $args[2] !~ m/^\d+$/;
 997         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 998
 999         my $vmid = $args[2];
1000
1001         push @{$res->{$vmid}}, $pid;
1002     });
1003
1004     return $res;
1005 }
1006
1007 sub find_lxc_pid {
1008     my ($vmid) = @_;
1009
1010     my $pid = undef;
1011     my $parser = sub {
1012         my $line = shift;
1013         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1014     };
1015     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1016
1017     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1018
1019     return $pid;
1020 }
1021
1022 # Note: we cannot use Net:IP, because that only allows strict
1023 # CIDR networks
1024 sub parse_ipv4_cidr {
1025     my ($cidr, $noerr) = @_;
1026
1027     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1028         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1029     }
1030
1031     return undef if $noerr;
1032
1033     die "unable to parse ipv4 address/mask\n";
1034 }
1035
1036 sub check_lock {
1037     my ($conf) = @_;
1038
1039     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1040 }
1041
1042 sub check_protection {
1043     my ($vm_conf, $err_msg) = @_;
1044
1045     if ($vm_conf->{protection}) {
1046         die "$err_msg - protection mode enabled\n";
1047     }
1048 }
1049
1050 sub update_lxc_config {
1051     my ($storage_cfg, $vmid, $conf) = @_;
1052
1053     my $dir = "/var/lib/lxc/$vmid";
1054
1055     if ($conf->{template}) {
1056
1057         unlink "$dir/config";
1058
1059         return;
1060     }
1061
1062     my $raw = '';
1063
1064     die "missing 'arch' - internal error" if !$conf->{arch};
1065     $raw .= "lxc.arch = $conf->{arch}\n";
1066
1067     my $unprivileged = $conf->{unprivileged};
1068     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1069
1070     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1071     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux)$/x) {
1072         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1073         if ($unprivileged || $custom_idmap) {
1074             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1075         }
1076     } else {
1077         die "implement me (ostype $ostype)";
1078     }
1079
1080     $raw .= "lxc.monitor.unshare = 1\n";
1081
1082     # Should we read them from /etc/subuid?
1083     if ($unprivileged && !$custom_idmap) {
1084         $raw .= "lxc.id_map = u 0 100000 65536\n";
1085         $raw .= "lxc.id_map = g 0 100000 65536\n";
1086     }
1087
1088     if (!has_dev_console($conf)) {
1089         $raw .= "lxc.console = none\n";
1090         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1091     }
1092
1093     my $ttycount = get_tty_count($conf);
1094     $raw .= "lxc.tty = $ttycount\n";
1095
1096     # some init scripts expect a linux terminal (turnkey).
1097     $raw .= "lxc.environment = TERM=linux\n";
1098
1099     my $utsname = $conf->{hostname} || "CT$vmid";
1100     $raw .= "lxc.utsname = $utsname\n";
1101
1102     my $memory = $conf->{memory} || 512;
1103     my $swap = $conf->{swap} // 0;
1104
1105     my $lxcmem = int($memory*1024*1024);
1106     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1107
1108     my $lxcswap = int(($memory + $swap)*1024*1024);
1109     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1110
1111     if (my $cpulimit = $conf->{cpulimit}) {
1112         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1113         my $value = int(100000*$cpulimit);
1114         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1115     }
1116
1117     my $shares = $conf->{cpuunits} || 1024;
1118     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1119
1120     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1121
1122     $raw .= "lxc.rootfs = $dir/rootfs\n";
1123
1124     my $netcount = 0;
1125     foreach my $k (keys %$conf) {
1126         next if $k !~ m/^net(\d+)$/;
1127         my $ind = $1;
1128         my $d = parse_lxc_network($conf->{$k});
1129         $netcount++;
1130         $raw .= "lxc.network.type = veth\n";
1131         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1132         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1133         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1134         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1135     }
1136
1137     if (my $lxcconf = $conf->{lxc}) {
1138         foreach my $entry (@$lxcconf) {
1139             my ($k, $v) = @$entry;
1140             $netcount++ if $k eq 'lxc.network.type';
1141             $raw .= "$k = $v\n";
1142         }
1143     }
1144
1145     $raw .= "lxc.network.type = empty\n" if !$netcount;
1146
1147     File::Path::mkpath("$dir/rootfs");
1148
1149     PVE::Tools::file_set_contents("$dir/config", $raw);
1150 }
1151
1152 # verify and cleanup nameserver list (replace \0 with ' ')
1153 sub verify_nameserver_list {
1154     my ($nameserver_list) = @_;
1155
1156     my @list = ();
1157     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1158         PVE::JSONSchema::pve_verify_ip($server);
1159         push @list, $server;
1160     }
1161
1162     return join(' ', @list);
1163 }
1164
1165 sub verify_searchdomain_list {
1166     my ($searchdomain_list) = @_;
1167
1168     my @list = ();
1169     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1170         # todo: should we add checks for valid dns domains?
1171         push @list, $server;
1172     }
1173
1174     return join(' ', @list);
1175 }
1176
1177 sub add_unused_volume {
1178     my ($config, $volid) = @_;
1179
1180     my $key;
1181     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1182         my $test = "unused$ind";
1183         if (my $vid = $config->{$test}) {
1184             return if $vid eq $volid; # do not add duplicates
1185         } else {
1186             $key = $test;
1187         }
1188     }
1189
1190     die "Too many unused volumes - please delete them first.\n" if !$key;
1191
1192     $config->{$key} = $volid;
1193
1194     return $key;
1195 }
1196
1197 sub update_pct_config {
1198     my ($vmid, $conf, $running, $param, $delete) = @_;
1199
1200     my @nohotplug;
1201
1202     my $new_disks = 0;
1203     my @deleted_volumes;
1204
1205     my $rootdir;
1206     if ($running) {
1207         my $pid = find_lxc_pid($vmid);
1208         $rootdir = "/proc/$pid/root";
1209     }
1210
1211     my $hotplug_error = sub {
1212         if ($running) {
1213             push @nohotplug, @_;
1214             return 1;
1215         } else {
1216             return 0;
1217         }
1218     };
1219
1220     if (defined($delete)) {
1221         foreach my $opt (@$delete) {
1222             if (!exists($conf->{$opt})) {
1223                 warn "no such option: $opt\n";
1224                 next;
1225             }
1226
1227             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1228                 die "unable to delete required option '$opt'\n";
1229             } elsif ($opt eq 'swap') {
1230                 delete $conf->{$opt};
1231                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1232             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1233                 delete $conf->{$opt};
1234             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1235                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1236                 next if $hotplug_error->($opt);
1237                 delete $conf->{$opt};
1238             } elsif ($opt =~ m/^net(\d)$/) {
1239                 delete $conf->{$opt};
1240                 next if !$running;
1241                 my $netid = $1;
1242                 PVE::Network::veth_delete("veth${vmid}i$netid");
1243             } elsif ($opt eq 'protection') {
1244                 delete $conf->{$opt};
1245             } elsif ($opt =~ m/^unused(\d+)$/) {
1246                 next if $hotplug_error->($opt);
1247                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1248                 push @deleted_volumes, $conf->{$opt};
1249                 delete $conf->{$opt};
1250             } elsif ($opt =~ m/^mp(\d+)$/) {
1251                 next if $hotplug_error->($opt);
1252                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1253                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1254                 if ($mountpoint->{type} eq 'volume') {
1255                     add_unused_volume($conf, $mountpoint->{volume})
1256                 }
1257                 delete $conf->{$opt};
1258             } elsif ($opt eq 'unprivileged') {
1259                 die "unable to delete read-only option: '$opt'\n";
1260             } else {
1261                 die "implement me (delete: $opt)"
1262             }
1263             write_config($vmid, $conf) if $running;
1264         }
1265     }
1266
1267     # There's no separate swap size to configure, there's memory and "total"
1268     # memory (iow. memory+swap). This means we have to change them together.
1269     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1270     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1271     if (defined($wanted_memory) || defined($wanted_swap)) {
1272
1273         my $old_memory = ($conf->{memory} || 512);
1274         my $old_swap = ($conf->{swap} || 0);
1275
1276         $wanted_memory //= $old_memory;
1277         $wanted_swap //= $old_swap;
1278
1279         my $total = $wanted_memory + $wanted_swap;
1280         if ($running) {
1281             my $old_total = $old_memory + $old_swap;
1282             if ($total > $old_total) {
1283                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1284                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1285             } else {
1286                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1287                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1288             }
1289         }
1290         $conf->{memory} = $wanted_memory;
1291         $conf->{swap} = $wanted_swap;
1292
1293         write_config($vmid, $conf) if $running;
1294     }
1295
1296     foreach my $opt (keys %$param) {
1297         my $value = $param->{$opt};
1298         if ($opt eq 'hostname') {
1299             $conf->{$opt} = $value;
1300         } elsif ($opt eq 'onboot') {
1301             $conf->{$opt} = $value ? 1 : 0;
1302         } elsif ($opt eq 'startup') {
1303             $conf->{$opt} = $value;
1304         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1305             next if $hotplug_error->($opt);
1306             $conf->{$opt} = $value;
1307         } elsif ($opt eq 'nameserver') {
1308             next if $hotplug_error->($opt);
1309             my $list = verify_nameserver_list($value);
1310             $conf->{$opt} = $list;
1311         } elsif ($opt eq 'searchdomain') {
1312             next if $hotplug_error->($opt);
1313             my $list = verify_searchdomain_list($value);
1314             $conf->{$opt} = $list;
1315         } elsif ($opt eq 'cpulimit') {
1316             next if $hotplug_error->($opt); # FIXME: hotplug
1317             $conf->{$opt} = $value;
1318         } elsif ($opt eq 'cpuunits') {
1319             $conf->{$opt} = $value;
1320             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1321         } elsif ($opt eq 'description') {
1322             $conf->{$opt} = PVE::Tools::encode_text($value);
1323         } elsif ($opt =~ m/^net(\d+)$/) {
1324             my $netid = $1;
1325             my $net = parse_lxc_network($value);
1326             if (!$running) {
1327                 $conf->{$opt} = print_lxc_network($net);
1328             } else {
1329                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1330             }
1331         } elsif ($opt eq 'protection') {
1332             $conf->{$opt} = $value ? 1 : 0;
1333         } elsif ($opt =~ m/^mp(\d+)$/) {
1334             next if $hotplug_error->($opt);
1335             check_protection($conf, "can't update CT $vmid drive '$opt'");
1336             $conf->{$opt} = $value;
1337             $new_disks = 1;
1338         } elsif ($opt eq 'rootfs') {
1339             check_protection($conf, "can't update CT $vmid drive '$opt'");
1340             die "implement me: $opt";
1341         } elsif ($opt eq 'unprivileged') {
1342             die "unable to modify read-only option: '$opt'\n";
1343         } else {
1344             die "implement me: $opt";
1345         }
1346         write_config($vmid, $conf) if $running;
1347     }
1348
1349     if (@deleted_volumes) {
1350         my $storage_cfg = PVE::Storage::config();
1351         foreach my $volume (@deleted_volumes) {
1352             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1353         }
1354     }
1355
1356     if ($new_disks) {
1357         my $storage_cfg = PVE::Storage::config();
1358         create_disks($storage_cfg, $vmid, $conf, $conf);
1359     }
1360
1361     # This should be the last thing we do here
1362     if ($running && scalar(@nohotplug)) {
1363         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1364     }
1365 }
1366
1367 sub has_dev_console {
1368     my ($conf) = @_;
1369
1370     return !(defined($conf->{console}) && !$conf->{console});
1371 }
1372
1373 sub get_tty_count {
1374     my ($conf) = @_;
1375
1376     return $conf->{tty} // $confdesc->{tty}->{default};
1377 }
1378
1379 sub get_cmode {
1380     my ($conf) = @_;
1381
1382     return $conf->{cmode} // $confdesc->{cmode}->{default};
1383 }
1384
1385 sub get_console_command {
1386     my ($vmid, $conf) = @_;
1387
1388     my $cmode = get_cmode($conf);
1389
1390     if ($cmode eq 'console') {
1391         return ['lxc-console', '-n',  $vmid, '-t', 0];
1392     } elsif ($cmode eq 'tty') {
1393         return ['lxc-console', '-n',  $vmid];
1394     } elsif ($cmode eq 'shell') {
1395         return ['lxc-attach', '--clear-env', '-n', $vmid];
1396     } else {
1397         die "internal error";
1398     }
1399 }
1400
1401 sub get_primary_ips {
1402     my ($conf) = @_;
1403
1404     # return data from net0
1405
1406     return undef if !defined($conf->{net0});
1407     my $net = parse_lxc_network($conf->{net0});
1408
1409     my $ipv4 = $net->{ip};
1410     if ($ipv4) {
1411         if ($ipv4 =~ /^(dhcp|manual)$/) {
1412             $ipv4 = undef
1413         } else {
1414             $ipv4 =~ s!/\d+$!!;
1415         }
1416     }
1417     my $ipv6 = $net->{ip6};
1418     if ($ipv6) {
1419         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1420             $ipv6 = undef;
1421         } else {
1422             $ipv6 =~ s!/\d+$!!;
1423         }
1424     }
1425
1426     return ($ipv4, $ipv6);
1427 }
1428
1429 sub delete_mountpoint_volume {
1430     my ($storage_cfg, $vmid, $volume) = @_;
1431
1432     return if classify_mountpoint($volume) ne 'volume';
1433
1434     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1435     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1436 }
1437
1438 sub destroy_lxc_container {
1439     my ($storage_cfg, $vmid, $conf) = @_;
1440
1441     foreach_mountpoint($conf, sub {
1442         my ($ms, $mountpoint) = @_;
1443         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1444     });
1445
1446     rmdir "/var/lib/lxc/$vmid/rootfs";
1447     unlink "/var/lib/lxc/$vmid/config";
1448     rmdir "/var/lib/lxc/$vmid";
1449     destroy_config($vmid);
1450
1451     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1452     #PVE::Tools::run_command($cmd);
1453 }
1454
1455 sub vm_stop_cleanup {
1456     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1457
1458     eval {
1459         if (!$keepActive) {
1460
1461             my $vollist = get_vm_volumes($conf);
1462             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1463         }
1464     };
1465     warn $@ if $@; # avoid errors - just warn
1466 }
1467
1468 my $safe_num_ne = sub {
1469     my ($a, $b) = @_;
1470
1471     return 0 if !defined($a) && !defined($b);
1472     return 1 if !defined($a);
1473     return 1 if !defined($b);
1474
1475     return $a != $b;
1476 };
1477
1478 my $safe_string_ne = sub {
1479     my ($a, $b) = @_;
1480
1481     return 0 if !defined($a) && !defined($b);
1482     return 1 if !defined($a);
1483     return 1 if !defined($b);
1484
1485     return $a ne $b;
1486 };
1487
1488 sub update_net {
1489     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1490
1491     if ($newnet->{type} ne 'veth') {
1492         # for when there are physical interfaces
1493         die "cannot update interface of type $newnet->{type}";
1494     }
1495
1496     my $veth = "veth${vmid}i${netid}";
1497     my $eth = $newnet->{name};
1498
1499     if (my $oldnetcfg = $conf->{$opt}) {
1500         my $oldnet = parse_lxc_network($oldnetcfg);
1501
1502         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1503             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1504
1505             PVE::Network::veth_delete($veth);
1506             delete $conf->{$opt};
1507             write_config($vmid, $conf);
1508
1509             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1510
1511         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1512                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1513                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1514
1515                 if ($oldnet->{bridge}) {
1516                     PVE::Network::tap_unplug($veth);
1517                     foreach (qw(bridge tag firewall)) {
1518                         delete $oldnet->{$_};
1519                     }
1520                     $conf->{$opt} = print_lxc_network($oldnet);
1521                     write_config($vmid, $conf);
1522                 }
1523
1524                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1525                 foreach (qw(bridge tag firewall)) {
1526                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1527                 }
1528                 $conf->{$opt} = print_lxc_network($oldnet);
1529                 write_config($vmid, $conf);
1530         }
1531     } else {
1532         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1533     }
1534
1535     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1536 }
1537
1538 sub hotplug_net {
1539     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1540
1541     my $veth = "veth${vmid}i${netid}";
1542     my $vethpeer = $veth . "p";
1543     my $eth = $newnet->{name};
1544
1545     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1546     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1547
1548     # attach peer in container
1549     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1550     PVE::Tools::run_command($cmd);
1551
1552     # link up peer in container
1553     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1554     PVE::Tools::run_command($cmd);
1555
1556     my $done = { type => 'veth' };
1557     foreach (qw(bridge tag firewall hwaddr name)) {
1558         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1559     }
1560     $conf->{$opt} = print_lxc_network($done);
1561
1562     write_config($vmid, $conf);
1563 }
1564
1565 sub update_ipconfig {
1566     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1567
1568     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1569
1570     my $optdata = parse_lxc_network($conf->{$opt});
1571     my $deleted = [];
1572     my $added = [];
1573     my $nscmd = sub {
1574         my $cmdargs = shift;
1575         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1576     };
1577     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1578
1579     my $change_ip_config = sub {
1580         my ($ipversion) = @_;
1581
1582         my $family_opt = "-$ipversion";
1583         my $suffix = $ipversion == 4 ? '' : $ipversion;
1584         my $gw= "gw$suffix";
1585         my $ip= "ip$suffix";
1586
1587         my $newip = $newnet->{$ip};
1588         my $newgw = $newnet->{$gw};
1589         my $oldip = $optdata->{$ip};
1590
1591         my $change_ip = &$safe_string_ne($oldip, $newip);
1592         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1593
1594         return if !$change_ip && !$change_gw;
1595
1596         # step 1: add new IP, if this fails we cancel
1597         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1598         if ($change_ip && $is_real_ip) {
1599             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1600             if (my $err = $@) {
1601                 warn $err;
1602                 return;
1603             }
1604         }
1605
1606         # step 2: replace gateway
1607         #   If this fails we delete the added IP and cancel.
1608         #   If it succeeds we save the config and delete the old IP, ignoring
1609         #   errors. The config is then saved.
1610         # Note: 'ip route replace' can add
1611         if ($change_gw) {
1612             if ($newgw) {
1613                 eval {
1614                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1615                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1616                     }
1617                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1618                 };
1619                 if (my $err = $@) {
1620                     warn $err;
1621                     # the route was not replaced, the old IP is still available
1622                     # rollback (delete new IP) and cancel
1623                     if ($change_ip) {
1624                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1625                         warn $@ if $@; # no need to die here
1626                     }
1627                     return;
1628                 }
1629             } else {
1630                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1631                 # if the route was not deleted, the guest might have deleted it manually
1632                 # warn and continue
1633                 warn $@ if $@;
1634             }
1635         }
1636
1637         # from this point on we save the configuration
1638         # step 3: delete old IP ignoring errors
1639         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1640             # We need to enable promote_secondaries, otherwise our newly added
1641             # address will be removed along with the old one.
1642             my $promote = 0;
1643             eval {
1644                 if ($ipversion == 4) {
1645                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1646                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1647                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1648                 }
1649                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1650             };
1651             warn $@ if $@; # no need to die here
1652
1653             if ($ipversion == 4) {
1654                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1655             }
1656         }
1657
1658         foreach my $property ($ip, $gw) {
1659             if ($newnet->{$property}) {
1660                 $optdata->{$property} = $newnet->{$property};
1661             } else {
1662                 delete $optdata->{$property};
1663             }
1664         }
1665         $conf->{$opt} = print_lxc_network($optdata);
1666         write_config($vmid, $conf);
1667         $lxc_setup->setup_network($conf);
1668     };
1669
1670     &$change_ip_config(4);
1671     &$change_ip_config(6);
1672
1673 }
1674
1675 # Internal snapshots
1676
1677 # NOTE: Snapshot create/delete involves several non-atomic
1678 # actions, and can take a long time.
1679 # So we try to avoid locking the file and use the 'lock' variable
1680 # inside the config file instead.
1681
1682 my $snapshot_copy_config = sub {
1683     my ($source, $dest) = @_;
1684
1685     foreach my $k (keys %$source) {
1686         next if $k eq 'snapshots';
1687         next if $k eq 'snapstate';
1688         next if $k eq 'snaptime';
1689         next if $k eq 'vmstate';
1690         next if $k eq 'lock';
1691         next if $k eq 'digest';
1692         next if $k eq 'description';
1693
1694         $dest->{$k} = $source->{$k};
1695     }
1696 };
1697
1698 my $snapshot_prepare = sub {
1699     my ($vmid, $snapname, $comment) = @_;
1700
1701     my $snap;
1702
1703     my $updatefn =  sub {
1704
1705         my $conf = load_config($vmid);
1706
1707         die "you can't take a snapshot if it's a template\n"
1708             if is_template($conf);
1709
1710         check_lock($conf);
1711
1712         $conf->{lock} = 'snapshot';
1713
1714         die "snapshot name '$snapname' already used\n"
1715             if defined($conf->{snapshots}->{$snapname});
1716
1717         my $storecfg = PVE::Storage::config();
1718         my $feature = $snapname eq 'vzdump' ? 'vzdump' : 'snapshot';
1719         die "snapshot feature is not available\n" if !has_feature($feature, $conf, $storecfg);
1720
1721         $snap = $conf->{snapshots}->{$snapname} = {};
1722
1723         &$snapshot_copy_config($conf, $snap);
1724
1725         $snap->{'snapstate'} = "prepare";
1726         $snap->{'snaptime'} = time();
1727         $snap->{'description'} = $comment if $comment;
1728         $conf->{snapshots}->{$snapname} = $snap;
1729
1730         write_config($vmid, $conf);
1731     };
1732
1733     lock_container($vmid, 10, $updatefn);
1734
1735     return $snap;
1736 };
1737
1738 my $snapshot_commit = sub {
1739     my ($vmid, $snapname) = @_;
1740
1741     my $updatefn = sub {
1742
1743         my $conf = load_config($vmid);
1744
1745         die "missing snapshot lock\n"
1746             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1747
1748         die "snapshot '$snapname' does not exist\n"
1749             if !defined($conf->{snapshots}->{$snapname});
1750
1751         die "wrong snapshot state\n"
1752             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1753                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1754
1755         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1756         delete $conf->{lock};
1757         $conf->{parent} = $snapname;
1758
1759         write_config($vmid, $conf);
1760     };
1761
1762     lock_container($vmid, 10 ,$updatefn);
1763 };
1764
1765 sub has_feature {
1766     my ($feature, $conf, $storecfg, $snapname) = @_;
1767
1768     my $err;
1769     my $vzdump = $feature eq 'vzdump';
1770     $feature = 'snapshot' if $vzdump;
1771
1772     foreach_mountpoint($conf, sub {
1773         my ($ms, $mountpoint) = @_;
1774
1775         return if $err; # skip further test
1776         return if $vzdump && $ms ne 'rootfs' && !$mountpoint->{backup};
1777
1778         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1779
1780         # TODO: implement support for mountpoints
1781         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1782             if $ms ne 'rootfs';
1783     });
1784
1785     return $err ? 0 : 1;
1786 }
1787
1788 sub snapshot_create {
1789     my ($vmid, $snapname, $comment) = @_;
1790
1791     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1792
1793     my $conf = load_config($vmid);
1794
1795     my $running = check_running($vmid);
1796
1797     my $unfreeze = 0;
1798
1799     eval {
1800         if ($running) {
1801             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1802             $unfreeze = 1;
1803             PVE::Tools::run_command(['/bin/sync']);
1804         };
1805
1806         my $storecfg = PVE::Storage::config();
1807         my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1808         my $volid = $rootinfo->{volume};
1809
1810         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1811         &$snapshot_commit($vmid, $snapname);
1812     };
1813     my $err = $@;
1814
1815     if ($unfreeze) {
1816         eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
1817         warn $@ if $@;
1818     }
1819
1820     if ($err) {
1821         snapshot_delete($vmid, $snapname, 1);
1822         die "$err\n";
1823     }
1824 }
1825
1826 sub snapshot_delete {
1827     my ($vmid, $snapname, $force) = @_;
1828
1829     my $snap;
1830
1831     my $conf;
1832
1833     my $updatefn =  sub {
1834
1835         $conf = load_config($vmid);
1836
1837         die "you can't delete a snapshot if vm is a template\n"
1838             if is_template($conf);
1839
1840         $snap = $conf->{snapshots}->{$snapname};
1841
1842         check_lock($conf);
1843
1844         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1845
1846         $snap->{snapstate} = 'delete';
1847
1848         write_config($vmid, $conf);
1849     };
1850
1851     lock_container($vmid, 10, $updatefn);
1852
1853     my $storecfg = PVE::Storage::config();
1854
1855     my $unlink_parent = sub {
1856
1857         my ($confref, $new_parent) = @_;
1858
1859         if ($confref->{parent} && $confref->{parent} eq $snapname) {
1860             if ($new_parent) {
1861                 $confref->{parent} = $new_parent;
1862             } else {
1863                 delete $confref->{parent};
1864             }
1865         }
1866     };
1867
1868     my $del_snap =  sub {
1869
1870         check_lock($conf);
1871
1872         my $parent = $conf->{snapshots}->{$snapname}->{parent};
1873         foreach my $snapkey (keys %{$conf->{snapshots}}) {
1874             &$unlink_parent($conf->{snapshots}->{$snapkey}, $parent);
1875         }
1876
1877         &$unlink_parent($conf, $parent);
1878
1879         delete $conf->{snapshots}->{$snapname};
1880
1881         write_config($vmid, $conf);
1882     };
1883
1884     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1885     my $rootinfo = parse_ct_rootfs($rootfs);
1886     my $volid = $rootinfo->{volume};
1887
1888     eval {
1889         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1890     };
1891     my $err = $@;
1892
1893     if(!$err || ($err && $force)) {
1894         lock_container($vmid, 10, $del_snap);
1895         if ($err) {
1896             die "Can't delete snapshot: $vmid $snapname $err\n";
1897         }
1898     }
1899 }
1900
1901 sub snapshot_rollback {
1902     my ($vmid, $snapname) = @_;
1903
1904     my $storecfg = PVE::Storage::config();
1905
1906     my $conf = load_config($vmid);
1907
1908     die "you can't rollback if vm is a template\n" if is_template($conf);
1909
1910     my $snap = $conf->{snapshots}->{$snapname};
1911
1912     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1913
1914     my $rootfs = $snap->{rootfs};
1915     my $rootinfo = parse_ct_rootfs($rootfs);
1916     my $volid = $rootinfo->{volume};
1917
1918     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1919
1920     my $updatefn = sub {
1921
1922         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1923             if $snap->{snapstate};
1924
1925         check_lock($conf);
1926
1927         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1928
1929         die "unable to rollback vm $vmid: vm is running\n"
1930             if check_running($vmid);
1931
1932         $conf->{lock} = 'rollback';
1933
1934         my $forcemachine;
1935
1936         # copy snapshot config to current config
1937
1938         my $tmp_conf = $conf;
1939         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1940         $conf->{snapshots} = $tmp_conf->{snapshots};
1941         delete $conf->{snaptime};
1942         delete $conf->{snapname};
1943         $conf->{parent} = $snapname;
1944
1945         write_config($vmid, $conf);
1946     };
1947
1948     my $unlockfn = sub {
1949         delete $conf->{lock};
1950         write_config($vmid, $conf);
1951     };
1952
1953     lock_container($vmid, 10, $updatefn);
1954
1955     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1956
1957     lock_container($vmid, 5, $unlockfn);
1958 }
1959
1960 sub template_create {
1961     my ($vmid, $conf) = @_;
1962
1963     my $storecfg = PVE::Storage::config();
1964
1965     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1966     my $volid = $rootinfo->{volume};
1967
1968     die "Template feature is not available for '$volid'\n"
1969         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1970
1971     PVE::Storage::activate_volumes($storecfg, [$volid]);
1972
1973     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1974     $rootinfo->{volume} = $template_volid;
1975     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1976
1977     write_config($vmid, $conf);
1978 }
1979
1980 sub is_template {
1981     my ($conf) = @_;
1982
1983     return 1 if defined $conf->{template} && $conf->{template} == 1;
1984 }
1985
1986 sub mountpoint_names {
1987     my ($reverse) = @_;
1988
1989     my @names = ('rootfs');
1990
1991     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1992         push @names, "mp$i";
1993     }
1994
1995     return $reverse ? reverse @names : @names;
1996 }
1997
1998
1999 sub foreach_mountpoint_full {
2000     my ($conf, $reverse, $func) = @_;
2001
2002     foreach my $key (mountpoint_names($reverse)) {
2003         my $value = $conf->{$key};
2004         next if !defined($value);
2005         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2006         next if !defined($mountpoint);
2007
2008         &$func($key, $mountpoint);
2009     }
2010 }
2011
2012 sub foreach_mountpoint {
2013     my ($conf, $func) = @_;
2014
2015     foreach_mountpoint_full($conf, 0, $func);
2016 }
2017
2018 sub foreach_mountpoint_reverse {
2019     my ($conf, $func) = @_;
2020
2021     foreach_mountpoint_full($conf, 1, $func);
2022 }
2023
2024 sub check_ct_modify_config_perm {
2025     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2026
2027     return 1 if $authuser ne 'root@pam';
2028
2029     foreach my $opt (@$key_list) {
2030
2031         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2032             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2033         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2034             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2035         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2036             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2037         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2038                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2039             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2040         } else {
2041             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2042         }
2043     }
2044
2045     return 1;
2046 }
2047
2048 sub umount_all {
2049     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2050
2051     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2052     my $volid_list = get_vm_volumes($conf);
2053
2054     foreach_mountpoint_reverse($conf, sub {
2055         my ($ms, $mountpoint) = @_;
2056
2057         my $volid = $mountpoint->{volume};
2058         my $mount = $mountpoint->{mp};
2059
2060         return if !$volid || !$mount;
2061
2062         my $mount_path = "$rootdir/$mount";
2063         $mount_path =~ s!/+!/!g;
2064
2065         return if !PVE::ProcFSTools::is_mounted($mount_path);
2066
2067         eval {
2068             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2069         };
2070         if (my $err = $@) {
2071             if ($noerr) {
2072                 warn $err;
2073             } else {
2074                 die $err;
2075             }
2076         }
2077     });
2078 }
2079
2080 sub mount_all {
2081     my ($vmid, $storage_cfg, $conf) = @_;
2082
2083     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2084     File::Path::make_path($rootdir);
2085
2086     my $volid_list = get_vm_volumes($conf);
2087     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2088
2089     eval {
2090         foreach_mountpoint($conf, sub {
2091             my ($ms, $mountpoint) = @_;
2092
2093             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2094         });
2095     };
2096     if (my $err = $@) {
2097         warn "mounting container failed\n";
2098         umount_all($vmid, $storage_cfg, $conf, 1);
2099         die $err;
2100     }
2101
2102     return $rootdir;
2103 }
2104
2105
2106 sub mountpoint_mount_path {
2107     my ($mountpoint, $storage_cfg, $snapname) = @_;
2108
2109     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2110 }
2111
2112 my $check_mount_path = sub {
2113     my ($path) = @_;
2114     $path = File::Spec->canonpath($path);
2115     my $real = Cwd::realpath($path);
2116     if ($real ne $path) {
2117         die "mount path modified by symlink: $path != $real";
2118     }
2119 };
2120
2121 sub query_loopdev {
2122     my ($path) = @_;
2123     my $found;
2124     my $parser = sub {
2125         my $line = shift;
2126         if ($line =~ m@^(/dev/loop\d+):@) {
2127             $found = $1;
2128         }
2129     };
2130     my $cmd = ['losetup', '--associated', $path];
2131     PVE::Tools::run_command($cmd, outfunc => $parser);
2132     return $found;
2133 }
2134
2135 # use $rootdir = undef to just return the corresponding mount path
2136 sub mountpoint_mount {
2137     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2138
2139     my $volid = $mountpoint->{volume};
2140     my $mount = $mountpoint->{mp};
2141     my $type = $mountpoint->{type};
2142
2143     return if !$volid || !$mount;
2144
2145     my $mount_path;
2146
2147     if (defined($rootdir)) {
2148         $rootdir =~ s!/+$!!;
2149         $mount_path = "$rootdir/$mount";
2150         $mount_path =~ s!/+!/!g;
2151         &$check_mount_path($mount_path);
2152         File::Path::mkpath($mount_path);
2153     }
2154
2155     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2156
2157     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2158
2159     my $optstring = '';
2160     if (defined($mountpoint->{acl})) {
2161         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
2162     }
2163     if ($mountpoint->{ro}) {
2164         $optstring .= ',' if $optstring;
2165         $optstring .= 'ro';
2166     }
2167
2168     my @extra_opts = ('-o', $optstring);
2169
2170     if ($storage) {
2171
2172         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2173         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2174
2175         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2176             PVE::Storage::parse_volname($storage_cfg, $volid);
2177
2178         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2179
2180         if ($format eq 'subvol') {
2181             if ($mount_path) {
2182                 if ($snapname) {
2183                     if ($scfg->{type} eq 'zfspool') {
2184                         my $path_arg = $path;
2185                         $path_arg =~ s!^/+!!;
2186                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
2187                     } else {
2188                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2189                     }
2190                 } else {
2191                     if ($mountpoint->{ro}) {
2192                         die "read-only bind mounts not supported\n";
2193                     }
2194                     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $path, $mount_path]);
2195                 }
2196             }
2197             return wantarray ? ($path, 0) : $path;
2198         } elsif ($format eq 'raw' || $format eq 'iso') {
2199             my $use_loopdev = 0;
2200             if ($scfg->{path}) {
2201                 push @extra_opts, '-o', 'loop';
2202                 $use_loopdev = 1;
2203             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2204                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2205                 # do nothing
2206             } else {
2207                 die "unsupported storage type '$scfg->{type}'\n";
2208             }
2209             if ($mount_path) {
2210                 if ($format eq 'iso') {
2211                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2212                 } elsif ($isBase || defined($snapname)) {
2213                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2214                 } else {
2215                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2216                 }
2217             }
2218             return wantarray ? ($path, $use_loopdev) : $path;
2219         } else {
2220             die "unsupported image format '$format'\n";
2221         }
2222     } elsif ($type eq 'device') {
2223         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
2224         return wantarray ? ($volid, 0) : $volid;
2225     } elsif ($type eq 'bind') {
2226         if ($mountpoint->{ro}) {
2227             die "read-only bind mounts not supported\n";
2228             # Theoretically we'd have to execute both:
2229             # mount -o bind $a $b
2230             # mount -o bind,remount,ro $a $b
2231         }
2232         die "directory '$volid' does not exist\n" if ! -d $volid;
2233         &$check_mount_path($volid);
2234         PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $volid, $mount_path]) if $mount_path;
2235         return wantarray ? ($volid, 0) : $volid;
2236     }
2237
2238     die "unsupported storage";
2239 }
2240
2241 sub get_vm_volumes {
2242     my ($conf, $excludes) = @_;
2243
2244     my $vollist = [];
2245
2246     foreach_mountpoint($conf, sub {
2247         my ($ms, $mountpoint) = @_;
2248
2249         return if $excludes && $ms eq $excludes;
2250
2251         my $volid = $mountpoint->{volume};
2252
2253         return if !$volid || $mountpoint->{type} ne 'volume';
2254
2255         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2256         return if !$sid;
2257
2258         push @$vollist, $volid;
2259     });
2260
2261     return $vollist;
2262 }
2263
2264 sub mkfs {
2265     my ($dev, $rootuid, $rootgid) = @_;
2266
2267     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2268                              '-E', "root_owner=$rootuid:$rootgid",
2269                              $dev]);
2270 }
2271
2272 sub format_disk {
2273     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2274
2275     if ($volid =~ m!^/dev/.+!) {
2276         mkfs($volid);
2277         return;
2278     }
2279
2280     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2281
2282     die "cannot format volume '$volid' with no storage\n" if !$storage;
2283
2284     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2285
2286     my $path = PVE::Storage::path($storage_cfg, $volid);
2287
2288     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2289         PVE::Storage::parse_volname($storage_cfg, $volid);
2290
2291     die "cannot format volume '$volid' (format == $format)\n"
2292         if $format ne 'raw';
2293
2294     mkfs($path, $rootuid, $rootgid);
2295 }
2296
2297 sub destroy_disks {
2298     my ($storecfg, $vollist) = @_;
2299
2300     foreach my $volid (@$vollist) {
2301         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2302         warn $@ if $@;
2303     }
2304 }
2305
2306 sub create_disks {
2307     my ($storecfg, $vmid, $settings, $conf) = @_;
2308
2309     my $vollist = [];
2310
2311     eval {
2312         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2313         my $chown_vollist = [];
2314
2315         foreach_mountpoint($settings, sub {
2316             my ($ms, $mountpoint) = @_;
2317
2318             my $volid = $mountpoint->{volume};
2319             my $mp = $mountpoint->{mp};
2320
2321             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2322
2323             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2324                 my ($storeid, $size_gb) = ($1, $2);
2325
2326                 my $size_kb = int(${size_gb}*1024) * 1024;
2327
2328                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2329                 # fixme: use better naming ct-$vmid-disk-X.raw?
2330
2331                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2332                     if ($size_kb > 0) {
2333                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2334                                                            undef, $size_kb);
2335                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2336                     } else {
2337                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2338                                                            undef, 0);
2339                         push @$chown_vollist, $volid;
2340                     }
2341                 } elsif ($scfg->{type} eq 'zfspool') {
2342
2343                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2344                                                        undef, $size_kb);
2345                     push @$chown_vollist, $volid;
2346                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2347
2348                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2349                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2350
2351                 } elsif ($scfg->{type} eq 'rbd') {
2352
2353                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2354                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2355                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2356                 } else {
2357                     die "unable to create containers on storage type '$scfg->{type}'\n";
2358                 }
2359                 push @$vollist, $volid;
2360                 $mountpoint->{volume} = $volid;
2361                 $mountpoint->{size} = $size_kb * 1024;
2362                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2363             } else {
2364                 # use specified/existing volid/dir/device
2365                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2366             }
2367         });
2368
2369         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2370         foreach my $volid (@$chown_vollist) {
2371             my $path = PVE::Storage::path($storecfg, $volid, undef);
2372             chown($rootuid, $rootgid, $path);
2373         }
2374         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2375     };
2376     # free allocated images on error
2377     if (my $err = $@) {
2378         destroy_disks($storecfg, $vollist);
2379         die $err;
2380     }
2381     return $vollist;
2382 }
2383
2384 # bash completion helper
2385
2386 sub complete_os_templates {
2387     my ($cmdname, $pname, $cvalue) = @_;
2388
2389     my $cfg = PVE::Storage::config();
2390
2391     my $storeid;
2392
2393     if ($cvalue =~ m/^([^:]+):/) {
2394         $storeid = $1;
2395     }
2396
2397     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2398     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2399
2400     my $res = [];
2401     foreach my $id (keys %$data) {
2402         foreach my $item (@{$data->{$id}}) {
2403             push @$res, $item->{volid} if defined($item->{volid});
2404         }
2405     }
2406
2407     return $res;
2408 }
2409
2410 my $complete_ctid_full = sub {
2411     my ($running) = @_;
2412
2413     my $idlist = vmstatus();
2414
2415     my $active_hash = list_active_containers();
2416
2417     my $res = [];
2418
2419     foreach my $id (keys %$idlist) {
2420         my $d = $idlist->{$id};
2421         if (defined($running)) {
2422             next if $d->{template};
2423             next if $running && !$active_hash->{$id};
2424             next if !$running && $active_hash->{$id};
2425         }
2426         push @$res, $id;
2427
2428     }
2429     return $res;
2430 };
2431
2432 sub complete_ctid {
2433     return &$complete_ctid_full();
2434 }
2435
2436 sub complete_ctid_stopped {
2437     return &$complete_ctid_full(0);
2438 }
2439
2440 sub complete_ctid_running {
2441     return &$complete_ctid_full(1);
2442 }
2443
2444 sub parse_id_maps {
2445     my ($conf) = @_;
2446
2447     my $id_map = [];
2448     my $rootuid = 0;
2449     my $rootgid = 0;
2450
2451     my $lxc = $conf->{lxc};
2452     foreach my $entry (@$lxc) {
2453         my ($key, $value) = @$entry;
2454         next if $key ne 'lxc.id_map';
2455         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2456             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2457             push @$id_map, [$type, $ct, $host, $length];
2458             if ($ct == 0) {
2459                 $rootuid = $host if $type eq 'u';
2460                 $rootgid = $host if $type eq 'g';
2461             }
2462         } else {
2463             die "failed to parse id_map: $value\n";
2464         }
2465     }
2466
2467     if (!@$id_map && $conf->{unprivileged}) {
2468         # Should we read them from /etc/subuid?
2469         $id_map = [ ['u', '0', '100000', '65536'],
2470                     ['g', '0', '100000', '65536'] ];
2471         $rootuid = $rootgid = 100000;
2472     }
2473
2474     return ($id_map, $rootuid, $rootgid);
2475 }
2476
2477 sub userns_command {
2478     my ($id_map) = @_;
2479     if (@$id_map) {
2480         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2481     }
2482     return [];
2483 }
2484
2485 1;