src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX qw(EINTR);
   7
   8 use Socket;
   9
  10 use File::Path;
  11 use File::Spec;
  12 use Cwd qw();
  13 use Fcntl qw(O_RDONLY);
  14
  15 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  16 use PVE::Exception qw(raise_perm_exc);
  17 use PVE::Storage;
  18 use PVE::SafeSyslog;
  19 use PVE::INotify;
  20 use PVE::JSONSchema qw(get_standard_option);
  21 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full);
  22 use PVE::Network;
  23 use PVE::AccessControl;
  24 use PVE::ProcFSTools;
  25 use PVE::LXC::Config;
  26 use Time::HiRes qw (gettimeofday);
  27
  28 use Data::Dumper;
  29
  30 my $nodename = PVE::INotify::nodename();
  31
  32 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  33
  34 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  35                           '--xattrs',
  36                           '--xattrs-include=user.*',
  37                           '--xattrs-include=security.capability',
  38                           '--warning=no-xattr-write' ];
  39
  40 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  41
  42 my $rootfs_desc = {
  43     volume => {
  44         type => 'string',
  45         default_key => 1,
  46         format => 'pve-lxc-mp-string',
  47         format_description => 'volume',
  48         description => 'Volume, device or directory to mount into the container.',
  49     },
  50     backup => {
  51         type => 'boolean',
  52         format_description => '[1|0]',
  53         description => 'Whether to include the mountpoint in backups.',
  54         optional => 1,
  55     },
  56     size => {
  57         type => 'string',
  58         format => 'disk-size',
  59         format_description => 'DiskSize',
  60         description => 'Volume size (read only value).',
  61         optional => 1,
  62     },
  63     acl => {
  64         type => 'boolean',
  65         format_description => 'acl',
  66         description => 'Explicitly enable or disable ACL support.',
  67         optional => 1,
  68     },
  69     ro => {
  70         type => 'boolean',
  71         format_description => 'ro',
  72         description => 'Read-only mountpoint (not supported with bind mounts)',
  73         optional => 1,
  74     },
  75     quota => {
  76         type => 'boolean',
  77         format_description => '[0|1]',
  78         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
  79         optional => 1,
  80     },
  81 };
  82
  83 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  84     type => 'string', format => $rootfs_desc,
  85     description => "Use volume as container root.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  90     description => "The name of the snapshot.",
  91     type => 'string', format => 'pve-configid',
  92     maxLength => 40,
  93 });
  94
  95 my $confdesc = {
  96     lock => {
  97         optional => 1,
  98         type => 'string',
  99         description => "Lock/unlock the VM.",
 100         enum => [qw(migrate backup snapshot rollback)],
 101     },
 102     onboot => {
 103         optional => 1,
 104         type => 'boolean',
 105         description => "Specifies whether a VM will be started during system bootup.",
 106         default => 0,
 107     },
 108     startup => get_standard_option('pve-startup-order'),
 109     template => {
 110         optional => 1,
 111         type => 'boolean',
 112         description => "Enable/disable Template.",
 113         default => 0,
 114     },
 115     arch => {
 116         optional => 1,
 117         type => 'string',
 118         enum => ['amd64', 'i386'],
 119         description => "OS architecture type.",
 120         default => 'amd64',
 121     },
 122     ostype => {
 123         optional => 1,
 124         type => 'string',
 125         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux', 'alpine', 'unmanaged'],
 126         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 127     },
 128     console => {
 129         optional => 1,
 130         type => 'boolean',
 131         description => "Attach a console device (/dev/console) to the container.",
 132         default => 1,
 133     },
 134     tty => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Specify the number of tty available to the container",
 138         minimum => 0,
 139         maximum => 6,
 140         default => 2,
 141     },
 142     cpulimit => {
 143         optional => 1,
 144         type => 'number',
 145         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 146         minimum => 0,
 147         maximum => 128,
 148         default => 0,
 149     },
 150     cpuunits => {
 151         optional => 1,
 152         type => 'integer',
 153         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 154         minimum => 0,
 155         maximum => 500000,
 156         default => 1024,
 157     },
 158     memory => {
 159         optional => 1,
 160         type => 'integer',
 161         description => "Amount of RAM for the VM in MB.",
 162         minimum => 16,
 163         default => 512,
 164     },
 165     swap => {
 166         optional => 1,
 167         type => 'integer',
 168         description => "Amount of SWAP for the VM in MB.",
 169         minimum => 0,
 170         default => 512,
 171     },
 172     hostname => {
 173         optional => 1,
 174         description => "Set a host name for the container.",
 175         type => 'string', format => 'dns-name',
 176         maxLength => 255,
 177     },
 178     description => {
 179         optional => 1,
 180         type => 'string',
 181         description => "Container description. Only used on the configuration web interface.",
 182     },
 183     searchdomain => {
 184         optional => 1,
 185         type => 'string', format => 'dns-name-list',
 186         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 187     },
 188     nameserver => {
 189         optional => 1,
 190         type => 'string', format => 'address-list',
 191         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 192     },
 193     rootfs => get_standard_option('pve-ct-rootfs'),
 194     parent => {
 195         optional => 1,
 196         type => 'string', format => 'pve-configid',
 197         maxLength => 40,
 198         description => "Parent snapshot name. This is used internally, and should not be modified.",
 199     },
 200     snaptime => {
 201         optional => 1,
 202         description => "Timestamp for snapshots.",
 203         type => 'integer',
 204         minimum => 0,
 205     },
 206     cmode => {
 207         optional => 1,
 208         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 209         type => 'string',
 210         enum => ['shell', 'console', 'tty'],
 211         default => 'tty',
 212     },
 213     protection => {
 214         optional => 1,
 215         type => 'boolean',
 216         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 217         default => 0,
 218     },
 219     unprivileged => {
 220         optional => 1,
 221         type => 'boolean',
 222         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 223         default => 0,
 224     },
 225 };
 226
 227 my $valid_lxc_conf_keys = {
 228     'lxc.include' => 1,
 229     'lxc.arch' => 1,
 230     'lxc.utsname' => 1,
 231     'lxc.haltsignal' => 1,
 232     'lxc.rebootsignal' => 1,
 233     'lxc.stopsignal' => 1,
 234     'lxc.init_cmd' => 1,
 235     'lxc.network.type' => 1,
 236     'lxc.network.flags' => 1,
 237     'lxc.network.link' => 1,
 238     'lxc.network.mtu' => 1,
 239     'lxc.network.name' => 1,
 240     'lxc.network.hwaddr' => 1,
 241     'lxc.network.ipv4' => 1,
 242     'lxc.network.ipv4.gateway' => 1,
 243     'lxc.network.ipv6' => 1,
 244     'lxc.network.ipv6.gateway' => 1,
 245     'lxc.network.script.up' => 1,
 246     'lxc.network.script.down' => 1,
 247     'lxc.pts' => 1,
 248     'lxc.console.logfile' => 1,
 249     'lxc.console' => 1,
 250     'lxc.tty' => 1,
 251     'lxc.devttydir' => 1,
 252     'lxc.hook.autodev' => 1,
 253     'lxc.autodev' => 1,
 254     'lxc.kmsg' => 1,
 255     'lxc.mount' => 1,
 256     'lxc.mount.entry' => 1,
 257     'lxc.mount.auto' => 1,
 258     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 259     'lxc.rootfs.mount' => 1,
 260     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 261                             ', please use mountpoint options in the "rootfs" key',
 262     # lxc.cgroup.*
 263     'lxc.cap.drop' => 1,
 264     'lxc.cap.keep' => 1,
 265     'lxc.aa_profile' => 1,
 266     'lxc.aa_allow_incomplete' => 1,
 267     'lxc.se_context' => 1,
 268     'lxc.seccomp' => 1,
 269     'lxc.id_map' => 1,
 270     'lxc.hook.pre-start' => 1,
 271     'lxc.hook.pre-mount' => 1,
 272     'lxc.hook.mount' => 1,
 273     'lxc.hook.start' => 1,
 274     'lxc.hook.stop' => 1,
 275     'lxc.hook.post-stop' => 1,
 276     'lxc.hook.clone' => 1,
 277     'lxc.hook.destroy' => 1,
 278     'lxc.loglevel' => 1,
 279     'lxc.logfile' => 1,
 280     'lxc.start.auto' => 1,
 281     'lxc.start.delay' => 1,
 282     'lxc.start.order' => 1,
 283     'lxc.group' => 1,
 284     'lxc.environment' => 1,
 285 };
 286
 287 my $netconf_desc = {
 288     type => {
 289         type => 'string',
 290         optional => 1,
 291         description => "Network interface type.",
 292         enum => [qw(veth)],
 293     },
 294     name => {
 295         type => 'string',
 296         format_description => 'String',
 297         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 298         pattern => '[-_.\w\d]+',
 299     },
 300     bridge => {
 301         type => 'string',
 302         format_description => 'vmbr<Number>',
 303         description => 'Bridge to attach the network device to.',
 304         pattern => '[-_.\w\d]+',
 305         optional => 1,
 306     },
 307     hwaddr => {
 308         type => 'string',
 309         format_description => 'MAC',
 310         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 311         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 312         optional => 1,
 313     },
 314     mtu => {
 315         type => 'integer',
 316         format_description => 'Number',
 317         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 318         minimum => 64, # minimum ethernet frame is 64 bytes
 319         optional => 1,
 320     },
 321     ip => {
 322         type => 'string',
 323         format => 'pve-ipv4-config',
 324         format_description => 'IPv4Format/CIDR',
 325         description => 'IPv4 address in CIDR format.',
 326         optional => 1,
 327     },
 328     gw => {
 329         type => 'string',
 330         format => 'ipv4',
 331         format_description => 'GatewayIPv4',
 332         description => 'Default gateway for IPv4 traffic.',
 333         optional => 1,
 334     },
 335     ip6 => {
 336         type => 'string',
 337         format => 'pve-ipv6-config',
 338         format_description => 'IPv6Format/CIDR',
 339         description => 'IPv6 address in CIDR format.',
 340         optional => 1,
 341     },
 342     gw6 => {
 343         type => 'string',
 344         format => 'ipv6',
 345         format_description => 'GatewayIPv6',
 346         description => 'Default gateway for IPv6 traffic.',
 347         optional => 1,
 348     },
 349     firewall => {
 350         type => 'boolean',
 351         format_description => '[1|0]',
 352         description => "Controls whether this interface's firewall rules should be used.",
 353         optional => 1,
 354     },
 355     tag => {
 356         type => 'integer',
 357         format_description => 'VlanNo',
 358         minimum => '2',
 359         maximum => '4094',
 360         description => "VLAN tag for this interface.",
 361         optional => 1,
 362     },
 363     trunks => {
 364         type => 'string',
 365         pattern => qr/\d+(?:;\d+)*/,
 366         format_description => 'vlanid[;vlanid...]',
 367         description => "VLAN ids to pass through the interface",
 368         optional => 1,
 369     },
 370 };
 371 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 372
 373 my $MAX_LXC_NETWORKS = 10;
 374 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 375     $confdesc->{"net$i"} = {
 376         optional => 1,
 377         type => 'string', format => $netconf_desc,
 378         description => "Specifies network interfaces for the container.",
 379     };
 380 }
 381
 382 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 383 sub verify_lxc_mp_string{
 384     my ($mp, $noerr) = @_;
 385
 386     # do not allow:
 387     # /./ or /../
 388     # /. or /.. at the end
 389     # ../ at the beginning
 390
 391     if($mp =~ m@/\.\.?/@ ||
 392        $mp =~ m@/\.\.?$@ ||
 393        $mp =~ m@^\.\./@){
 394         return undef if $noerr;
 395         die "$mp contains illegal character sequences\n";
 396     }
 397     return $mp;
 398 }
 399
 400 my $mp_desc = {
 401     %$rootfs_desc,
 402     mp => {
 403         type => 'string',
 404         format => 'pve-lxc-mp-string',
 405         format_description => 'Path',
 406         description => 'Path to the mountpoint as seen from inside the container.',
 407     },
 408 };
 409 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 410
 411 my $unuseddesc = {
 412     optional => 1,
 413     type => 'string', format => 'pve-volume-id',
 414     description => "Reference to unused volumes.",
 415 };
 416
 417 my $MAX_MOUNT_POINTS = 10;
 418 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 419     $confdesc->{"mp$i"} = {
 420         optional => 1,
 421         type => 'string', format => $mp_desc,
 422         description => "Use volume as container mount point (experimental feature).",
 423         optional => 1,
 424     };
 425 }
 426
 427 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 428 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 429     $confdesc->{"unused$i"} = $unuseddesc;
 430 }
 431
 432 sub write_pct_config {
 433     my ($filename, $conf) = @_;
 434
 435     delete $conf->{snapstate}; # just to be sure
 436     my $volidlist = PVE::LXC::Config->get_vm_volumes($conf);
 437     my $used_volids = {};
 438     foreach my $vid (@$volidlist) {
 439         $used_volids->{$vid} = 1;
 440     }
 441
 442     # remove 'unusedX' settings if the volume is still used
 443     foreach my $key (keys %$conf) {
 444         my $value = $conf->{$key};
 445         if ($key =~ m/^unused/ && $used_volids->{$value}) {
 446             delete $conf->{$key};
 447         }
 448     }
 449
 450     my $generate_raw_config = sub {
 451         my ($conf) = @_;
 452
 453         my $raw = '';
 454
 455         # add description as comment to top of file
 456         my $descr = $conf->{description} || '';
 457         foreach my $cl (split(/\n/, $descr)) {
 458             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 459         }
 460
 461         foreach my $key (sort keys %$conf) {
 462             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 463                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 464             my $value = $conf->{$key};
 465             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 466             $raw .= "$key: $value\n";
 467         }
 468
 469         if (my $lxcconf = $conf->{lxc}) {
 470             foreach my $entry (@$lxcconf) {
 471                 my ($k, $v) = @$entry;
 472                 $raw .= "$k: $v\n";
 473             }
 474         }
 475
 476         return $raw;
 477     };
 478
 479     my $raw = &$generate_raw_config($conf);
 480
 481     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 482         $raw .= "\n[$snapname]\n";
 483         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 484     }
 485
 486     return $raw;
 487 }
 488
 489 sub check_type {
 490     my ($key, $value) = @_;
 491
 492     die "unknown setting '$key'\n" if !$confdesc->{$key};
 493
 494     my $type = $confdesc->{$key}->{type};
 495
 496     if (!defined($value)) {
 497         die "got undefined value\n";
 498     }
 499
 500     if ($value =~ m/[\n\r]/) {
 501         die "property contains a line feed\n";
 502     }
 503
 504     if ($type eq 'boolean') {
 505         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 506         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 507         die "type check ('boolean') failed - got '$value'\n";
 508     } elsif ($type eq 'integer') {
 509         return int($1) if $value =~ m/^(\d+)$/;
 510         die "type check ('integer') failed - got '$value'\n";
 511     } elsif ($type eq 'number') {
 512         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 513         die "type check ('number') failed - got '$value'\n";
 514     } elsif ($type eq 'string') {
 515         if (my $fmt = $confdesc->{$key}->{format}) {
 516             PVE::JSONSchema::check_format($fmt, $value);
 517             return $value;
 518         }
 519         return $value;
 520     } else {
 521         die "internal error"
 522     }
 523 }
 524
 525 sub parse_pct_config {
 526     my ($filename, $raw) = @_;
 527
 528     return undef if !defined($raw);
 529
 530     my $res = {
 531         digest => Digest::SHA::sha1_hex($raw),
 532         snapshots => {},
 533     };
 534
 535     $filename =~ m|/lxc/(\d+).conf$|
 536         || die "got strange filename '$filename'";
 537
 538     my $vmid = $1;
 539
 540     my $conf = $res;
 541     my $descr = '';
 542     my $section = '';
 543
 544     my @lines = split(/\n/, $raw);
 545     foreach my $line (@lines) {
 546         next if $line =~ m/^\s*$/;
 547
 548         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 549             $section = $1;
 550             $conf->{description} = $descr if $descr;
 551             $descr = '';
 552             $conf = $res->{snapshots}->{$section} = {};
 553             next;
 554         }
 555
 556         if ($line =~ m/^\#(.*)\s*$/) {
 557             $descr .= PVE::Tools::decode_text($1) . "\n";
 558             next;
 559         }
 560
 561         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 562             my $key = $1;
 563             my $value = $3;
 564             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 565             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 566                 push @{$conf->{lxc}}, [$key, $value];
 567             } elsif (my $errmsg = $validity) {
 568                 warn "vm $vmid - $key: $errmsg\n";
 569             } else {
 570                 warn "vm $vmid - unable to parse config: $line\n";
 571             }
 572         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 573             $descr .= PVE::Tools::decode_text($2);
 574         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 575             $conf->{snapstate} = $1;
 576         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 577             my $key = $1;
 578             my $value = $2;
 579             eval { $value = check_type($key, $value); };
 580             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 581             $conf->{$key} = $value;
 582         } else {
 583             warn "vm $vmid - unable to parse config: $line\n";
 584         }
 585     }
 586
 587     $conf->{description} = $descr if $descr;
 588
 589     delete $res->{snapstate}; # just to be sure
 590
 591     return $res;
 592 }
 593
 594 sub config_list {
 595     my $vmlist = PVE::Cluster::get_vmlist();
 596     my $res = {};
 597     return $res if !$vmlist || !$vmlist->{ids};
 598     my $ids = $vmlist->{ids};
 599
 600     foreach my $vmid (keys %$ids) {
 601         next if !$vmid; # skip CT0
 602         my $d = $ids->{$vmid};
 603         next if !$d->{node} || $d->{node} ne $nodename;
 604         next if !$d->{type} || $d->{type} ne 'lxc';
 605         $res->{$vmid}->{type} = 'lxc';
 606     }
 607     return $res;
 608 }
 609
 610 sub destroy_config {
 611     my ($vmid) = @_;
 612
 613     unlink PVE::LXC::Config->config_file($vmid, $nodename);
 614 }
 615
 616 sub option_exists {
 617     my ($name) = @_;
 618
 619     return defined($confdesc->{$name});
 620 }
 621
 622 # add JSON properties for create and set function
 623 sub json_config_properties {
 624     my $prop = shift;
 625
 626     foreach my $opt (keys %$confdesc) {
 627         next if $opt eq 'parent' || $opt eq 'snaptime';
 628         next if $prop->{$opt};
 629         $prop->{$opt} = $confdesc->{$opt};
 630     }
 631
 632     return $prop;
 633 }
 634
 635 # container status helpers
 636
 637 sub list_active_containers {
 638
 639     my $filename = "/proc/net/unix";
 640
 641     # similar test is used by lcxcontainers.c: list_active_containers
 642     my $res = {};
 643
 644     my $fh = IO::File->new ($filename, "r");
 645     return $res if !$fh;
 646
 647     while (defined(my $line = <$fh>)) {
 648         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 649             my $path = $1;
 650             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 651                 $res->{$1} = 1;
 652             }
 653         }
 654     }
 655
 656     close($fh);
 657
 658     return $res;
 659 }
 660
 661 # warning: this is slow
 662 sub check_running {
 663     my ($vmid) = @_;
 664
 665     my $active_hash = list_active_containers();
 666
 667     return 1 if defined($active_hash->{$vmid});
 668
 669     return undef;
 670 }
 671
 672 sub get_container_disk_usage {
 673     my ($vmid, $pid) = @_;
 674
 675     return PVE::Tools::df("/proc/$pid/root/", 1);
 676 }
 677
 678 my $last_proc_vmid_stat;
 679
 680 my $parse_cpuacct_stat = sub {
 681     my ($vmid) = @_;
 682
 683     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 684
 685     my $stat = {};
 686
 687     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 688
 689         $stat->{utime} = $1;
 690         $stat->{stime} = $2;
 691
 692     }
 693
 694     return $stat;
 695 };
 696
 697 sub vmstatus {
 698     my ($opt_vmid) = @_;
 699
 700     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 701
 702     my $active_hash = list_active_containers();
 703
 704     my $cpucount = $cpuinfo->{cpus} || 1;
 705
 706     my $cdtime = gettimeofday;
 707
 708     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 709
 710     foreach my $vmid (keys %$list) {
 711         my $d = $list->{$vmid};
 712
 713         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 714         warn $@ if $@; # ignore errors (consider them stopped)
 715
 716         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 717
 718         my $cfspath = PVE::LXC::Config->cfs_config_path($vmid);
 719         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 720
 721         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 722         $d->{name} =~ s/[\s]//g;
 723
 724         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 725
 726         $d->{lock} = $conf->{lock} || '';
 727
 728         if ($d->{pid}) {
 729             my $res = get_container_disk_usage($vmid, $d->{pid});
 730             $d->{disk} = $res->{used};
 731             $d->{maxdisk} = $res->{total};
 732         } else {
 733             $d->{disk} = 0;
 734             # use 4GB by default ??
 735             if (my $rootfs = $conf->{rootfs}) {
 736                 my $rootinfo = parse_ct_rootfs($rootfs);
 737                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 738             } else {
 739                 $d->{maxdisk} = 4*1024*1024*1024;
 740             }
 741         }
 742
 743         $d->{mem} = 0;
 744         $d->{swap} = 0;
 745         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 746         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 747
 748         $d->{uptime} = 0;
 749         $d->{cpu} = 0;
 750
 751         $d->{netout} = 0;
 752         $d->{netin} = 0;
 753
 754         $d->{diskread} = 0;
 755         $d->{diskwrite} = 0;
 756
 757         $d->{template} = PVE::LXC::Config->is_template($conf);
 758     }
 759
 760     foreach my $vmid (keys %$list) {
 761         my $d = $list->{$vmid};
 762         my $pid = $d->{pid};
 763
 764         next if !$pid; # skip stopped CTs
 765
 766         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 767         $d->{uptime} = time - $ctime; # the method lxcfs uses
 768
 769         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 770         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 771
 772         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 773         my @bytes = split(/\n/, $blkio_bytes);
 774         foreach my $byte (@bytes) {
 775             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 776                 $d->{diskread} = $2 if $key eq 'Read';
 777                 $d->{diskwrite} = $2 if $key eq 'Write';
 778             }
 779         }
 780
 781         my $pstat = &$parse_cpuacct_stat($vmid);
 782
 783         my $used = $pstat->{utime} + $pstat->{stime};
 784
 785         my $old = $last_proc_vmid_stat->{$vmid};
 786         if (!$old) {
 787             $last_proc_vmid_stat->{$vmid} = {
 788                 time => $cdtime,
 789                 used => $used,
 790                 cpu => 0,
 791             };
 792             next;
 793         }
 794
 795         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 796
 797         if ($dtime > 1000) {
 798             my $dutime = $used -  $old->{used};
 799
 800             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 801             $last_proc_vmid_stat->{$vmid} = {
 802                 time => $cdtime,
 803                 used => $used,
 804                 cpu => $d->{cpu},
 805             };
 806         } else {
 807             $d->{cpu} = $old->{cpu};
 808         }
 809     }
 810
 811     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 812
 813     foreach my $dev (keys %$netdev) {
 814         next if $dev !~ m/^veth([1-9]\d*)i/;
 815         my $vmid = $1;
 816         my $d = $list->{$vmid};
 817
 818         next if !$d;
 819
 820         $d->{netout} += $netdev->{$dev}->{receive};
 821         $d->{netin} += $netdev->{$dev}->{transmit};
 822
 823     }
 824
 825     return $list;
 826 }
 827
 828 my $parse_ct_mountpoint_full = sub {
 829     my ($desc, $data, $noerr) = @_;
 830
 831     $data //= '';
 832
 833     my $res;
 834     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 835     if ($@) {
 836         return undef if $noerr;
 837         die $@;
 838     }
 839
 840     if (defined(my $size = $res->{size})) {
 841         $size = PVE::JSONSchema::parse_size($size);
 842         if (!defined($size)) {
 843             return undef if $noerr;
 844             die "invalid size: $size\n";
 845         }
 846         $res->{size} = $size;
 847     }
 848
 849     $res->{type} = PVE::LXC::Config->classify_mountpoint($res->{volume});
 850
 851     return $res;
 852 };
 853
 854 sub parse_ct_rootfs {
 855     my ($data, $noerr) = @_;
 856
 857     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 858
 859     $res->{mp} = '/' if defined($res);
 860
 861     return $res;
 862 }
 863
 864 sub parse_ct_mountpoint {
 865     my ($data, $noerr) = @_;
 866
 867     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 868 }
 869
 870 sub print_ct_mountpoint {
 871     my ($info, $nomp) = @_;
 872     my $skip = [ 'type' ];
 873     push @$skip, 'mp' if $nomp;
 874     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 875 }
 876
 877 sub print_lxc_network {
 878     my $net = shift;
 879     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 880 }
 881
 882 sub parse_lxc_network {
 883     my ($data) = @_;
 884
 885     my $res = {};
 886
 887     return $res if !$data;
 888
 889     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 890
 891     $res->{type} = 'veth';
 892     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 893
 894     return $res;
 895 }
 896
 897 sub read_cgroup_value {
 898     my ($group, $vmid, $name, $full) = @_;
 899
 900     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 901
 902     return PVE::Tools::file_get_contents($path) if $full;
 903
 904     return PVE::Tools::file_read_firstline($path);
 905 }
 906
 907 sub write_cgroup_value {
 908    my ($group, $vmid, $name, $value) = @_;
 909
 910    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 911    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 912
 913 }
 914
 915 sub find_lxc_console_pids {
 916
 917     my $res = {};
 918
 919     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 920         my ($pid) = @_;
 921
 922         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 923         return if !$cmdline;
 924
 925         my @args = split(/\0/, $cmdline);
 926
 927         # search for lxc-console -n <vmid>
 928         return if scalar(@args) != 3;
 929         return if $args[1] ne '-n';
 930         return if $args[2] !~ m/^\d+$/;
 931         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 932
 933         my $vmid = $args[2];
 934
 935         push @{$res->{$vmid}}, $pid;
 936     });
 937
 938     return $res;
 939 }
 940
 941 sub find_lxc_pid {
 942     my ($vmid) = @_;
 943
 944     my $pid = undef;
 945     my $parser = sub {
 946         my $line = shift;
 947         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 948     };
 949     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
 950
 951     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 952
 953     return $pid;
 954 }
 955
 956 # Note: we cannot use Net:IP, because that only allows strict
 957 # CIDR networks
 958 sub parse_ipv4_cidr {
 959     my ($cidr, $noerr) = @_;
 960
 961     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
 962         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
 963     }
 964
 965     return undef if $noerr;
 966
 967     die "unable to parse ipv4 address/mask\n";
 968 }
 969
 970
 971 sub update_lxc_config {
 972     my ($storage_cfg, $vmid, $conf) = @_;
 973
 974     my $dir = "/var/lib/lxc/$vmid";
 975
 976     if ($conf->{template}) {
 977
 978         unlink "$dir/config";
 979
 980         return;
 981     }
 982
 983     my $raw = '';
 984
 985     die "missing 'arch' - internal error" if !$conf->{arch};
 986     $raw .= "lxc.arch = $conf->{arch}\n";
 987
 988     my $unprivileged = $conf->{unprivileged};
 989     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
 990
 991     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
 992     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux | alpine | unmanaged)$/x) {
 993         my $inc ="/usr/share/lxc/config/$ostype.common.conf";
 994         $inc ="/usr/share/lxc/config/common.conf" if !-f $inc;
 995         $raw .= "lxc.include = $inc\n";
 996         if ($unprivileged || $custom_idmap) {
 997             $inc = "/usr/share/lxc/config/$ostype.userns.conf";
 998             $inc = "/usr/share/lxc/config/userns.conf" if !-f $inc;
 999             $raw .= "lxc.include = $inc\n"
1000         }
1001     } else {
1002         die "implement me (ostype $ostype)";
1003     }
1004
1005     # WARNING: DO NOT REMOVE this without making sure that loop device nodes
1006     # cannot be exposed to the container with r/w access (cgroup perms).
1007     # When this is enabled mounts will still remain in the monitor's namespace
1008     # after the container unmounted them and thus will not detach from their
1009     # files while the container is running!
1010     $raw .= "lxc.monitor.unshare = 1\n";
1011
1012     # Should we read them from /etc/subuid?
1013     if ($unprivileged && !$custom_idmap) {
1014         $raw .= "lxc.id_map = u 0 100000 65536\n";
1015         $raw .= "lxc.id_map = g 0 100000 65536\n";
1016     }
1017
1018     if (!PVE::LXC::Config->has_dev_console($conf)) {
1019         $raw .= "lxc.console = none\n";
1020         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1021     }
1022
1023     my $ttycount = get_tty_count($conf);
1024     $raw .= "lxc.tty = $ttycount\n";
1025
1026     # some init scripts expect a linux terminal (turnkey).
1027     $raw .= "lxc.environment = TERM=linux\n";
1028
1029     my $utsname = $conf->{hostname} || "CT$vmid";
1030     $raw .= "lxc.utsname = $utsname\n";
1031
1032     my $memory = $conf->{memory} || 512;
1033     my $swap = $conf->{swap} // 0;
1034
1035     my $lxcmem = int($memory*1024*1024);
1036     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1037
1038     my $lxcswap = int(($memory + $swap)*1024*1024);
1039     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1040
1041     if (my $cpulimit = $conf->{cpulimit}) {
1042         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1043         my $value = int(100000*$cpulimit);
1044         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1045     }
1046
1047     my $shares = $conf->{cpuunits} || 1024;
1048     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1049
1050     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1051
1052     $raw .= "lxc.rootfs = $dir/rootfs\n";
1053
1054     my $netcount = 0;
1055     foreach my $k (keys %$conf) {
1056         next if $k !~ m/^net(\d+)$/;
1057         my $ind = $1;
1058         my $d = parse_lxc_network($conf->{$k});
1059         $netcount++;
1060         $raw .= "lxc.network.type = veth\n";
1061         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1062         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1063         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1064         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1065     }
1066
1067     if (my $lxcconf = $conf->{lxc}) {
1068         foreach my $entry (@$lxcconf) {
1069             my ($k, $v) = @$entry;
1070             $netcount++ if $k eq 'lxc.network.type';
1071             $raw .= "$k = $v\n";
1072         }
1073     }
1074
1075     $raw .= "lxc.network.type = empty\n" if !$netcount;
1076
1077     File::Path::mkpath("$dir/rootfs");
1078
1079     PVE::Tools::file_set_contents("$dir/config", $raw);
1080 }
1081
1082 # verify and cleanup nameserver list (replace \0 with ' ')
1083 sub verify_nameserver_list {
1084     my ($nameserver_list) = @_;
1085
1086     my @list = ();
1087     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1088         PVE::JSONSchema::pve_verify_ip($server);
1089         push @list, $server;
1090     }
1091
1092     return join(' ', @list);
1093 }
1094
1095 sub verify_searchdomain_list {
1096     my ($searchdomain_list) = @_;
1097
1098     my @list = ();
1099     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1100         # todo: should we add checks for valid dns domains?
1101         push @list, $server;
1102     }
1103
1104     return join(' ', @list);
1105 }
1106
1107 sub update_pct_config {
1108     my ($vmid, $conf, $running, $param, $delete) = @_;
1109
1110     my @nohotplug;
1111
1112     my $new_disks = 0;
1113     my @deleted_volumes;
1114
1115     my $rootdir;
1116     if ($running) {
1117         my $pid = find_lxc_pid($vmid);
1118         $rootdir = "/proc/$pid/root";
1119     }
1120
1121     my $hotplug_error = sub {
1122         if ($running) {
1123             push @nohotplug, @_;
1124             return 1;
1125         } else {
1126             return 0;
1127         }
1128     };
1129
1130     if (defined($delete)) {
1131         foreach my $opt (@$delete) {
1132             if (!exists($conf->{$opt})) {
1133                 warn "no such option: $opt\n";
1134                 next;
1135             }
1136
1137             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1138                 die "unable to delete required option '$opt'\n";
1139             } elsif ($opt eq 'swap') {
1140                 delete $conf->{$opt};
1141                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1142             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1143                 delete $conf->{$opt};
1144             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1145                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1146                 next if $hotplug_error->($opt);
1147                 delete $conf->{$opt};
1148             } elsif ($opt =~ m/^net(\d)$/) {
1149                 delete $conf->{$opt};
1150                 next if !$running;
1151                 my $netid = $1;
1152                 PVE::Network::veth_delete("veth${vmid}i$netid");
1153             } elsif ($opt eq 'protection') {
1154                 delete $conf->{$opt};
1155             } elsif ($opt =~ m/^unused(\d+)$/) {
1156                 next if $hotplug_error->($opt);
1157                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1158                 push @deleted_volumes, $conf->{$opt};
1159                 delete $conf->{$opt};
1160             } elsif ($opt =~ m/^mp(\d+)$/) {
1161                 next if $hotplug_error->($opt);
1162                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1163                 my $mp = parse_ct_mountpoint($conf->{$opt});
1164                 delete $conf->{$opt};
1165                 if ($mp->{type} eq 'volume') {
1166                     PVE::LXC::Config->add_unused_volume($conf, $mp->{volume});
1167                 }
1168             } elsif ($opt eq 'unprivileged') {
1169                 die "unable to delete read-only option: '$opt'\n";
1170             } else {
1171                 die "implement me (delete: $opt)"
1172             }
1173             PVE::LXC::Config->write_config($vmid, $conf) if $running;
1174         }
1175     }
1176
1177     # There's no separate swap size to configure, there's memory and "total"
1178     # memory (iow. memory+swap). This means we have to change them together.
1179     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1180     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1181     if (defined($wanted_memory) || defined($wanted_swap)) {
1182
1183         my $old_memory = ($conf->{memory} || 512);
1184         my $old_swap = ($conf->{swap} || 0);
1185
1186         $wanted_memory //= $old_memory;
1187         $wanted_swap //= $old_swap;
1188
1189         my $total = $wanted_memory + $wanted_swap;
1190         if ($running) {
1191             my $old_total = $old_memory + $old_swap;
1192             if ($total > $old_total) {
1193                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1194                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1195             } else {
1196                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1197                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1198             }
1199         }
1200         $conf->{memory} = $wanted_memory;
1201         $conf->{swap} = $wanted_swap;
1202
1203         PVE::LXC::Config->write_config($vmid, $conf) if $running;
1204     }
1205
1206     my $used_volids = {};
1207
1208     foreach my $opt (keys %$param) {
1209         my $value = $param->{$opt};
1210         if ($opt eq 'hostname') {
1211             $conf->{$opt} = $value;
1212         } elsif ($opt eq 'onboot') {
1213             $conf->{$opt} = $value ? 1 : 0;
1214         } elsif ($opt eq 'startup') {
1215             $conf->{$opt} = $value;
1216         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1217             next if $hotplug_error->($opt);
1218             $conf->{$opt} = $value;
1219         } elsif ($opt eq 'nameserver') {
1220             next if $hotplug_error->($opt);
1221             my $list = verify_nameserver_list($value);
1222             $conf->{$opt} = $list;
1223         } elsif ($opt eq 'searchdomain') {
1224             next if $hotplug_error->($opt);
1225             my $list = verify_searchdomain_list($value);
1226             $conf->{$opt} = $list;
1227         } elsif ($opt eq 'cpulimit') {
1228             next if $hotplug_error->($opt); # FIXME: hotplug
1229             $conf->{$opt} = $value;
1230         } elsif ($opt eq 'cpuunits') {
1231             $conf->{$opt} = $value;
1232             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1233         } elsif ($opt eq 'description') {
1234             $conf->{$opt} = PVE::Tools::encode_text($value);
1235         } elsif ($opt =~ m/^net(\d+)$/) {
1236             my $netid = $1;
1237             my $net = parse_lxc_network($value);
1238             if (!$running) {
1239                 $conf->{$opt} = print_lxc_network($net);
1240             } else {
1241                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1242             }
1243         } elsif ($opt eq 'protection') {
1244             $conf->{$opt} = $value ? 1 : 0;
1245         } elsif ($opt =~ m/^mp(\d+)$/) {
1246             next if $hotplug_error->($opt);
1247             check_protection($conf, "can't update CT $vmid drive '$opt'");
1248             my $old = $conf->{$opt};
1249             $conf->{$opt} = $value;
1250             if (defined($old)) {
1251                 my $mp = parse_ct_mountpoint($old);
1252                 if ($mp->{type} eq 'volume') {
1253                     PVE::LXC::Config->add_unused_volume($conf, $mp->{volume});
1254                 }
1255             }
1256             $new_disks = 1;
1257             my $mp = parse_ct_mountpoint($value);
1258             $used_volids->{$mp->{volume}} = 1;
1259         } elsif ($opt eq 'rootfs') {
1260             next if $hotplug_error->($opt);
1261             check_protection($conf, "can't update CT $vmid drive '$opt'");
1262             my $old = $conf->{$opt};
1263             $conf->{$opt} = $value;
1264             if (defined($old)) {
1265                 my $mp = parse_ct_rootfs($old);
1266                 if ($mp->{type} eq 'volume') {
1267                     PVE::LXC::Config->add_unused_volume($conf, $mp->{volume});
1268                 }
1269             }
1270             my $mp = parse_ct_rootfs($value);
1271             $used_volids->{$mp->{volume}} = 1;
1272         } elsif ($opt eq 'unprivileged') {
1273             die "unable to modify read-only option: '$opt'\n";
1274         } elsif ($opt eq 'ostype') {
1275             next if $hotplug_error->($opt);
1276             $conf->{$opt} = $value;
1277         } else {
1278             die "implement me: $opt";
1279         }
1280         PVE::LXC::Config->write_config($vmid, $conf) if $running;
1281     }
1282
1283     # Apply deletions and creations of new volumes
1284     if (@deleted_volumes) {
1285         my $storage_cfg = PVE::Storage::config();
1286         foreach my $volume (@deleted_volumes) {
1287             next if $used_volids->{$volume}; # could have been re-added, too
1288             # also check for references in snapshots
1289             next if PVE::LXC::Config->is_volume_in_use($conf, $volume, 1);
1290             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1291         }
1292     }
1293
1294     if ($new_disks) {
1295         my $storage_cfg = PVE::Storage::config();
1296         create_disks($storage_cfg, $vmid, $conf, $conf);
1297     }
1298
1299     # This should be the last thing we do here
1300     if ($running && scalar(@nohotplug)) {
1301         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1302     }
1303 }
1304
1305 sub get_tty_count {
1306     my ($conf) = @_;
1307
1308     return $conf->{tty} // $confdesc->{tty}->{default};
1309 }
1310
1311 sub get_cmode {
1312     my ($conf) = @_;
1313
1314     return $conf->{cmode} // $confdesc->{cmode}->{default};
1315 }
1316
1317 sub get_console_command {
1318     my ($vmid, $conf) = @_;
1319
1320     my $cmode = get_cmode($conf);
1321
1322     if ($cmode eq 'console') {
1323         return ['lxc-console', '-n',  $vmid, '-t', 0];
1324     } elsif ($cmode eq 'tty') {
1325         return ['lxc-console', '-n',  $vmid];
1326     } elsif ($cmode eq 'shell') {
1327         return ['lxc-attach', '--clear-env', '-n', $vmid];
1328     } else {
1329         die "internal error";
1330     }
1331 }
1332
1333 sub get_primary_ips {
1334     my ($conf) = @_;
1335
1336     # return data from net0
1337
1338     return undef if !defined($conf->{net0});
1339     my $net = parse_lxc_network($conf->{net0});
1340
1341     my $ipv4 = $net->{ip};
1342     if ($ipv4) {
1343         if ($ipv4 =~ /^(dhcp|manual)$/) {
1344             $ipv4 = undef
1345         } else {
1346             $ipv4 =~ s!/\d+$!!;
1347         }
1348     }
1349     my $ipv6 = $net->{ip6};
1350     if ($ipv6) {
1351         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1352             $ipv6 = undef;
1353         } else {
1354             $ipv6 =~ s!/\d+$!!;
1355         }
1356     }
1357
1358     return ($ipv4, $ipv6);
1359 }
1360
1361 sub delete_mountpoint_volume {
1362     my ($storage_cfg, $vmid, $volume) = @_;
1363
1364     return if PVE::LXC::Config->classify_mountpoint($volume) ne 'volume';
1365
1366     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1367     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1368 }
1369
1370 sub destroy_lxc_container {
1371     my ($storage_cfg, $vmid, $conf) = @_;
1372
1373     PVE::LXC::Config->foreach_mountpoint($conf, sub {
1374         my ($ms, $mountpoint) = @_;
1375         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1376     });
1377
1378     rmdir "/var/lib/lxc/$vmid/rootfs";
1379     unlink "/var/lib/lxc/$vmid/config";
1380     rmdir "/var/lib/lxc/$vmid";
1381     destroy_config($vmid);
1382
1383     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1384     #PVE::Tools::run_command($cmd);
1385 }
1386
1387 sub vm_stop_cleanup {
1388     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1389
1390     eval {
1391         if (!$keepActive) {
1392
1393             my $vollist = PVE::LXC::Config->get_vm_volumes($conf);
1394             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1395         }
1396     };
1397     warn $@ if $@; # avoid errors - just warn
1398 }
1399
1400 my $safe_num_ne = sub {
1401     my ($a, $b) = @_;
1402
1403     return 0 if !defined($a) && !defined($b);
1404     return 1 if !defined($a);
1405     return 1 if !defined($b);
1406
1407     return $a != $b;
1408 };
1409
1410 my $safe_string_ne = sub {
1411     my ($a, $b) = @_;
1412
1413     return 0 if !defined($a) && !defined($b);
1414     return 1 if !defined($a);
1415     return 1 if !defined($b);
1416
1417     return $a ne $b;
1418 };
1419
1420 sub update_net {
1421     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1422
1423     if ($newnet->{type} ne 'veth') {
1424         # for when there are physical interfaces
1425         die "cannot update interface of type $newnet->{type}";
1426     }
1427
1428     my $veth = "veth${vmid}i${netid}";
1429     my $eth = $newnet->{name};
1430
1431     if (my $oldnetcfg = $conf->{$opt}) {
1432         my $oldnet = parse_lxc_network($oldnetcfg);
1433
1434         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1435             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1436
1437             PVE::Network::veth_delete($veth);
1438             delete $conf->{$opt};
1439             PVE::LXC::Config->write_config($vmid, $conf);
1440
1441             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1442
1443         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1444                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1445                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1446
1447                 if ($oldnet->{bridge}) {
1448                     PVE::Network::tap_unplug($veth);
1449                     foreach (qw(bridge tag firewall)) {
1450                         delete $oldnet->{$_};
1451                     }
1452                     $conf->{$opt} = print_lxc_network($oldnet);
1453                     PVE::LXC::Config->write_config($vmid, $conf);
1454                 }
1455
1456                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1457                 foreach (qw(bridge tag firewall)) {
1458                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1459                 }
1460                 $conf->{$opt} = print_lxc_network($oldnet);
1461                 PVE::LXC::Config->write_config($vmid, $conf);
1462         }
1463     } else {
1464         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1465     }
1466
1467     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1468 }
1469
1470 sub hotplug_net {
1471     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1472
1473     my $veth = "veth${vmid}i${netid}";
1474     my $vethpeer = $veth . "p";
1475     my $eth = $newnet->{name};
1476
1477     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1478     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1479
1480     # attach peer in container
1481     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1482     PVE::Tools::run_command($cmd);
1483
1484     # link up peer in container
1485     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1486     PVE::Tools::run_command($cmd);
1487
1488     my $done = { type => 'veth' };
1489     foreach (qw(bridge tag firewall hwaddr name)) {
1490         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1491     }
1492     $conf->{$opt} = print_lxc_network($done);
1493
1494     PVE::LXC::Config->write_config($vmid, $conf);
1495 }
1496
1497 sub update_ipconfig {
1498     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1499
1500     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1501
1502     my $optdata = parse_lxc_network($conf->{$opt});
1503     my $deleted = [];
1504     my $added = [];
1505     my $nscmd = sub {
1506         my $cmdargs = shift;
1507         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1508     };
1509     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1510
1511     my $change_ip_config = sub {
1512         my ($ipversion) = @_;
1513
1514         my $family_opt = "-$ipversion";
1515         my $suffix = $ipversion == 4 ? '' : $ipversion;
1516         my $gw= "gw$suffix";
1517         my $ip= "ip$suffix";
1518
1519         my $newip = $newnet->{$ip};
1520         my $newgw = $newnet->{$gw};
1521         my $oldip = $optdata->{$ip};
1522
1523         my $change_ip = &$safe_string_ne($oldip, $newip);
1524         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1525
1526         return if !$change_ip && !$change_gw;
1527
1528         # step 1: add new IP, if this fails we cancel
1529         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1530         if ($change_ip && $is_real_ip) {
1531             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1532             if (my $err = $@) {
1533                 warn $err;
1534                 return;
1535             }
1536         }
1537
1538         # step 2: replace gateway
1539         #   If this fails we delete the added IP and cancel.
1540         #   If it succeeds we save the config and delete the old IP, ignoring
1541         #   errors. The config is then saved.
1542         # Note: 'ip route replace' can add
1543         if ($change_gw) {
1544             if ($newgw) {
1545                 eval {
1546                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1547                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1548                     }
1549                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1550                 };
1551                 if (my $err = $@) {
1552                     warn $err;
1553                     # the route was not replaced, the old IP is still available
1554                     # rollback (delete new IP) and cancel
1555                     if ($change_ip) {
1556                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1557                         warn $@ if $@; # no need to die here
1558                     }
1559                     return;
1560                 }
1561             } else {
1562                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1563                 # if the route was not deleted, the guest might have deleted it manually
1564                 # warn and continue
1565                 warn $@ if $@;
1566             }
1567         }
1568
1569         # from this point on we save the configuration
1570         # step 3: delete old IP ignoring errors
1571         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1572             # We need to enable promote_secondaries, otherwise our newly added
1573             # address will be removed along with the old one.
1574             my $promote = 0;
1575             eval {
1576                 if ($ipversion == 4) {
1577                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1578                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1579                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1580                 }
1581                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1582             };
1583             warn $@ if $@; # no need to die here
1584
1585             if ($ipversion == 4) {
1586                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1587             }
1588         }
1589
1590         foreach my $property ($ip, $gw) {
1591             if ($newnet->{$property}) {
1592                 $optdata->{$property} = $newnet->{$property};
1593             } else {
1594                 delete $optdata->{$property};
1595             }
1596         }
1597         $conf->{$opt} = print_lxc_network($optdata);
1598         PVE::LXC::Config->write_config($vmid, $conf);
1599         $lxc_setup->setup_network($conf);
1600     };
1601
1602     &$change_ip_config(4);
1603     &$change_ip_config(6);
1604
1605 }
1606
1607 my $enter_namespace = sub {
1608     my ($vmid, $pid, $which, $type) = @_;
1609     sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY
1610         or die "failed to open $which namespace of container $vmid: $!\n";
1611     PVE::Tools::setns(fileno($fd), $type)
1612         or die "failed to enter $which namespace of container $vmid: $!\n";
1613     close $fd;
1614 };
1615
1616 my $do_syncfs = sub {
1617     my ($vmid, $pid, $socket) = @_;
1618
1619     &$enter_namespace($vmid, $pid, 'mnt', PVE::Tools::CLONE_NEWNS);
1620
1621     # Tell the parent process to start reading our /proc/mounts
1622     print {$socket} "go\n";
1623     $socket->flush();
1624
1625     # Receive /proc/self/mounts
1626     my $mountdata = do { local $/ = undef; <$socket> };
1627     close $socket;
1628
1629     # Now sync all mountpoints...
1630     my $mounts = PVE::ProcFSTools::parse_mounts($mountdata);
1631     foreach my $mp (@$mounts) {
1632         my ($what, $dir, $fs) = @$mp;
1633         next if $fs eq 'fuse.lxcfs';
1634         eval { PVE::Tools::sync_mountpoint($dir); };
1635         warn $@ if $@;
1636     }
1637 };
1638
1639 sub sync_container_namespace {
1640     my ($vmid) = @_;
1641     my $pid = find_lxc_pid($vmid);
1642
1643     # SOCK_DGRAM is nicer for barriers but cannot be slurped
1644     socketpair my $pfd, my $cfd, AF_UNIX, SOCK_STREAM, PF_UNSPEC
1645         or die "failed to create socketpair: $!\n";
1646
1647     my $child = fork();
1648     die "fork failed: $!\n" if !defined($child);
1649
1650     if (!$child) {
1651         eval {
1652             close $pfd;
1653             &$do_syncfs($vmid, $pid, $cfd);
1654         };
1655         if (my $err = $@) {
1656             warn $err;
1657             POSIX::_exit(1);
1658         }
1659         POSIX::_exit(0);
1660     }
1661     close $cfd;
1662     my $go = <$pfd>;
1663     die "failed to enter container namespace\n" if $go ne "go\n";
1664
1665     open my $mounts, '<', "/proc/$child/mounts"
1666         or die "failed to open container's /proc/mounts: $!\n";
1667     my $mountdata = do { local $/ = undef; <$mounts> };
1668     close $mounts;
1669     print {$pfd} $mountdata;
1670     close $pfd;
1671
1672     while (waitpid($child, 0) != $child) {}
1673     die "failed to sync container namespace\n" if $? != 0;
1674 }
1675
1676 sub template_create {
1677     my ($vmid, $conf) = @_;
1678
1679     my $storecfg = PVE::Storage::config();
1680
1681     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1682     my $volid = $rootinfo->{volume};
1683
1684     die "Template feature is not available for '$volid'\n"
1685         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1686
1687     PVE::Storage::activate_volumes($storecfg, [$volid]);
1688
1689     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1690     $rootinfo->{volume} = $template_volid;
1691     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1692
1693     PVE::LXC::Config->write_config($vmid, $conf);
1694 }
1695
1696 sub check_ct_modify_config_perm {
1697     my ($rpcenv, $authuser, $vmid, $pool, $newconf, $delete) = @_;
1698
1699     return 1 if $authuser eq 'root@pam';
1700
1701     my $check = sub {
1702         my ($opt, $delete) = @_;
1703         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1704             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1705         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1706             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1707             return if $delete;
1708             my $data = $opt eq 'rootfs' ? parse_ct_rootfs($newconf->{$opt})
1709                                         : parse_ct_mountpoint($newconf->{$opt});
1710             raise_perm_exc("mountpoint type $data->{type}") if $data->{type} ne 'volume';
1711         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1712             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1713         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1714                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1715             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1716         } else {
1717             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1718         }
1719     };
1720
1721     foreach my $opt (keys %$newconf) {
1722         &$check($opt, 0);
1723     }
1724     foreach my $opt (@$delete) {
1725         &$check($opt, 1);
1726     }
1727
1728     return 1;
1729 }
1730
1731 sub umount_all {
1732     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
1733
1734     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1735     my $volid_list = PVE::LXC::Config->get_vm_volumes($conf);
1736
1737     PVE::LXC::Config->foreach_mountpoint_reverse($conf, sub {
1738         my ($ms, $mountpoint) = @_;
1739
1740         my $volid = $mountpoint->{volume};
1741         my $mount = $mountpoint->{mp};
1742
1743         return if !$volid || !$mount;
1744
1745         my $mount_path = "$rootdir/$mount";
1746         $mount_path =~ s!/+!/!g;
1747
1748         return if !PVE::ProcFSTools::is_mounted($mount_path);
1749
1750         eval {
1751             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1752         };
1753         if (my $err = $@) {
1754             if ($noerr) {
1755                 warn $err;
1756             } else {
1757                 die $err;
1758             }
1759         }
1760     });
1761 }
1762
1763 sub mount_all {
1764     my ($vmid, $storage_cfg, $conf) = @_;
1765
1766     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1767     File::Path::make_path($rootdir);
1768
1769     my $volid_list = PVE::LXC::Config->get_vm_volumes($conf);
1770     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
1771
1772     eval {
1773         PVE::LXC::Config->foreach_mountpoint($conf, sub {
1774             my ($ms, $mountpoint) = @_;
1775
1776             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
1777         });
1778     };
1779     if (my $err = $@) {
1780         warn "mounting container failed\n";
1781         umount_all($vmid, $storage_cfg, $conf, 1);
1782         die $err;
1783     }
1784
1785     return $rootdir;
1786 }
1787
1788
1789 sub mountpoint_mount_path {
1790     my ($mountpoint, $storage_cfg, $snapname) = @_;
1791
1792     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
1793 }
1794
1795 my $check_mount_path = sub {
1796     my ($path) = @_;
1797     $path = File::Spec->canonpath($path);
1798     my $real = Cwd::realpath($path);
1799     if ($real ne $path) {
1800         die "mount path modified by symlink: $path != $real";
1801     }
1802 };
1803
1804 sub query_loopdev {
1805     my ($path) = @_;
1806     my $found;
1807     my $parser = sub {
1808         my $line = shift;
1809         if ($line =~ m@^(/dev/loop\d+):@) {
1810             $found = $1;
1811         }
1812     };
1813     my $cmd = ['losetup', '--associated', $path];
1814     PVE::Tools::run_command($cmd, outfunc => $parser);
1815     return $found;
1816 }
1817
1818 # Run a function with a file attached to a loop device.
1819 # The loop device is always detached afterwards (or set to autoclear).
1820 # Returns the loop device.
1821 sub run_with_loopdev {
1822     my ($func, $file) = @_;
1823     my $device = query_loopdev($file);
1824     # Try to reuse an existing device
1825     if ($device) {
1826         # We assume that whoever setup the loop device is responsible for
1827         # detaching it.
1828         &$func($device);
1829         return $device;
1830     }
1831
1832     my $parser = sub {
1833         my $line = shift;
1834         if ($line =~ m@^(/dev/loop\d+)$@) {
1835             $device = $1;
1836         }
1837     };
1838     PVE::Tools::run_command(['losetup', '--show', '-f', $file], outfunc => $parser);
1839     die "failed to setup loop device for $file\n" if !$device;
1840     eval { &$func($device); };
1841     my $err = $@;
1842     PVE::Tools::run_command(['losetup', '-d', $device]);
1843     die $err if $err;
1844     return $device;
1845 }
1846
1847 sub bindmount {
1848     my ($dir, $dest, $ro, @extra_opts) = @_;
1849     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $dir, $dest]);
1850     if ($ro) {
1851         eval { PVE::Tools::run_command(['mount', '-o', 'bind,remount,ro', $dest]); };
1852         if (my $err = $@) {
1853             warn "bindmount error\n";
1854             # don't leave writable bind-mounts behind...
1855             PVE::Tools::run_command(['umount', $dest]);
1856             die $err;
1857         }
1858     }
1859 }
1860
1861 # use $rootdir = undef to just return the corresponding mount path
1862 sub mountpoint_mount {
1863     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
1864
1865     my $volid = $mountpoint->{volume};
1866     my $mount = $mountpoint->{mp};
1867     my $type = $mountpoint->{type};
1868     my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota};
1869     my $mounted_dev;
1870
1871     return if !$volid || !$mount;
1872
1873     my $mount_path;
1874
1875     if (defined($rootdir)) {
1876         $rootdir =~ s!/+$!!;
1877         $mount_path = "$rootdir/$mount";
1878         $mount_path =~ s!/+!/!g;
1879         &$check_mount_path($mount_path);
1880         File::Path::mkpath($mount_path);
1881     }
1882
1883     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1884
1885     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
1886
1887     my $optstring = '';
1888     if (defined($mountpoint->{acl})) {
1889         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
1890     }
1891     my $readonly = $mountpoint->{ro};
1892
1893     my @extra_opts = ('-o', $optstring);
1894
1895     if ($storage) {
1896
1897         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1898         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
1899
1900         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1901             PVE::Storage::parse_volname($storage_cfg, $volid);
1902
1903         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
1904
1905         if ($format eq 'subvol') {
1906             if ($mount_path) {
1907                 if ($snapname) {
1908                     if ($scfg->{type} eq 'zfspool') {
1909                         my $path_arg = $path;
1910                         $path_arg =~ s!^/+!!;
1911                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
1912                     } else {
1913                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
1914                     }
1915                 } else {
1916                     bindmount($path, $mount_path, $readonly, @extra_opts);
1917                     warn "cannot enable quota control for bind mounted subvolumes\n" if $quota;
1918                 }
1919             }
1920             return wantarray ? ($path, 0, $mounted_dev) : $path;
1921         } elsif ($format eq 'raw' || $format eq 'iso') {
1922             my $domount = sub {
1923                 my ($path) = @_;
1924                 if ($mount_path) {
1925                     if ($format eq 'iso') {
1926                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
1927                     } elsif ($isBase || defined($snapname)) {
1928                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
1929                     } else {
1930                         if ($quota) {
1931                             push @extra_opts, '-o', 'usrjquota=aquota.user,grpjquota=aquota.group,jqfmt=vfsv0';
1932                         }
1933                         push @extra_opts, '-o', 'ro' if $readonly;
1934                         PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
1935                     }
1936                 }
1937             };
1938             my $use_loopdev = 0;
1939             if ($scfg->{path}) {
1940                 $mounted_dev = run_with_loopdev($domount, $path);
1941                 $use_loopdev = 1;
1942             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
1943                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
1944                 $mounted_dev = $path;
1945                 &$domount($path);
1946             } else {
1947                 die "unsupported storage type '$scfg->{type}'\n";
1948             }
1949             return wantarray ? ($path, $use_loopdev, $mounted_dev) : $path;
1950         } else {
1951             die "unsupported image format '$format'\n";
1952         }
1953     } elsif ($type eq 'device') {
1954                         push @extra_opts, '-o', 'ro' if $readonly;
1955         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
1956         return wantarray ? ($volid, 0, $volid) : $volid;
1957     } elsif ($type eq 'bind') {
1958         die "directory '$volid' does not exist\n" if ! -d $volid;
1959         &$check_mount_path($volid);
1960         bindmount($volid, $mount_path, $readonly, @extra_opts) if $mount_path;
1961         warn "cannot enable quota control for bind mounts\n" if $quota;
1962         return wantarray ? ($volid, 0, undef) : $volid;
1963     }
1964
1965     die "unsupported storage";
1966 }
1967
1968 sub mkfs {
1969     my ($dev, $rootuid, $rootgid) = @_;
1970
1971     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
1972                              '-E', "root_owner=$rootuid:$rootgid",
1973                              $dev]);
1974 }
1975
1976 sub format_disk {
1977     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
1978
1979     if ($volid =~ m!^/dev/.+!) {
1980         mkfs($volid);
1981         return;
1982     }
1983
1984     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1985
1986     die "cannot format volume '$volid' with no storage\n" if !$storage;
1987
1988     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
1989
1990     my $path = PVE::Storage::path($storage_cfg, $volid);
1991
1992     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1993         PVE::Storage::parse_volname($storage_cfg, $volid);
1994
1995     die "cannot format volume '$volid' (format == $format)\n"
1996         if $format ne 'raw';
1997
1998     mkfs($path, $rootuid, $rootgid);
1999 }
2000
2001 sub destroy_disks {
2002     my ($storecfg, $vollist) = @_;
2003
2004     foreach my $volid (@$vollist) {
2005         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2006         warn $@ if $@;
2007     }
2008 }
2009
2010 sub create_disks {
2011     my ($storecfg, $vmid, $settings, $conf) = @_;
2012
2013     my $vollist = [];
2014
2015     eval {
2016         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2017         my $chown_vollist = [];
2018
2019         PVE::LXC::Config->foreach_mountpoint($settings, sub {
2020             my ($ms, $mountpoint) = @_;
2021
2022             my $volid = $mountpoint->{volume};
2023             my $mp = $mountpoint->{mp};
2024
2025             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2026
2027             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2028                 my ($storeid, $size_gb) = ($1, $2);
2029
2030                 my $size_kb = int(${size_gb}*1024) * 1024;
2031
2032                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2033                 # fixme: use better naming ct-$vmid-disk-X.raw?
2034
2035                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2036                     if ($size_kb > 0) {
2037                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2038                                                            undef, $size_kb);
2039                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2040                     } else {
2041                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2042                                                            undef, 0);
2043                         push @$chown_vollist, $volid;
2044                     }
2045                 } elsif ($scfg->{type} eq 'zfspool') {
2046
2047                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2048                                                        undef, $size_kb);
2049                     push @$chown_vollist, $volid;
2050                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2051
2052                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2053                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2054
2055                 } elsif ($scfg->{type} eq 'rbd') {
2056
2057                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2058                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2059                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2060                 } else {
2061                     die "unable to create containers on storage type '$scfg->{type}'\n";
2062                 }
2063                 push @$vollist, $volid;
2064                 $mountpoint->{volume} = $volid;
2065                 $mountpoint->{size} = $size_kb * 1024;
2066                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2067             } else {
2068                 # use specified/existing volid/dir/device
2069                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2070             }
2071         });
2072
2073         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2074         foreach my $volid (@$chown_vollist) {
2075             my $path = PVE::Storage::path($storecfg, $volid, undef);
2076             chown($rootuid, $rootgid, $path);
2077         }
2078         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2079     };
2080     # free allocated images on error
2081     if (my $err = $@) {
2082         destroy_disks($storecfg, $vollist);
2083         die $err;
2084     }
2085     return $vollist;
2086 }
2087
2088 # bash completion helper
2089
2090 sub complete_os_templates {
2091     my ($cmdname, $pname, $cvalue) = @_;
2092
2093     my $cfg = PVE::Storage::config();
2094
2095     my $storeid;
2096
2097     if ($cvalue =~ m/^([^:]+):/) {
2098         $storeid = $1;
2099     }
2100
2101     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2102     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2103
2104     my $res = [];
2105     foreach my $id (keys %$data) {
2106         foreach my $item (@{$data->{$id}}) {
2107             push @$res, $item->{volid} if defined($item->{volid});
2108         }
2109     }
2110
2111     return $res;
2112 }
2113
2114 my $complete_ctid_full = sub {
2115     my ($running) = @_;
2116
2117     my $idlist = vmstatus();
2118
2119     my $active_hash = list_active_containers();
2120
2121     my $res = [];
2122
2123     foreach my $id (keys %$idlist) {
2124         my $d = $idlist->{$id};
2125         if (defined($running)) {
2126             next if $d->{template};
2127             next if $running && !$active_hash->{$id};
2128             next if !$running && $active_hash->{$id};
2129         }
2130         push @$res, $id;
2131
2132     }
2133     return $res;
2134 };
2135
2136 sub complete_ctid {
2137     return &$complete_ctid_full();
2138 }
2139
2140 sub complete_ctid_stopped {
2141     return &$complete_ctid_full(0);
2142 }
2143
2144 sub complete_ctid_running {
2145     return &$complete_ctid_full(1);
2146 }
2147
2148 sub parse_id_maps {
2149     my ($conf) = @_;
2150
2151     my $id_map = [];
2152     my $rootuid = 0;
2153     my $rootgid = 0;
2154
2155     my $lxc = $conf->{lxc};
2156     foreach my $entry (@$lxc) {
2157         my ($key, $value) = @$entry;
2158         next if $key ne 'lxc.id_map';
2159         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2160             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2161             push @$id_map, [$type, $ct, $host, $length];
2162             if ($ct == 0) {
2163                 $rootuid = $host if $type eq 'u';
2164                 $rootgid = $host if $type eq 'g';
2165             }
2166         } else {
2167             die "failed to parse id_map: $value\n";
2168         }
2169     }
2170
2171     if (!@$id_map && $conf->{unprivileged}) {
2172         # Should we read them from /etc/subuid?
2173         $id_map = [ ['u', '0', '100000', '65536'],
2174                     ['g', '0', '100000', '65536'] ];
2175         $rootuid = $rootgid = 100000;
2176     }
2177
2178     return ($id_map, $rootuid, $rootgid);
2179 }
2180
2181 sub userns_command {
2182     my ($id_map) = @_;
2183     if (@$id_map) {
2184         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2185     }
2186     return [];
2187 }
2188
2189
2190 1;