src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX qw(EINTR);
   7
   8 use Socket;
   9
  10 use File::Path;
  11 use File::Spec;
  12 use Cwd qw();
  13 use Fcntl qw(O_RDONLY);
  14
  15 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  16 use PVE::Exception qw(raise_perm_exc);
  17 use PVE::Storage;
  18 use PVE::SafeSyslog;
  19 use PVE::INotify;
  20 use PVE::JSONSchema qw(get_standard_option);
  21 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full);
  22 use PVE::Network;
  23 use PVE::AccessControl;
  24 use PVE::ProcFSTools;
  25 use PVE::LXC::Config;
  26 use Time::HiRes qw (gettimeofday);
  27
  28 use Data::Dumper;
  29
  30 my $nodename = PVE::INotify::nodename();
  31
  32 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  33
  34 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  35                           '--xattrs',
  36                           '--xattrs-include=user.*',
  37                           '--xattrs-include=security.capability',
  38                           '--warning=no-xattr-write' ];
  39
  40 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  41
  42 my $rootfs_desc = {
  43     volume => {
  44         type => 'string',
  45         default_key => 1,
  46         format => 'pve-lxc-mp-string',
  47         format_description => 'volume',
  48         description => 'Volume, device or directory to mount into the container.',
  49     },
  50     backup => {
  51         type => 'boolean',
  52         format_description => '[1|0]',
  53         description => 'Whether to include the mountpoint in backups.',
  54         optional => 1,
  55     },
  56     size => {
  57         type => 'string',
  58         format => 'disk-size',
  59         format_description => 'DiskSize',
  60         description => 'Volume size (read only value).',
  61         optional => 1,
  62     },
  63     acl => {
  64         type => 'boolean',
  65         format_description => 'acl',
  66         description => 'Explicitly enable or disable ACL support.',
  67         optional => 1,
  68     },
  69     ro => {
  70         type => 'boolean',
  71         format_description => 'ro',
  72         description => 'Read-only mountpoint (not supported with bind mounts)',
  73         optional => 1,
  74     },
  75     quota => {
  76         type => 'boolean',
  77         format_description => '[0|1]',
  78         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
  79         optional => 1,
  80     },
  81 };
  82
  83 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  84     type => 'string', format => $rootfs_desc,
  85     description => "Use volume as container root.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  90     description => "The name of the snapshot.",
  91     type => 'string', format => 'pve-configid',
  92     maxLength => 40,
  93 });
  94
  95 my $confdesc = {
  96     lock => {
  97         optional => 1,
  98         type => 'string',
  99         description => "Lock/unlock the VM.",
 100         enum => [qw(migrate backup snapshot rollback)],
 101     },
 102     onboot => {
 103         optional => 1,
 104         type => 'boolean',
 105         description => "Specifies whether a VM will be started during system bootup.",
 106         default => 0,
 107     },
 108     startup => get_standard_option('pve-startup-order'),
 109     template => {
 110         optional => 1,
 111         type => 'boolean',
 112         description => "Enable/disable Template.",
 113         default => 0,
 114     },
 115     arch => {
 116         optional => 1,
 117         type => 'string',
 118         enum => ['amd64', 'i386'],
 119         description => "OS architecture type.",
 120         default => 'amd64',
 121     },
 122     ostype => {
 123         optional => 1,
 124         type => 'string',
 125         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux', 'alpine', 'unmanaged'],
 126         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 127     },
 128     console => {
 129         optional => 1,
 130         type => 'boolean',
 131         description => "Attach a console device (/dev/console) to the container.",
 132         default => 1,
 133     },
 134     tty => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Specify the number of tty available to the container",
 138         minimum => 0,
 139         maximum => 6,
 140         default => 2,
 141     },
 142     cpulimit => {
 143         optional => 1,
 144         type => 'number',
 145         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 146         minimum => 0,
 147         maximum => 128,
 148         default => 0,
 149     },
 150     cpuunits => {
 151         optional => 1,
 152         type => 'integer',
 153         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 154         minimum => 0,
 155         maximum => 500000,
 156         default => 1024,
 157     },
 158     memory => {
 159         optional => 1,
 160         type => 'integer',
 161         description => "Amount of RAM for the VM in MB.",
 162         minimum => 16,
 163         default => 512,
 164     },
 165     swap => {
 166         optional => 1,
 167         type => 'integer',
 168         description => "Amount of SWAP for the VM in MB.",
 169         minimum => 0,
 170         default => 512,
 171     },
 172     hostname => {
 173         optional => 1,
 174         description => "Set a host name for the container.",
 175         type => 'string', format => 'dns-name',
 176         maxLength => 255,
 177     },
 178     description => {
 179         optional => 1,
 180         type => 'string',
 181         description => "Container description. Only used on the configuration web interface.",
 182     },
 183     searchdomain => {
 184         optional => 1,
 185         type => 'string', format => 'dns-name-list',
 186         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 187     },
 188     nameserver => {
 189         optional => 1,
 190         type => 'string', format => 'address-list',
 191         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 192     },
 193     rootfs => get_standard_option('pve-ct-rootfs'),
 194     parent => {
 195         optional => 1,
 196         type => 'string', format => 'pve-configid',
 197         maxLength => 40,
 198         description => "Parent snapshot name. This is used internally, and should not be modified.",
 199     },
 200     snaptime => {
 201         optional => 1,
 202         description => "Timestamp for snapshots.",
 203         type => 'integer',
 204         minimum => 0,
 205     },
 206     cmode => {
 207         optional => 1,
 208         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 209         type => 'string',
 210         enum => ['shell', 'console', 'tty'],
 211         default => 'tty',
 212     },
 213     protection => {
 214         optional => 1,
 215         type => 'boolean',
 216         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 217         default => 0,
 218     },
 219     unprivileged => {
 220         optional => 1,
 221         type => 'boolean',
 222         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 223         default => 0,
 224     },
 225 };
 226
 227 my $valid_lxc_conf_keys = {
 228     'lxc.include' => 1,
 229     'lxc.arch' => 1,
 230     'lxc.utsname' => 1,
 231     'lxc.haltsignal' => 1,
 232     'lxc.rebootsignal' => 1,
 233     'lxc.stopsignal' => 1,
 234     'lxc.init_cmd' => 1,
 235     'lxc.network.type' => 1,
 236     'lxc.network.flags' => 1,
 237     'lxc.network.link' => 1,
 238     'lxc.network.mtu' => 1,
 239     'lxc.network.name' => 1,
 240     'lxc.network.hwaddr' => 1,
 241     'lxc.network.ipv4' => 1,
 242     'lxc.network.ipv4.gateway' => 1,
 243     'lxc.network.ipv6' => 1,
 244     'lxc.network.ipv6.gateway' => 1,
 245     'lxc.network.script.up' => 1,
 246     'lxc.network.script.down' => 1,
 247     'lxc.pts' => 1,
 248     'lxc.console.logfile' => 1,
 249     'lxc.console' => 1,
 250     'lxc.tty' => 1,
 251     'lxc.devttydir' => 1,
 252     'lxc.hook.autodev' => 1,
 253     'lxc.autodev' => 1,
 254     'lxc.kmsg' => 1,
 255     'lxc.mount' => 1,
 256     'lxc.mount.entry' => 1,
 257     'lxc.mount.auto' => 1,
 258     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 259     'lxc.rootfs.mount' => 1,
 260     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 261                             ', please use mountpoint options in the "rootfs" key',
 262     # lxc.cgroup.*
 263     'lxc.cap.drop' => 1,
 264     'lxc.cap.keep' => 1,
 265     'lxc.aa_profile' => 1,
 266     'lxc.aa_allow_incomplete' => 1,
 267     'lxc.se_context' => 1,
 268     'lxc.seccomp' => 1,
 269     'lxc.id_map' => 1,
 270     'lxc.hook.pre-start' => 1,
 271     'lxc.hook.pre-mount' => 1,
 272     'lxc.hook.mount' => 1,
 273     'lxc.hook.start' => 1,
 274     'lxc.hook.stop' => 1,
 275     'lxc.hook.post-stop' => 1,
 276     'lxc.hook.clone' => 1,
 277     'lxc.hook.destroy' => 1,
 278     'lxc.loglevel' => 1,
 279     'lxc.logfile' => 1,
 280     'lxc.start.auto' => 1,
 281     'lxc.start.delay' => 1,
 282     'lxc.start.order' => 1,
 283     'lxc.group' => 1,
 284     'lxc.environment' => 1,
 285 };
 286
 287 my $netconf_desc = {
 288     type => {
 289         type => 'string',
 290         optional => 1,
 291         description => "Network interface type.",
 292         enum => [qw(veth)],
 293     },
 294     name => {
 295         type => 'string',
 296         format_description => 'String',
 297         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 298         pattern => '[-_.\w\d]+',
 299     },
 300     bridge => {
 301         type => 'string',
 302         format_description => 'vmbr<Number>',
 303         description => 'Bridge to attach the network device to.',
 304         pattern => '[-_.\w\d]+',
 305         optional => 1,
 306     },
 307     hwaddr => {
 308         type => 'string',
 309         format_description => 'MAC',
 310         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 311         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 312         optional => 1,
 313     },
 314     mtu => {
 315         type => 'integer',
 316         format_description => 'Number',
 317         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 318         minimum => 64, # minimum ethernet frame is 64 bytes
 319         optional => 1,
 320     },
 321     ip => {
 322         type => 'string',
 323         format => 'pve-ipv4-config',
 324         format_description => 'IPv4Format/CIDR',
 325         description => 'IPv4 address in CIDR format.',
 326         optional => 1,
 327     },
 328     gw => {
 329         type => 'string',
 330         format => 'ipv4',
 331         format_description => 'GatewayIPv4',
 332         description => 'Default gateway for IPv4 traffic.',
 333         optional => 1,
 334     },
 335     ip6 => {
 336         type => 'string',
 337         format => 'pve-ipv6-config',
 338         format_description => 'IPv6Format/CIDR',
 339         description => 'IPv6 address in CIDR format.',
 340         optional => 1,
 341     },
 342     gw6 => {
 343         type => 'string',
 344         format => 'ipv6',
 345         format_description => 'GatewayIPv6',
 346         description => 'Default gateway for IPv6 traffic.',
 347         optional => 1,
 348     },
 349     firewall => {
 350         type => 'boolean',
 351         format_description => '[1|0]',
 352         description => "Controls whether this interface's firewall rules should be used.",
 353         optional => 1,
 354     },
 355     tag => {
 356         type => 'integer',
 357         format_description => 'VlanNo',
 358         minimum => '2',
 359         maximum => '4094',
 360         description => "VLAN tag for this interface.",
 361         optional => 1,
 362     },
 363     trunks => {
 364         type => 'string',
 365         pattern => qr/\d+(?:;\d+)*/,
 366         format_description => 'vlanid[;vlanid...]',
 367         description => "VLAN ids to pass through the interface",
 368         optional => 1,
 369     },
 370 };
 371 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 372
 373 my $MAX_LXC_NETWORKS = 10;
 374 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 375     $confdesc->{"net$i"} = {
 376         optional => 1,
 377         type => 'string', format => $netconf_desc,
 378         description => "Specifies network interfaces for the container.",
 379     };
 380 }
 381
 382 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 383 sub verify_lxc_mp_string{
 384     my ($mp, $noerr) = @_;
 385
 386     # do not allow:
 387     # /./ or /../
 388     # /. or /.. at the end
 389     # ../ at the beginning
 390
 391     if($mp =~ m@/\.\.?/@ ||
 392        $mp =~ m@/\.\.?$@ ||
 393        $mp =~ m@^\.\./@){
 394         return undef if $noerr;
 395         die "$mp contains illegal character sequences\n";
 396     }
 397     return $mp;
 398 }
 399
 400 my $mp_desc = {
 401     %$rootfs_desc,
 402     mp => {
 403         type => 'string',
 404         format => 'pve-lxc-mp-string',
 405         format_description => 'Path',
 406         description => 'Path to the mountpoint as seen from inside the container.',
 407     },
 408 };
 409 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 410
 411 my $unuseddesc = {
 412     optional => 1,
 413     type => 'string', format => 'pve-volume-id',
 414     description => "Reference to unused volumes.",
 415 };
 416
 417 my $MAX_MOUNT_POINTS = 10;
 418 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 419     $confdesc->{"mp$i"} = {
 420         optional => 1,
 421         type => 'string', format => $mp_desc,
 422         description => "Use volume as container mount point (experimental feature).",
 423         optional => 1,
 424     };
 425 }
 426
 427 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 428 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 429     $confdesc->{"unused$i"} = $unuseddesc;
 430 }
 431
 432 sub write_pct_config {
 433     my ($filename, $conf) = @_;
 434
 435     delete $conf->{snapstate}; # just to be sure
 436     my $volidlist = get_vm_volumes($conf);
 437     my $used_volids = {};
 438     foreach my $vid (@$volidlist) {
 439         $used_volids->{$vid} = 1;
 440     }
 441
 442     # remove 'unusedX' settings if the volume is still used
 443     foreach my $key (keys %$conf) {
 444         my $value = $conf->{$key};
 445         if ($key =~ m/^unused/ && $used_volids->{$value}) {
 446             delete $conf->{$key};
 447         }
 448     }
 449
 450     my $generate_raw_config = sub {
 451         my ($conf) = @_;
 452
 453         my $raw = '';
 454
 455         # add description as comment to top of file
 456         my $descr = $conf->{description} || '';
 457         foreach my $cl (split(/\n/, $descr)) {
 458             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 459         }
 460
 461         foreach my $key (sort keys %$conf) {
 462             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 463                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 464             my $value = $conf->{$key};
 465             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 466             $raw .= "$key: $value\n";
 467         }
 468
 469         if (my $lxcconf = $conf->{lxc}) {
 470             foreach my $entry (@$lxcconf) {
 471                 my ($k, $v) = @$entry;
 472                 $raw .= "$k: $v\n";
 473             }
 474         }
 475
 476         return $raw;
 477     };
 478
 479     my $raw = &$generate_raw_config($conf);
 480
 481     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 482         $raw .= "\n[$snapname]\n";
 483         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 484     }
 485
 486     return $raw;
 487 }
 488
 489 sub check_type {
 490     my ($key, $value) = @_;
 491
 492     die "unknown setting '$key'\n" if !$confdesc->{$key};
 493
 494     my $type = $confdesc->{$key}->{type};
 495
 496     if (!defined($value)) {
 497         die "got undefined value\n";
 498     }
 499
 500     if ($value =~ m/[\n\r]/) {
 501         die "property contains a line feed\n";
 502     }
 503
 504     if ($type eq 'boolean') {
 505         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 506         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 507         die "type check ('boolean') failed - got '$value'\n";
 508     } elsif ($type eq 'integer') {
 509         return int($1) if $value =~ m/^(\d+)$/;
 510         die "type check ('integer') failed - got '$value'\n";
 511     } elsif ($type eq 'number') {
 512         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 513         die "type check ('number') failed - got '$value'\n";
 514     } elsif ($type eq 'string') {
 515         if (my $fmt = $confdesc->{$key}->{format}) {
 516             PVE::JSONSchema::check_format($fmt, $value);
 517             return $value;
 518         }
 519         return $value;
 520     } else {
 521         die "internal error"
 522     }
 523 }
 524
 525 sub parse_pct_config {
 526     my ($filename, $raw) = @_;
 527
 528     return undef if !defined($raw);
 529
 530     my $res = {
 531         digest => Digest::SHA::sha1_hex($raw),
 532         snapshots => {},
 533     };
 534
 535     $filename =~ m|/lxc/(\d+).conf$|
 536         || die "got strange filename '$filename'";
 537
 538     my $vmid = $1;
 539
 540     my $conf = $res;
 541     my $descr = '';
 542     my $section = '';
 543
 544     my @lines = split(/\n/, $raw);
 545     foreach my $line (@lines) {
 546         next if $line =~ m/^\s*$/;
 547
 548         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 549             $section = $1;
 550             $conf->{description} = $descr if $descr;
 551             $descr = '';
 552             $conf = $res->{snapshots}->{$section} = {};
 553             next;
 554         }
 555
 556         if ($line =~ m/^\#(.*)\s*$/) {
 557             $descr .= PVE::Tools::decode_text($1) . "\n";
 558             next;
 559         }
 560
 561         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 562             my $key = $1;
 563             my $value = $3;
 564             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 565             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 566                 push @{$conf->{lxc}}, [$key, $value];
 567             } elsif (my $errmsg = $validity) {
 568                 warn "vm $vmid - $key: $errmsg\n";
 569             } else {
 570                 warn "vm $vmid - unable to parse config: $line\n";
 571             }
 572         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 573             $descr .= PVE::Tools::decode_text($2);
 574         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 575             $conf->{snapstate} = $1;
 576         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 577             my $key = $1;
 578             my $value = $2;
 579             eval { $value = check_type($key, $value); };
 580             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 581             $conf->{$key} = $value;
 582         } else {
 583             warn "vm $vmid - unable to parse config: $line\n";
 584         }
 585     }
 586
 587     $conf->{description} = $descr if $descr;
 588
 589     delete $res->{snapstate}; # just to be sure
 590
 591     return $res;
 592 }
 593
 594 sub config_list {
 595     my $vmlist = PVE::Cluster::get_vmlist();
 596     my $res = {};
 597     return $res if !$vmlist || !$vmlist->{ids};
 598     my $ids = $vmlist->{ids};
 599
 600     foreach my $vmid (keys %$ids) {
 601         next if !$vmid; # skip CT0
 602         my $d = $ids->{$vmid};
 603         next if !$d->{node} || $d->{node} ne $nodename;
 604         next if !$d->{type} || $d->{type} ne 'lxc';
 605         $res->{$vmid}->{type} = 'lxc';
 606     }
 607     return $res;
 608 }
 609
 610 sub destroy_config {
 611     my ($vmid) = @_;
 612
 613     unlink PVE::LXC::Config->config_file($vmid, $nodename);
 614 }
 615
 616 sub option_exists {
 617     my ($name) = @_;
 618
 619     return defined($confdesc->{$name});
 620 }
 621
 622 # add JSON properties for create and set function
 623 sub json_config_properties {
 624     my $prop = shift;
 625
 626     foreach my $opt (keys %$confdesc) {
 627         next if $opt eq 'parent' || $opt eq 'snaptime';
 628         next if $prop->{$opt};
 629         $prop->{$opt} = $confdesc->{$opt};
 630     }
 631
 632     return $prop;
 633 }
 634
 635 # container status helpers
 636
 637 sub list_active_containers {
 638
 639     my $filename = "/proc/net/unix";
 640
 641     # similar test is used by lcxcontainers.c: list_active_containers
 642     my $res = {};
 643
 644     my $fh = IO::File->new ($filename, "r");
 645     return $res if !$fh;
 646
 647     while (defined(my $line = <$fh>)) {
 648         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 649             my $path = $1;
 650             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 651                 $res->{$1} = 1;
 652             }
 653         }
 654     }
 655
 656     close($fh);
 657
 658     return $res;
 659 }
 660
 661 # warning: this is slow
 662 sub check_running {
 663     my ($vmid) = @_;
 664
 665     my $active_hash = list_active_containers();
 666
 667     return 1 if defined($active_hash->{$vmid});
 668
 669     return undef;
 670 }
 671
 672 sub get_container_disk_usage {
 673     my ($vmid, $pid) = @_;
 674
 675     return PVE::Tools::df("/proc/$pid/root/", 1);
 676 }
 677
 678 my $last_proc_vmid_stat;
 679
 680 my $parse_cpuacct_stat = sub {
 681     my ($vmid) = @_;
 682
 683     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 684
 685     my $stat = {};
 686
 687     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 688
 689         $stat->{utime} = $1;
 690         $stat->{stime} = $2;
 691
 692     }
 693
 694     return $stat;
 695 };
 696
 697 sub vmstatus {
 698     my ($opt_vmid) = @_;
 699
 700     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 701
 702     my $active_hash = list_active_containers();
 703
 704     my $cpucount = $cpuinfo->{cpus} || 1;
 705
 706     my $cdtime = gettimeofday;
 707
 708     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 709
 710     foreach my $vmid (keys %$list) {
 711         my $d = $list->{$vmid};
 712
 713         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 714         warn $@ if $@; # ignore errors (consider them stopped)
 715
 716         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 717
 718         my $cfspath = PVE::LXC::Config->cfs_config_path($vmid);
 719         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 720
 721         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 722         $d->{name} =~ s/[\s]//g;
 723
 724         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 725
 726         $d->{lock} = $conf->{lock} || '';
 727
 728         if ($d->{pid}) {
 729             my $res = get_container_disk_usage($vmid, $d->{pid});
 730             $d->{disk} = $res->{used};
 731             $d->{maxdisk} = $res->{total};
 732         } else {
 733             $d->{disk} = 0;
 734             # use 4GB by default ??
 735             if (my $rootfs = $conf->{rootfs}) {
 736                 my $rootinfo = parse_ct_rootfs($rootfs);
 737                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 738             } else {
 739                 $d->{maxdisk} = 4*1024*1024*1024;
 740             }
 741         }
 742
 743         $d->{mem} = 0;
 744         $d->{swap} = 0;
 745         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 746         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 747
 748         $d->{uptime} = 0;
 749         $d->{cpu} = 0;
 750
 751         $d->{netout} = 0;
 752         $d->{netin} = 0;
 753
 754         $d->{diskread} = 0;
 755         $d->{diskwrite} = 0;
 756
 757         $d->{template} = PVE::LXC::Config->is_template($conf);
 758     }
 759
 760     foreach my $vmid (keys %$list) {
 761         my $d = $list->{$vmid};
 762         my $pid = $d->{pid};
 763
 764         next if !$pid; # skip stopped CTs
 765
 766         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 767         $d->{uptime} = time - $ctime; # the method lxcfs uses
 768
 769         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 770         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 771
 772         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 773         my @bytes = split(/\n/, $blkio_bytes);
 774         foreach my $byte (@bytes) {
 775             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 776                 $d->{diskread} = $2 if $key eq 'Read';
 777                 $d->{diskwrite} = $2 if $key eq 'Write';
 778             }
 779         }
 780
 781         my $pstat = &$parse_cpuacct_stat($vmid);
 782
 783         my $used = $pstat->{utime} + $pstat->{stime};
 784
 785         my $old = $last_proc_vmid_stat->{$vmid};
 786         if (!$old) {
 787             $last_proc_vmid_stat->{$vmid} = {
 788                 time => $cdtime,
 789                 used => $used,
 790                 cpu => 0,
 791             };
 792             next;
 793         }
 794
 795         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 796
 797         if ($dtime > 1000) {
 798             my $dutime = $used -  $old->{used};
 799
 800             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 801             $last_proc_vmid_stat->{$vmid} = {
 802                 time => $cdtime,
 803                 used => $used,
 804                 cpu => $d->{cpu},
 805             };
 806         } else {
 807             $d->{cpu} = $old->{cpu};
 808         }
 809     }
 810
 811     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 812
 813     foreach my $dev (keys %$netdev) {
 814         next if $dev !~ m/^veth([1-9]\d*)i/;
 815         my $vmid = $1;
 816         my $d = $list->{$vmid};
 817
 818         next if !$d;
 819
 820         $d->{netout} += $netdev->{$dev}->{receive};
 821         $d->{netin} += $netdev->{$dev}->{transmit};
 822
 823     }
 824
 825     return $list;
 826 }
 827
 828 sub classify_mountpoint {
 829     my ($vol) = @_;
 830     if ($vol =~ m!^/!) {
 831         return 'device' if $vol =~ m!^/dev/!;
 832         return 'bind';
 833     }
 834     return 'volume';
 835 }
 836
 837 my $parse_ct_mountpoint_full = sub {
 838     my ($desc, $data, $noerr) = @_;
 839
 840     $data //= '';
 841
 842     my $res;
 843     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 844     if ($@) {
 845         return undef if $noerr;
 846         die $@;
 847     }
 848
 849     if (defined(my $size = $res->{size})) {
 850         $size = PVE::JSONSchema::parse_size($size);
 851         if (!defined($size)) {
 852             return undef if $noerr;
 853             die "invalid size: $size\n";
 854         }
 855         $res->{size} = $size;
 856     }
 857
 858     $res->{type} = classify_mountpoint($res->{volume});
 859
 860     return $res;
 861 };
 862
 863 sub parse_ct_rootfs {
 864     my ($data, $noerr) = @_;
 865
 866     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 867
 868     $res->{mp} = '/' if defined($res);
 869
 870     return $res;
 871 }
 872
 873 sub parse_ct_mountpoint {
 874     my ($data, $noerr) = @_;
 875
 876     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 877 }
 878
 879 sub print_ct_mountpoint {
 880     my ($info, $nomp) = @_;
 881     my $skip = [ 'type' ];
 882     push @$skip, 'mp' if $nomp;
 883     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 884 }
 885
 886 sub print_lxc_network {
 887     my $net = shift;
 888     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 889 }
 890
 891 sub parse_lxc_network {
 892     my ($data) = @_;
 893
 894     my $res = {};
 895
 896     return $res if !$data;
 897
 898     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 899
 900     $res->{type} = 'veth';
 901     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 902
 903     return $res;
 904 }
 905
 906 sub read_cgroup_value {
 907     my ($group, $vmid, $name, $full) = @_;
 908
 909     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 910
 911     return PVE::Tools::file_get_contents($path) if $full;
 912
 913     return PVE::Tools::file_read_firstline($path);
 914 }
 915
 916 sub write_cgroup_value {
 917    my ($group, $vmid, $name, $value) = @_;
 918
 919    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 920    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 921
 922 }
 923
 924 sub find_lxc_console_pids {
 925
 926     my $res = {};
 927
 928     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 929         my ($pid) = @_;
 930
 931         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 932         return if !$cmdline;
 933
 934         my @args = split(/\0/, $cmdline);
 935
 936         # search for lxc-console -n <vmid>
 937         return if scalar(@args) != 3;
 938         return if $args[1] ne '-n';
 939         return if $args[2] !~ m/^\d+$/;
 940         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 941
 942         my $vmid = $args[2];
 943
 944         push @{$res->{$vmid}}, $pid;
 945     });
 946
 947     return $res;
 948 }
 949
 950 sub find_lxc_pid {
 951     my ($vmid) = @_;
 952
 953     my $pid = undef;
 954     my $parser = sub {
 955         my $line = shift;
 956         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 957     };
 958     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
 959
 960     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 961
 962     return $pid;
 963 }
 964
 965 # Note: we cannot use Net:IP, because that only allows strict
 966 # CIDR networks
 967 sub parse_ipv4_cidr {
 968     my ($cidr, $noerr) = @_;
 969
 970     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
 971         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
 972     }
 973
 974     return undef if $noerr;
 975
 976     die "unable to parse ipv4 address/mask\n";
 977 }
 978
 979
 980 sub update_lxc_config {
 981     my ($storage_cfg, $vmid, $conf) = @_;
 982
 983     my $dir = "/var/lib/lxc/$vmid";
 984
 985     if ($conf->{template}) {
 986
 987         unlink "$dir/config";
 988
 989         return;
 990     }
 991
 992     my $raw = '';
 993
 994     die "missing 'arch' - internal error" if !$conf->{arch};
 995     $raw .= "lxc.arch = $conf->{arch}\n";
 996
 997     my $unprivileged = $conf->{unprivileged};
 998     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
 999
1000     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1001     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux | alpine | unmanaged)$/x) {
1002         my $inc ="/usr/share/lxc/config/$ostype.common.conf";
1003         $inc ="/usr/share/lxc/config/common.conf" if !-f $inc;
1004         $raw .= "lxc.include = $inc\n";
1005         if ($unprivileged || $custom_idmap) {
1006             $inc = "/usr/share/lxc/config/$ostype.userns.conf";
1007             $inc = "/usr/share/lxc/config/userns.conf" if !-f $inc;
1008             $raw .= "lxc.include = $inc\n"
1009         }
1010     } else {
1011         die "implement me (ostype $ostype)";
1012     }
1013
1014     # WARNING: DO NOT REMOVE this without making sure that loop device nodes
1015     # cannot be exposed to the container with r/w access (cgroup perms).
1016     # When this is enabled mounts will still remain in the monitor's namespace
1017     # after the container unmounted them and thus will not detach from their
1018     # files while the container is running!
1019     $raw .= "lxc.monitor.unshare = 1\n";
1020
1021     # Should we read them from /etc/subuid?
1022     if ($unprivileged && !$custom_idmap) {
1023         $raw .= "lxc.id_map = u 0 100000 65536\n";
1024         $raw .= "lxc.id_map = g 0 100000 65536\n";
1025     }
1026
1027     if (!has_dev_console($conf)) {
1028         $raw .= "lxc.console = none\n";
1029         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1030     }
1031
1032     my $ttycount = get_tty_count($conf);
1033     $raw .= "lxc.tty = $ttycount\n";
1034
1035     # some init scripts expect a linux terminal (turnkey).
1036     $raw .= "lxc.environment = TERM=linux\n";
1037
1038     my $utsname = $conf->{hostname} || "CT$vmid";
1039     $raw .= "lxc.utsname = $utsname\n";
1040
1041     my $memory = $conf->{memory} || 512;
1042     my $swap = $conf->{swap} // 0;
1043
1044     my $lxcmem = int($memory*1024*1024);
1045     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1046
1047     my $lxcswap = int(($memory + $swap)*1024*1024);
1048     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1049
1050     if (my $cpulimit = $conf->{cpulimit}) {
1051         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1052         my $value = int(100000*$cpulimit);
1053         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1054     }
1055
1056     my $shares = $conf->{cpuunits} || 1024;
1057     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1058
1059     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1060
1061     $raw .= "lxc.rootfs = $dir/rootfs\n";
1062
1063     my $netcount = 0;
1064     foreach my $k (keys %$conf) {
1065         next if $k !~ m/^net(\d+)$/;
1066         my $ind = $1;
1067         my $d = parse_lxc_network($conf->{$k});
1068         $netcount++;
1069         $raw .= "lxc.network.type = veth\n";
1070         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1071         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1072         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1073         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1074     }
1075
1076     if (my $lxcconf = $conf->{lxc}) {
1077         foreach my $entry (@$lxcconf) {
1078             my ($k, $v) = @$entry;
1079             $netcount++ if $k eq 'lxc.network.type';
1080             $raw .= "$k = $v\n";
1081         }
1082     }
1083
1084     $raw .= "lxc.network.type = empty\n" if !$netcount;
1085
1086     File::Path::mkpath("$dir/rootfs");
1087
1088     PVE::Tools::file_set_contents("$dir/config", $raw);
1089 }
1090
1091 # verify and cleanup nameserver list (replace \0 with ' ')
1092 sub verify_nameserver_list {
1093     my ($nameserver_list) = @_;
1094
1095     my @list = ();
1096     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1097         PVE::JSONSchema::pve_verify_ip($server);
1098         push @list, $server;
1099     }
1100
1101     return join(' ', @list);
1102 }
1103
1104 sub verify_searchdomain_list {
1105     my ($searchdomain_list) = @_;
1106
1107     my @list = ();
1108     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1109         # todo: should we add checks for valid dns domains?
1110         push @list, $server;
1111     }
1112
1113     return join(' ', @list);
1114 }
1115
1116 sub is_volume_in_use {
1117     my ($config, $volid, $include_snapshots) = @_;
1118     my $used = 0;
1119
1120     foreach_mountpoint($config, sub {
1121         my ($ms, $mountpoint) = @_;
1122         return if $used;
1123         if ($mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid) {
1124             $used = 1;
1125         }
1126     });
1127
1128     my $snapshots = $config->{snapshots};
1129     if ($include_snapshots && $snapshots) {
1130         foreach my $snap (keys %$snapshots) {
1131             $used ||= is_volume_in_use($snapshots->{$snap}, $volid);
1132         }
1133     }
1134
1135     return $used;
1136 }
1137
1138 sub add_unused_volume {
1139     my ($config, $volid) = @_;
1140
1141     my $key;
1142     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1143         my $test = "unused$ind";
1144         if (my $vid = $config->{$test}) {
1145             return if $vid eq $volid; # do not add duplicates
1146         } else {
1147             $key = $test;
1148         }
1149     }
1150
1151     die "Too many unused volumes - please delete them first.\n" if !$key;
1152
1153     $config->{$key} = $volid;
1154
1155     return $key;
1156 }
1157
1158 sub update_pct_config {
1159     my ($vmid, $conf, $running, $param, $delete) = @_;
1160
1161     my @nohotplug;
1162
1163     my $new_disks = 0;
1164     my @deleted_volumes;
1165
1166     my $rootdir;
1167     if ($running) {
1168         my $pid = find_lxc_pid($vmid);
1169         $rootdir = "/proc/$pid/root";
1170     }
1171
1172     my $hotplug_error = sub {
1173         if ($running) {
1174             push @nohotplug, @_;
1175             return 1;
1176         } else {
1177             return 0;
1178         }
1179     };
1180
1181     if (defined($delete)) {
1182         foreach my $opt (@$delete) {
1183             if (!exists($conf->{$opt})) {
1184                 warn "no such option: $opt\n";
1185                 next;
1186             }
1187
1188             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1189                 die "unable to delete required option '$opt'\n";
1190             } elsif ($opt eq 'swap') {
1191                 delete $conf->{$opt};
1192                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1193             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1194                 delete $conf->{$opt};
1195             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1196                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1197                 next if $hotplug_error->($opt);
1198                 delete $conf->{$opt};
1199             } elsif ($opt =~ m/^net(\d)$/) {
1200                 delete $conf->{$opt};
1201                 next if !$running;
1202                 my $netid = $1;
1203                 PVE::Network::veth_delete("veth${vmid}i$netid");
1204             } elsif ($opt eq 'protection') {
1205                 delete $conf->{$opt};
1206             } elsif ($opt =~ m/^unused(\d+)$/) {
1207                 next if $hotplug_error->($opt);
1208                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1209                 push @deleted_volumes, $conf->{$opt};
1210                 delete $conf->{$opt};
1211             } elsif ($opt =~ m/^mp(\d+)$/) {
1212                 next if $hotplug_error->($opt);
1213                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1214                 my $mp = parse_ct_mountpoint($conf->{$opt});
1215                 delete $conf->{$opt};
1216                 if ($mp->{type} eq 'volume') {
1217                     add_unused_volume($conf, $mp->{volume});
1218                 }
1219             } elsif ($opt eq 'unprivileged') {
1220                 die "unable to delete read-only option: '$opt'\n";
1221             } else {
1222                 die "implement me (delete: $opt)"
1223             }
1224             PVE::LXC::Config->write_config($vmid, $conf) if $running;
1225         }
1226     }
1227
1228     # There's no separate swap size to configure, there's memory and "total"
1229     # memory (iow. memory+swap). This means we have to change them together.
1230     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1231     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1232     if (defined($wanted_memory) || defined($wanted_swap)) {
1233
1234         my $old_memory = ($conf->{memory} || 512);
1235         my $old_swap = ($conf->{swap} || 0);
1236
1237         $wanted_memory //= $old_memory;
1238         $wanted_swap //= $old_swap;
1239
1240         my $total = $wanted_memory + $wanted_swap;
1241         if ($running) {
1242             my $old_total = $old_memory + $old_swap;
1243             if ($total > $old_total) {
1244                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1245                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1246             } else {
1247                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1248                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1249             }
1250         }
1251         $conf->{memory} = $wanted_memory;
1252         $conf->{swap} = $wanted_swap;
1253
1254         PVE::LXC::Config->write_config($vmid, $conf) if $running;
1255     }
1256
1257     my $used_volids = {};
1258
1259     foreach my $opt (keys %$param) {
1260         my $value = $param->{$opt};
1261         if ($opt eq 'hostname') {
1262             $conf->{$opt} = $value;
1263         } elsif ($opt eq 'onboot') {
1264             $conf->{$opt} = $value ? 1 : 0;
1265         } elsif ($opt eq 'startup') {
1266             $conf->{$opt} = $value;
1267         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1268             next if $hotplug_error->($opt);
1269             $conf->{$opt} = $value;
1270         } elsif ($opt eq 'nameserver') {
1271             next if $hotplug_error->($opt);
1272             my $list = verify_nameserver_list($value);
1273             $conf->{$opt} = $list;
1274         } elsif ($opt eq 'searchdomain') {
1275             next if $hotplug_error->($opt);
1276             my $list = verify_searchdomain_list($value);
1277             $conf->{$opt} = $list;
1278         } elsif ($opt eq 'cpulimit') {
1279             next if $hotplug_error->($opt); # FIXME: hotplug
1280             $conf->{$opt} = $value;
1281         } elsif ($opt eq 'cpuunits') {
1282             $conf->{$opt} = $value;
1283             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1284         } elsif ($opt eq 'description') {
1285             $conf->{$opt} = PVE::Tools::encode_text($value);
1286         } elsif ($opt =~ m/^net(\d+)$/) {
1287             my $netid = $1;
1288             my $net = parse_lxc_network($value);
1289             if (!$running) {
1290                 $conf->{$opt} = print_lxc_network($net);
1291             } else {
1292                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1293             }
1294         } elsif ($opt eq 'protection') {
1295             $conf->{$opt} = $value ? 1 : 0;
1296         } elsif ($opt =~ m/^mp(\d+)$/) {
1297             next if $hotplug_error->($opt);
1298             check_protection($conf, "can't update CT $vmid drive '$opt'");
1299             my $old = $conf->{$opt};
1300             $conf->{$opt} = $value;
1301             if (defined($old)) {
1302                 my $mp = parse_ct_mountpoint($old);
1303                 if ($mp->{type} eq 'volume') {
1304                     add_unused_volume($conf, $mp->{volume});
1305                 }
1306             }
1307             $new_disks = 1;
1308             my $mp = parse_ct_mountpoint($value);
1309             $used_volids->{$mp->{volume}} = 1;
1310         } elsif ($opt eq 'rootfs') {
1311             next if $hotplug_error->($opt);
1312             check_protection($conf, "can't update CT $vmid drive '$opt'");
1313             my $old = $conf->{$opt};
1314             $conf->{$opt} = $value;
1315             if (defined($old)) {
1316                 my $mp = parse_ct_rootfs($old);
1317                 if ($mp->{type} eq 'volume') {
1318                     add_unused_volume($conf, $mp->{volume});
1319                 }
1320             }
1321             my $mp = parse_ct_rootfs($value);
1322             $used_volids->{$mp->{volume}} = 1;
1323         } elsif ($opt eq 'unprivileged') {
1324             die "unable to modify read-only option: '$opt'\n";
1325         } elsif ($opt eq 'ostype') {
1326             next if $hotplug_error->($opt);
1327             $conf->{$opt} = $value;
1328         } else {
1329             die "implement me: $opt";
1330         }
1331         PVE::LXC::Config->write_config($vmid, $conf) if $running;
1332     }
1333
1334     # Apply deletions and creations of new volumes
1335     if (@deleted_volumes) {
1336         my $storage_cfg = PVE::Storage::config();
1337         foreach my $volume (@deleted_volumes) {
1338             next if $used_volids->{$volume}; # could have been re-added, too
1339             # also check for references in snapshots
1340             next if is_volume_in_use($conf, $volume, 1);
1341             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1342         }
1343     }
1344
1345     if ($new_disks) {
1346         my $storage_cfg = PVE::Storage::config();
1347         create_disks($storage_cfg, $vmid, $conf, $conf);
1348     }
1349
1350     # This should be the last thing we do here
1351     if ($running && scalar(@nohotplug)) {
1352         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1353     }
1354 }
1355
1356 sub has_dev_console {
1357     my ($conf) = @_;
1358
1359     return !(defined($conf->{console}) && !$conf->{console});
1360 }
1361
1362 sub get_tty_count {
1363     my ($conf) = @_;
1364
1365     return $conf->{tty} // $confdesc->{tty}->{default};
1366 }
1367
1368 sub get_cmode {
1369     my ($conf) = @_;
1370
1371     return $conf->{cmode} // $confdesc->{cmode}->{default};
1372 }
1373
1374 sub get_console_command {
1375     my ($vmid, $conf) = @_;
1376
1377     my $cmode = get_cmode($conf);
1378
1379     if ($cmode eq 'console') {
1380         return ['lxc-console', '-n',  $vmid, '-t', 0];
1381     } elsif ($cmode eq 'tty') {
1382         return ['lxc-console', '-n',  $vmid];
1383     } elsif ($cmode eq 'shell') {
1384         return ['lxc-attach', '--clear-env', '-n', $vmid];
1385     } else {
1386         die "internal error";
1387     }
1388 }
1389
1390 sub get_primary_ips {
1391     my ($conf) = @_;
1392
1393     # return data from net0
1394
1395     return undef if !defined($conf->{net0});
1396     my $net = parse_lxc_network($conf->{net0});
1397
1398     my $ipv4 = $net->{ip};
1399     if ($ipv4) {
1400         if ($ipv4 =~ /^(dhcp|manual)$/) {
1401             $ipv4 = undef
1402         } else {
1403             $ipv4 =~ s!/\d+$!!;
1404         }
1405     }
1406     my $ipv6 = $net->{ip6};
1407     if ($ipv6) {
1408         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1409             $ipv6 = undef;
1410         } else {
1411             $ipv6 =~ s!/\d+$!!;
1412         }
1413     }
1414
1415     return ($ipv4, $ipv6);
1416 }
1417
1418 sub delete_mountpoint_volume {
1419     my ($storage_cfg, $vmid, $volume) = @_;
1420
1421     return if classify_mountpoint($volume) ne 'volume';
1422
1423     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1424     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1425 }
1426
1427 sub destroy_lxc_container {
1428     my ($storage_cfg, $vmid, $conf) = @_;
1429
1430     foreach_mountpoint($conf, sub {
1431         my ($ms, $mountpoint) = @_;
1432         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1433     });
1434
1435     rmdir "/var/lib/lxc/$vmid/rootfs";
1436     unlink "/var/lib/lxc/$vmid/config";
1437     rmdir "/var/lib/lxc/$vmid";
1438     destroy_config($vmid);
1439
1440     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1441     #PVE::Tools::run_command($cmd);
1442 }
1443
1444 sub vm_stop_cleanup {
1445     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1446
1447     eval {
1448         if (!$keepActive) {
1449
1450             my $vollist = get_vm_volumes($conf);
1451             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1452         }
1453     };
1454     warn $@ if $@; # avoid errors - just warn
1455 }
1456
1457 my $safe_num_ne = sub {
1458     my ($a, $b) = @_;
1459
1460     return 0 if !defined($a) && !defined($b);
1461     return 1 if !defined($a);
1462     return 1 if !defined($b);
1463
1464     return $a != $b;
1465 };
1466
1467 my $safe_string_ne = sub {
1468     my ($a, $b) = @_;
1469
1470     return 0 if !defined($a) && !defined($b);
1471     return 1 if !defined($a);
1472     return 1 if !defined($b);
1473
1474     return $a ne $b;
1475 };
1476
1477 sub update_net {
1478     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1479
1480     if ($newnet->{type} ne 'veth') {
1481         # for when there are physical interfaces
1482         die "cannot update interface of type $newnet->{type}";
1483     }
1484
1485     my $veth = "veth${vmid}i${netid}";
1486     my $eth = $newnet->{name};
1487
1488     if (my $oldnetcfg = $conf->{$opt}) {
1489         my $oldnet = parse_lxc_network($oldnetcfg);
1490
1491         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1492             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1493
1494             PVE::Network::veth_delete($veth);
1495             delete $conf->{$opt};
1496             PVE::LXC::Config->write_config($vmid, $conf);
1497
1498             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1499
1500         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1501                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1502                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1503
1504                 if ($oldnet->{bridge}) {
1505                     PVE::Network::tap_unplug($veth);
1506                     foreach (qw(bridge tag firewall)) {
1507                         delete $oldnet->{$_};
1508                     }
1509                     $conf->{$opt} = print_lxc_network($oldnet);
1510                     PVE::LXC::Config->write_config($vmid, $conf);
1511                 }
1512
1513                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1514                 foreach (qw(bridge tag firewall)) {
1515                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1516                 }
1517                 $conf->{$opt} = print_lxc_network($oldnet);
1518                 PVE::LXC::Config->write_config($vmid, $conf);
1519         }
1520     } else {
1521         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1522     }
1523
1524     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1525 }
1526
1527 sub hotplug_net {
1528     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1529
1530     my $veth = "veth${vmid}i${netid}";
1531     my $vethpeer = $veth . "p";
1532     my $eth = $newnet->{name};
1533
1534     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1535     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1536
1537     # attach peer in container
1538     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1539     PVE::Tools::run_command($cmd);
1540
1541     # link up peer in container
1542     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1543     PVE::Tools::run_command($cmd);
1544
1545     my $done = { type => 'veth' };
1546     foreach (qw(bridge tag firewall hwaddr name)) {
1547         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1548     }
1549     $conf->{$opt} = print_lxc_network($done);
1550
1551     PVE::LXC::Config->write_config($vmid, $conf);
1552 }
1553
1554 sub update_ipconfig {
1555     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1556
1557     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1558
1559     my $optdata = parse_lxc_network($conf->{$opt});
1560     my $deleted = [];
1561     my $added = [];
1562     my $nscmd = sub {
1563         my $cmdargs = shift;
1564         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1565     };
1566     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1567
1568     my $change_ip_config = sub {
1569         my ($ipversion) = @_;
1570
1571         my $family_opt = "-$ipversion";
1572         my $suffix = $ipversion == 4 ? '' : $ipversion;
1573         my $gw= "gw$suffix";
1574         my $ip= "ip$suffix";
1575
1576         my $newip = $newnet->{$ip};
1577         my $newgw = $newnet->{$gw};
1578         my $oldip = $optdata->{$ip};
1579
1580         my $change_ip = &$safe_string_ne($oldip, $newip);
1581         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1582
1583         return if !$change_ip && !$change_gw;
1584
1585         # step 1: add new IP, if this fails we cancel
1586         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1587         if ($change_ip && $is_real_ip) {
1588             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1589             if (my $err = $@) {
1590                 warn $err;
1591                 return;
1592             }
1593         }
1594
1595         # step 2: replace gateway
1596         #   If this fails we delete the added IP and cancel.
1597         #   If it succeeds we save the config and delete the old IP, ignoring
1598         #   errors. The config is then saved.
1599         # Note: 'ip route replace' can add
1600         if ($change_gw) {
1601             if ($newgw) {
1602                 eval {
1603                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1604                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1605                     }
1606                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1607                 };
1608                 if (my $err = $@) {
1609                     warn $err;
1610                     # the route was not replaced, the old IP is still available
1611                     # rollback (delete new IP) and cancel
1612                     if ($change_ip) {
1613                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1614                         warn $@ if $@; # no need to die here
1615                     }
1616                     return;
1617                 }
1618             } else {
1619                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1620                 # if the route was not deleted, the guest might have deleted it manually
1621                 # warn and continue
1622                 warn $@ if $@;
1623             }
1624         }
1625
1626         # from this point on we save the configuration
1627         # step 3: delete old IP ignoring errors
1628         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1629             # We need to enable promote_secondaries, otherwise our newly added
1630             # address will be removed along with the old one.
1631             my $promote = 0;
1632             eval {
1633                 if ($ipversion == 4) {
1634                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1635                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1636                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1637                 }
1638                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1639             };
1640             warn $@ if $@; # no need to die here
1641
1642             if ($ipversion == 4) {
1643                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1644             }
1645         }
1646
1647         foreach my $property ($ip, $gw) {
1648             if ($newnet->{$property}) {
1649                 $optdata->{$property} = $newnet->{$property};
1650             } else {
1651                 delete $optdata->{$property};
1652             }
1653         }
1654         $conf->{$opt} = print_lxc_network($optdata);
1655         PVE::LXC::Config->write_config($vmid, $conf);
1656         $lxc_setup->setup_network($conf);
1657     };
1658
1659     &$change_ip_config(4);
1660     &$change_ip_config(6);
1661
1662 }
1663
1664 # Internal snapshots
1665
1666 # NOTE: Snapshot create/delete involves several non-atomic
1667 # actions, and can take a long time.
1668 # So we try to avoid locking the file and use the 'lock' variable
1669 # inside the config file instead.
1670
1671 my $snapshot_copy_config = sub {
1672     my ($source, $dest) = @_;
1673
1674     foreach my $k (keys %$source) {
1675         next if $k eq 'snapshots';
1676         next if $k eq 'snapstate';
1677         next if $k eq 'snaptime';
1678         next if $k eq 'vmstate';
1679         next if $k eq 'lock';
1680         next if $k eq 'digest';
1681         next if $k eq 'description';
1682         next if $k =~ m/^unused\d+$/;
1683
1684         $dest->{$k} = $source->{$k};
1685     }
1686 };
1687
1688 my $snapshot_apply_config = sub {
1689     my ($conf, $snap) = @_;
1690
1691     # copy snapshot list
1692     my $newconf = {
1693         snapshots => $conf->{snapshots},
1694     };
1695
1696     # keep description and list of unused disks
1697     foreach my $k (keys %$conf) {
1698         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
1699         $newconf->{$k} = $conf->{$k};
1700     }
1701
1702     &$snapshot_copy_config($snap, $newconf);
1703
1704     return $newconf;
1705 };
1706
1707 sub snapshot_save_vmstate {
1708     die "implement me - snapshot_save_vmstate\n";
1709 }
1710
1711 sub snapshot_prepare {
1712     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1713
1714     my $snap;
1715
1716     my $updatefn =  sub {
1717
1718         my $conf = PVE::LXC::Config->load_config($vmid);
1719
1720         die "you can't take a snapshot if it's a template\n"
1721             if PVE::LXC::Config->is_template($conf);
1722
1723         PVE::LXC::Config->check_lock($conf);
1724
1725         $conf->{lock} = 'snapshot';
1726
1727         die "snapshot name '$snapname' already used\n"
1728             if defined($conf->{snapshots}->{$snapname});
1729
1730         my $storecfg = PVE::Storage::config();
1731         die "snapshot feature is not available\n"
1732             if !has_feature('snapshot', $conf, $storecfg, undef, undef, $snapname eq 'vzdump');
1733
1734         $snap = $conf->{snapshots}->{$snapname} = {};
1735
1736         if ($save_vmstate && check_running($vmid)) {
1737             snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
1738         }
1739
1740         &$snapshot_copy_config($conf, $snap);
1741
1742         $snap->{snapstate} = "prepare";
1743         $snap->{snaptime} = time();
1744         $snap->{description} = $comment if $comment;
1745
1746         PVE::LXC::Config->write_config($vmid, $conf);
1747     };
1748
1749     PVE::LXC::Config->lock_config($vmid, $updatefn);
1750
1751     return $snap;
1752 }
1753
1754 sub snapshot_commit {
1755     my ($vmid, $snapname) = @_;
1756
1757     my $updatefn = sub {
1758
1759         my $conf = PVE::LXC::Config->load_config($vmid);
1760
1761         die "missing snapshot lock\n"
1762             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1763
1764         my $snap = $conf->{snapshots}->{$snapname};
1765         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1766
1767         die "wrong snapshot state\n"
1768             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
1769
1770         delete $snap->{snapstate};
1771         delete $conf->{lock};
1772
1773         $conf->{parent} = $snapname;
1774
1775         PVE::LXC::Config->write_config($vmid, $conf);
1776     };
1777
1778     PVE::LXC::Config->lock_config($vmid, $updatefn);
1779 }
1780
1781 sub has_feature {
1782     my ($feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
1783
1784     my $err;
1785
1786     foreach_mountpoint($conf, sub {
1787         my ($ms, $mountpoint) = @_;
1788
1789         return if $err; # skip further test
1790         return if $backup_only && $ms ne 'rootfs' && !$mountpoint->{backup};
1791
1792         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname, $running);
1793     });
1794
1795     return $err ? 0 : 1;
1796 }
1797
1798 my $enter_namespace = sub {
1799     my ($vmid, $pid, $which, $type) = @_;
1800     sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY
1801         or die "failed to open $which namespace of container $vmid: $!\n";
1802     PVE::Tools::setns(fileno($fd), $type)
1803         or die "failed to enter $which namespace of container $vmid: $!\n";
1804     close $fd;
1805 };
1806
1807 my $do_syncfs = sub {
1808     my ($vmid, $pid, $socket) = @_;
1809
1810     &$enter_namespace($vmid, $pid, 'mnt', PVE::Tools::CLONE_NEWNS);
1811
1812     # Tell the parent process to start reading our /proc/mounts
1813     print {$socket} "go\n";
1814     $socket->flush();
1815
1816     # Receive /proc/self/mounts
1817     my $mountdata = do { local $/ = undef; <$socket> };
1818     close $socket;
1819
1820     # Now sync all mountpoints...
1821     my $mounts = PVE::ProcFSTools::parse_mounts($mountdata);
1822     foreach my $mp (@$mounts) {
1823         my ($what, $dir, $fs) = @$mp;
1824         next if $fs eq 'fuse.lxcfs';
1825         eval { PVE::Tools::sync_mountpoint($dir); };
1826         warn $@ if $@;
1827     }
1828 };
1829
1830 sub sync_container_namespace {
1831     my ($vmid) = @_;
1832     my $pid = find_lxc_pid($vmid);
1833
1834     # SOCK_DGRAM is nicer for barriers but cannot be slurped
1835     socketpair my $pfd, my $cfd, AF_UNIX, SOCK_STREAM, PF_UNSPEC
1836         or die "failed to create socketpair: $!\n";
1837
1838     my $child = fork();
1839     die "fork failed: $!\n" if !defined($child);
1840
1841     if (!$child) {
1842         eval {
1843             close $pfd;
1844             &$do_syncfs($vmid, $pid, $cfd);
1845         };
1846         if (my $err = $@) {
1847             warn $err;
1848             POSIX::_exit(1);
1849         }
1850         POSIX::_exit(0);
1851     }
1852     close $cfd;
1853     my $go = <$pfd>;
1854     die "failed to enter container namespace\n" if $go ne "go\n";
1855
1856     open my $mounts, '<', "/proc/$child/mounts"
1857         or die "failed to open container's /proc/mounts: $!\n";
1858     my $mountdata = do { local $/ = undef; <$mounts> };
1859     close $mounts;
1860     print {$pfd} $mountdata;
1861     close $pfd;
1862
1863     while (waitpid($child, 0) != $child) {}
1864     die "failed to sync container namespace\n" if $? != 0;
1865 }
1866
1867 sub check_freeze_needed {
1868     my ($vmid, $config, $save_vmstate) = @_;
1869
1870     my $ret = check_running($vmid);
1871     return ($ret, $ret);
1872 }
1873
1874 sub snapshot_create {
1875     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1876
1877     my $snap = snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
1878
1879     $save_vmstate = 0 if !$snap->{vmstate};
1880
1881     my $conf = PVE::LXC::Config->load_config($vmid);
1882
1883     my ($running, $freezefs) = check_freeze_needed($vmid, $conf, $snap->{vmstate});
1884
1885     my $drivehash = {};
1886
1887     eval {
1888         if ($freezefs) {
1889             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1890             sync_container_namespace($vmid);
1891         }
1892
1893         my $storecfg = PVE::Storage::config();
1894         foreach_mountpoint($conf, sub {
1895             my ($ms, $mountpoint) = @_;
1896
1897             return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
1898             PVE::Storage::volume_snapshot($storecfg, $mountpoint->{volume}, $snapname);
1899             $drivehash->{$ms} = 1;
1900         });
1901     };
1902     my $err = $@;
1903
1904     if ($running) {
1905         if ($freezefs) {
1906             eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
1907             warn $@ if $@;
1908         }
1909     }
1910
1911     if ($err) {
1912         warn "snapshot create failed: starting cleanup\n";
1913         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
1914         warn "$@" if $@;
1915         die "$err\n";
1916     }
1917
1918     snapshot_commit($vmid, $snapname);
1919 }
1920
1921 # Note: $drivehash is only set when called from snapshot_create.
1922 sub snapshot_delete {
1923     my ($vmid, $snapname, $force, $drivehash) = @_;
1924
1925     my $prepare = 1;
1926
1927     my $snap;
1928     my $unused = [];
1929
1930     my $unlink_parent = sub {
1931         my ($confref, $new_parent) = @_;
1932
1933         if ($confref->{parent} && $confref->{parent} eq $snapname) {
1934             if ($new_parent) {
1935                 $confref->{parent} = $new_parent;
1936             } else {
1937                 delete $confref->{parent};
1938             }
1939         }
1940     };
1941
1942     my $updatefn =  sub {
1943         my ($remove_drive) = @_;
1944
1945         my $conf = PVE::LXC::Config->load_config($vmid);
1946
1947         if (!$drivehash) {
1948             PVE::LXC::Config->check_lock($conf);
1949             die "you can't delete a snapshot if vm is a template\n"
1950                 if PVE::LXC::Config->is_template($conf);
1951         }
1952
1953         $snap = $conf->{snapshots}->{$snapname};
1954
1955         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1956
1957         # remove parent refs
1958         if (!$prepare) {
1959             &$unlink_parent($conf, $snap->{parent});
1960             foreach my $sn (keys %{$conf->{snapshots}}) {
1961                 next if $sn eq $snapname;
1962                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
1963             }
1964         }
1965
1966         if ($remove_drive) {
1967             if ($remove_drive eq 'vmstate') {
1968                 die "implement me - saving vmstate\n";
1969             } else {
1970                 my $value = $snap->{$remove_drive};
1971                 my $mountpoint = $remove_drive eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
1972                 delete $snap->{$remove_drive};
1973                 add_unused_volume($snap, $mountpoint->{volume});
1974             }
1975         }
1976
1977         if ($prepare) {
1978             $snap->{snapstate} = 'delete';
1979         } else {
1980             delete $conf->{snapshots}->{$snapname};
1981             delete $conf->{lock} if $drivehash;
1982             foreach my $volid (@$unused) {
1983                 add_unused_volume($conf, $volid);
1984             }
1985         }
1986
1987         PVE::LXC::Config->write_config($vmid, $conf);
1988     };
1989
1990     PVE::LXC::Config->lock_config($vmid, $updatefn);
1991
1992     # now remove vmstate file
1993     # never set for LXC!
1994     my $storecfg = PVE::Storage::config();
1995
1996     if ($snap->{vmstate}) {
1997         die "implement me - saving vmstate\n";
1998     };
1999
2000     # now remove all volume snapshots
2001     foreach_mountpoint($snap, sub {
2002         my ($ms, $mountpoint) = @_;
2003
2004         return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
2005         if (!$drivehash || $drivehash->{$ms}) {
2006             eval { PVE::Storage::volume_snapshot_delete($storecfg, $mountpoint->{volume}, $snapname); };
2007             if (my $err = $@) {
2008                 die $err if !$force;
2009                 warn $err;
2010             }
2011         }
2012
2013         # save changes (remove mp from snapshot)
2014         PVE::LXC::Config->lock_config($vmid, $updatefn, $ms) if !$force;
2015         push @$unused, $mountpoint->{volume};
2016     });
2017
2018     # now cleanup config
2019     $prepare = 0;
2020     PVE::LXC::Config->lock_config($vmid, $updatefn);
2021 }
2022
2023 sub snapshot_rollback {
2024     my ($vmid, $snapname) = @_;
2025
2026     my $prepare = 1;
2027
2028     my $storecfg = PVE::Storage::config();
2029
2030     my $conf = PVE::LXC::Config->load_config($vmid);
2031
2032     my $get_snapshot_config = sub {
2033
2034         die "you can't rollback if vm is a template\n" if PVE::LXC::Config->is_template($conf);
2035
2036         my $res = $conf->{snapshots}->{$snapname};
2037
2038         die "snapshot '$snapname' does not exist\n" if !defined($res);
2039
2040         return $res;
2041     };
2042
2043     my $snap = &$get_snapshot_config();
2044
2045     foreach_mountpoint($snap, sub {
2046         my ($ms, $mountpoint) = @_;
2047
2048         PVE::Storage::volume_rollback_is_possible($storecfg, $mountpoint->{volume}, $snapname);
2049     });
2050
2051     my $updatefn = sub {
2052
2053         $conf = PVE::LXC::Config->load_config($vmid);
2054
2055         $snap = &$get_snapshot_config();
2056
2057         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
2058             if $snap->{snapstate};
2059
2060         if ($prepare) {
2061             PVE::LXC::Config->check_lock($conf);
2062             PVE::Tools::run_command(['/usr/bin/lxc-stop', '-n', $vmid, '--kill'])
2063                 if check_running($vmid);
2064         }
2065
2066         die "unable to rollback vm $vmid: vm is running\n"
2067             if check_running($vmid);
2068
2069         if ($prepare) {
2070             $conf->{lock} = 'rollback';
2071         } else {
2072             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
2073             delete $conf->{lock};
2074         }
2075
2076         my $forcemachine;
2077
2078         if (!$prepare) {
2079             # copy snapshot config to current config
2080             $conf = &$snapshot_apply_config($conf, $snap);
2081             $conf->{parent} = $snapname;
2082         }
2083
2084         PVE::LXC::Config->write_config($vmid, $conf);
2085
2086         if (!$prepare && $snap->{vmstate}) {
2087             die "implement me - save vmstate\n";
2088         }
2089     };
2090
2091     PVE::LXC::Config->lock_config($vmid, $updatefn);
2092
2093     foreach_mountpoint($snap, sub {
2094         my ($ms, $mountpoint) = @_;
2095
2096         PVE::Storage::volume_snapshot_rollback($storecfg, $mountpoint->{volume}, $snapname);
2097     });
2098
2099     $prepare = 0;
2100     PVE::LXC::Config->lock_config($vmid, $updatefn);
2101 }
2102
2103 sub template_create {
2104     my ($vmid, $conf) = @_;
2105
2106     my $storecfg = PVE::Storage::config();
2107
2108     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
2109     my $volid = $rootinfo->{volume};
2110
2111     die "Template feature is not available for '$volid'\n"
2112         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
2113
2114     PVE::Storage::activate_volumes($storecfg, [$volid]);
2115
2116     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
2117     $rootinfo->{volume} = $template_volid;
2118     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
2119
2120     PVE::LXC::Config->write_config($vmid, $conf);
2121 }
2122
2123 sub mountpoint_names {
2124     my ($reverse) = @_;
2125
2126     my @names = ('rootfs');
2127
2128     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
2129         push @names, "mp$i";
2130     }
2131
2132     return $reverse ? reverse @names : @names;
2133 }
2134
2135
2136 sub foreach_mountpoint_full {
2137     my ($conf, $reverse, $func) = @_;
2138
2139     foreach my $key (mountpoint_names($reverse)) {
2140         my $value = $conf->{$key};
2141         next if !defined($value);
2142         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2143         next if !defined($mountpoint);
2144
2145         &$func($key, $mountpoint);
2146     }
2147 }
2148
2149 sub foreach_mountpoint {
2150     my ($conf, $func) = @_;
2151
2152     foreach_mountpoint_full($conf, 0, $func);
2153 }
2154
2155 sub foreach_mountpoint_reverse {
2156     my ($conf, $func) = @_;
2157
2158     foreach_mountpoint_full($conf, 1, $func);
2159 }
2160
2161 sub check_ct_modify_config_perm {
2162     my ($rpcenv, $authuser, $vmid, $pool, $newconf, $delete) = @_;
2163
2164     return 1 if $authuser eq 'root@pam';
2165
2166     my $check = sub {
2167         my ($opt, $delete) = @_;
2168         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2169             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2170         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2171             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2172             return if $delete;
2173             my $data = $opt eq 'rootfs' ? parse_ct_rootfs($newconf->{$opt})
2174                                         : parse_ct_mountpoint($newconf->{$opt});
2175             raise_perm_exc("mountpoint type $data->{type}") if $data->{type} ne 'volume';
2176         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2177             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2178         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2179                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2180             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2181         } else {
2182             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2183         }
2184     };
2185
2186     foreach my $opt (keys %$newconf) {
2187         &$check($opt, 0);
2188     }
2189     foreach my $opt (@$delete) {
2190         &$check($opt, 1);
2191     }
2192
2193     return 1;
2194 }
2195
2196 sub umount_all {
2197     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2198
2199     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2200     my $volid_list = get_vm_volumes($conf);
2201
2202     foreach_mountpoint_reverse($conf, sub {
2203         my ($ms, $mountpoint) = @_;
2204
2205         my $volid = $mountpoint->{volume};
2206         my $mount = $mountpoint->{mp};
2207
2208         return if !$volid || !$mount;
2209
2210         my $mount_path = "$rootdir/$mount";
2211         $mount_path =~ s!/+!/!g;
2212
2213         return if !PVE::ProcFSTools::is_mounted($mount_path);
2214
2215         eval {
2216             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2217         };
2218         if (my $err = $@) {
2219             if ($noerr) {
2220                 warn $err;
2221             } else {
2222                 die $err;
2223             }
2224         }
2225     });
2226 }
2227
2228 sub mount_all {
2229     my ($vmid, $storage_cfg, $conf) = @_;
2230
2231     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2232     File::Path::make_path($rootdir);
2233
2234     my $volid_list = get_vm_volumes($conf);
2235     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2236
2237     eval {
2238         foreach_mountpoint($conf, sub {
2239             my ($ms, $mountpoint) = @_;
2240
2241             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2242         });
2243     };
2244     if (my $err = $@) {
2245         warn "mounting container failed\n";
2246         umount_all($vmid, $storage_cfg, $conf, 1);
2247         die $err;
2248     }
2249
2250     return $rootdir;
2251 }
2252
2253
2254 sub mountpoint_mount_path {
2255     my ($mountpoint, $storage_cfg, $snapname) = @_;
2256
2257     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2258 }
2259
2260 my $check_mount_path = sub {
2261     my ($path) = @_;
2262     $path = File::Spec->canonpath($path);
2263     my $real = Cwd::realpath($path);
2264     if ($real ne $path) {
2265         die "mount path modified by symlink: $path != $real";
2266     }
2267 };
2268
2269 sub query_loopdev {
2270     my ($path) = @_;
2271     my $found;
2272     my $parser = sub {
2273         my $line = shift;
2274         if ($line =~ m@^(/dev/loop\d+):@) {
2275             $found = $1;
2276         }
2277     };
2278     my $cmd = ['losetup', '--associated', $path];
2279     PVE::Tools::run_command($cmd, outfunc => $parser);
2280     return $found;
2281 }
2282
2283 # Run a function with a file attached to a loop device.
2284 # The loop device is always detached afterwards (or set to autoclear).
2285 # Returns the loop device.
2286 sub run_with_loopdev {
2287     my ($func, $file) = @_;
2288     my $device = query_loopdev($file);
2289     # Try to reuse an existing device
2290     if ($device) {
2291         # We assume that whoever setup the loop device is responsible for
2292         # detaching it.
2293         &$func($device);
2294         return $device;
2295     }
2296
2297     my $parser = sub {
2298         my $line = shift;
2299         if ($line =~ m@^(/dev/loop\d+)$@) {
2300             $device = $1;
2301         }
2302     };
2303     PVE::Tools::run_command(['losetup', '--show', '-f', $file], outfunc => $parser);
2304     die "failed to setup loop device for $file\n" if !$device;
2305     eval { &$func($device); };
2306     my $err = $@;
2307     PVE::Tools::run_command(['losetup', '-d', $device]);
2308     die $err if $err;
2309     return $device;
2310 }
2311
2312 sub bindmount {
2313     my ($dir, $dest, $ro, @extra_opts) = @_;
2314     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $dir, $dest]);
2315     if ($ro) {
2316         eval { PVE::Tools::run_command(['mount', '-o', 'bind,remount,ro', $dest]); };
2317         if (my $err = $@) {
2318             warn "bindmount error\n";
2319             # don't leave writable bind-mounts behind...
2320             PVE::Tools::run_command(['umount', $dest]);
2321             die $err;
2322         }
2323     }
2324 }
2325
2326 # use $rootdir = undef to just return the corresponding mount path
2327 sub mountpoint_mount {
2328     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2329
2330     my $volid = $mountpoint->{volume};
2331     my $mount = $mountpoint->{mp};
2332     my $type = $mountpoint->{type};
2333     my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota};
2334     my $mounted_dev;
2335
2336     return if !$volid || !$mount;
2337
2338     my $mount_path;
2339
2340     if (defined($rootdir)) {
2341         $rootdir =~ s!/+$!!;
2342         $mount_path = "$rootdir/$mount";
2343         $mount_path =~ s!/+!/!g;
2344         &$check_mount_path($mount_path);
2345         File::Path::mkpath($mount_path);
2346     }
2347
2348     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2349
2350     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2351
2352     my $optstring = '';
2353     if (defined($mountpoint->{acl})) {
2354         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
2355     }
2356     my $readonly = $mountpoint->{ro};
2357
2358     my @extra_opts = ('-o', $optstring);
2359
2360     if ($storage) {
2361
2362         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2363         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2364
2365         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2366             PVE::Storage::parse_volname($storage_cfg, $volid);
2367
2368         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2369
2370         if ($format eq 'subvol') {
2371             if ($mount_path) {
2372                 if ($snapname) {
2373                     if ($scfg->{type} eq 'zfspool') {
2374                         my $path_arg = $path;
2375                         $path_arg =~ s!^/+!!;
2376                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
2377                     } else {
2378                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2379                     }
2380                 } else {
2381                     bindmount($path, $mount_path, $readonly, @extra_opts);
2382                     warn "cannot enable quota control for bind mounted subvolumes\n" if $quota;
2383                 }
2384             }
2385             return wantarray ? ($path, 0, $mounted_dev) : $path;
2386         } elsif ($format eq 'raw' || $format eq 'iso') {
2387             my $domount = sub {
2388                 my ($path) = @_;
2389                 if ($mount_path) {
2390                     if ($format eq 'iso') {
2391                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2392                     } elsif ($isBase || defined($snapname)) {
2393                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2394                     } else {
2395                         if ($quota) {
2396                             push @extra_opts, '-o', 'usrjquota=aquota.user,grpjquota=aquota.group,jqfmt=vfsv0';
2397                         }
2398                         push @extra_opts, '-o', 'ro' if $readonly;
2399                         PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2400                     }
2401                 }
2402             };
2403             my $use_loopdev = 0;
2404             if ($scfg->{path}) {
2405                 $mounted_dev = run_with_loopdev($domount, $path);
2406                 $use_loopdev = 1;
2407             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2408                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2409                 $mounted_dev = $path;
2410                 &$domount($path);
2411             } else {
2412                 die "unsupported storage type '$scfg->{type}'\n";
2413             }
2414             return wantarray ? ($path, $use_loopdev, $mounted_dev) : $path;
2415         } else {
2416             die "unsupported image format '$format'\n";
2417         }
2418     } elsif ($type eq 'device') {
2419                         push @extra_opts, '-o', 'ro' if $readonly;
2420         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
2421         return wantarray ? ($volid, 0, $volid) : $volid;
2422     } elsif ($type eq 'bind') {
2423         die "directory '$volid' does not exist\n" if ! -d $volid;
2424         &$check_mount_path($volid);
2425         bindmount($volid, $mount_path, $readonly, @extra_opts) if $mount_path;
2426         warn "cannot enable quota control for bind mounts\n" if $quota;
2427         return wantarray ? ($volid, 0, undef) : $volid;
2428     }
2429
2430     die "unsupported storage";
2431 }
2432
2433 sub get_vm_volumes {
2434     my ($conf, $excludes) = @_;
2435
2436     my $vollist = [];
2437
2438     foreach_mountpoint($conf, sub {
2439         my ($ms, $mountpoint) = @_;
2440
2441         return if $excludes && $ms eq $excludes;
2442
2443         my $volid = $mountpoint->{volume};
2444
2445         return if !$volid || $mountpoint->{type} ne 'volume';
2446
2447         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2448         return if !$sid;
2449
2450         push @$vollist, $volid;
2451     });
2452
2453     return $vollist;
2454 }
2455
2456 sub mkfs {
2457     my ($dev, $rootuid, $rootgid) = @_;
2458
2459     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2460                              '-E', "root_owner=$rootuid:$rootgid",
2461                              $dev]);
2462 }
2463
2464 sub format_disk {
2465     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2466
2467     if ($volid =~ m!^/dev/.+!) {
2468         mkfs($volid);
2469         return;
2470     }
2471
2472     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2473
2474     die "cannot format volume '$volid' with no storage\n" if !$storage;
2475
2476     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2477
2478     my $path = PVE::Storage::path($storage_cfg, $volid);
2479
2480     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2481         PVE::Storage::parse_volname($storage_cfg, $volid);
2482
2483     die "cannot format volume '$volid' (format == $format)\n"
2484         if $format ne 'raw';
2485
2486     mkfs($path, $rootuid, $rootgid);
2487 }
2488
2489 sub destroy_disks {
2490     my ($storecfg, $vollist) = @_;
2491
2492     foreach my $volid (@$vollist) {
2493         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2494         warn $@ if $@;
2495     }
2496 }
2497
2498 sub create_disks {
2499     my ($storecfg, $vmid, $settings, $conf) = @_;
2500
2501     my $vollist = [];
2502
2503     eval {
2504         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2505         my $chown_vollist = [];
2506
2507         foreach_mountpoint($settings, sub {
2508             my ($ms, $mountpoint) = @_;
2509
2510             my $volid = $mountpoint->{volume};
2511             my $mp = $mountpoint->{mp};
2512
2513             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2514
2515             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2516                 my ($storeid, $size_gb) = ($1, $2);
2517
2518                 my $size_kb = int(${size_gb}*1024) * 1024;
2519
2520                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2521                 # fixme: use better naming ct-$vmid-disk-X.raw?
2522
2523                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2524                     if ($size_kb > 0) {
2525                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2526                                                            undef, $size_kb);
2527                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2528                     } else {
2529                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2530                                                            undef, 0);
2531                         push @$chown_vollist, $volid;
2532                     }
2533                 } elsif ($scfg->{type} eq 'zfspool') {
2534
2535                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2536                                                        undef, $size_kb);
2537                     push @$chown_vollist, $volid;
2538                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2539
2540                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2541                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2542
2543                 } elsif ($scfg->{type} eq 'rbd') {
2544
2545                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2546                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2547                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2548                 } else {
2549                     die "unable to create containers on storage type '$scfg->{type}'\n";
2550                 }
2551                 push @$vollist, $volid;
2552                 $mountpoint->{volume} = $volid;
2553                 $mountpoint->{size} = $size_kb * 1024;
2554                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2555             } else {
2556                 # use specified/existing volid/dir/device
2557                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2558             }
2559         });
2560
2561         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2562         foreach my $volid (@$chown_vollist) {
2563             my $path = PVE::Storage::path($storecfg, $volid, undef);
2564             chown($rootuid, $rootgid, $path);
2565         }
2566         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2567     };
2568     # free allocated images on error
2569     if (my $err = $@) {
2570         destroy_disks($storecfg, $vollist);
2571         die $err;
2572     }
2573     return $vollist;
2574 }
2575
2576 # bash completion helper
2577
2578 sub complete_os_templates {
2579     my ($cmdname, $pname, $cvalue) = @_;
2580
2581     my $cfg = PVE::Storage::config();
2582
2583     my $storeid;
2584
2585     if ($cvalue =~ m/^([^:]+):/) {
2586         $storeid = $1;
2587     }
2588
2589     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2590     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2591
2592     my $res = [];
2593     foreach my $id (keys %$data) {
2594         foreach my $item (@{$data->{$id}}) {
2595             push @$res, $item->{volid} if defined($item->{volid});
2596         }
2597     }
2598
2599     return $res;
2600 }
2601
2602 my $complete_ctid_full = sub {
2603     my ($running) = @_;
2604
2605     my $idlist = vmstatus();
2606
2607     my $active_hash = list_active_containers();
2608
2609     my $res = [];
2610
2611     foreach my $id (keys %$idlist) {
2612         my $d = $idlist->{$id};
2613         if (defined($running)) {
2614             next if $d->{template};
2615             next if $running && !$active_hash->{$id};
2616             next if !$running && $active_hash->{$id};
2617         }
2618         push @$res, $id;
2619
2620     }
2621     return $res;
2622 };
2623
2624 sub complete_ctid {
2625     return &$complete_ctid_full();
2626 }
2627
2628 sub complete_ctid_stopped {
2629     return &$complete_ctid_full(0);
2630 }
2631
2632 sub complete_ctid_running {
2633     return &$complete_ctid_full(1);
2634 }
2635
2636 sub parse_id_maps {
2637     my ($conf) = @_;
2638
2639     my $id_map = [];
2640     my $rootuid = 0;
2641     my $rootgid = 0;
2642
2643     my $lxc = $conf->{lxc};
2644     foreach my $entry (@$lxc) {
2645         my ($key, $value) = @$entry;
2646         next if $key ne 'lxc.id_map';
2647         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2648             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2649             push @$id_map, [$type, $ct, $host, $length];
2650             if ($ct == 0) {
2651                 $rootuid = $host if $type eq 'u';
2652                 $rootgid = $host if $type eq 'g';
2653             }
2654         } else {
2655             die "failed to parse id_map: $value\n";
2656         }
2657     }
2658
2659     if (!@$id_map && $conf->{unprivileged}) {
2660         # Should we read them from /etc/subuid?
2661         $id_map = [ ['u', '0', '100000', '65536'],
2662                     ['g', '0', '100000', '65536'] ];
2663         $rootuid = $rootgid = 100000;
2664     }
2665
2666     return ($id_map, $rootuid, $rootgid);
2667 }
2668
2669 sub userns_command {
2670     my ($id_map) = @_;
2671     if (@$id_map) {
2672         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2673     }
2674     return [];
2675 }
2676
2677
2678 1;