src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX qw(EINTR);
   7
   8 use Socket;
   9
  10 use File::Path;
  11 use File::Spec;
  12 use Cwd qw();
  13 use Fcntl qw(O_RDONLY);
  14
  15 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  16 use PVE::Exception qw(raise_perm_exc);
  17 use PVE::Storage;
  18 use PVE::SafeSyslog;
  19 use PVE::INotify;
  20 use PVE::JSONSchema qw(get_standard_option);
  21 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full);
  22 use PVE::Network;
  23 use PVE::AccessControl;
  24 use PVE::ProcFSTools;
  25 use PVE::LXC::Config;
  26 use Time::HiRes qw (gettimeofday);
  27
  28 use Data::Dumper;
  29
  30 my $nodename = PVE::INotify::nodename();
  31
  32 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  33
  34 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  35                           '--xattrs',
  36                           '--xattrs-include=user.*',
  37                           '--xattrs-include=security.capability',
  38                           '--warning=no-xattr-write' ];
  39
  40 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  41
  42 my $rootfs_desc = {
  43     volume => {
  44         type => 'string',
  45         default_key => 1,
  46         format => 'pve-lxc-mp-string',
  47         format_description => 'volume',
  48         description => 'Volume, device or directory to mount into the container.',
  49     },
  50     backup => {
  51         type => 'boolean',
  52         format_description => '[1|0]',
  53         description => 'Whether to include the mountpoint in backups.',
  54         optional => 1,
  55     },
  56     size => {
  57         type => 'string',
  58         format => 'disk-size',
  59         format_description => 'DiskSize',
  60         description => 'Volume size (read only value).',
  61         optional => 1,
  62     },
  63     acl => {
  64         type => 'boolean',
  65         format_description => 'acl',
  66         description => 'Explicitly enable or disable ACL support.',
  67         optional => 1,
  68     },
  69     ro => {
  70         type => 'boolean',
  71         format_description => 'ro',
  72         description => 'Read-only mountpoint (not supported with bind mounts)',
  73         optional => 1,
  74     },
  75     quota => {
  76         type => 'boolean',
  77         format_description => '[0|1]',
  78         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
  79         optional => 1,
  80     },
  81 };
  82
  83 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  84     type => 'string', format => $rootfs_desc,
  85     description => "Use volume as container root.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  90     description => "The name of the snapshot.",
  91     type => 'string', format => 'pve-configid',
  92     maxLength => 40,
  93 });
  94
  95 my $confdesc = {
  96     lock => {
  97         optional => 1,
  98         type => 'string',
  99         description => "Lock/unlock the VM.",
 100         enum => [qw(migrate backup snapshot rollback)],
 101     },
 102     onboot => {
 103         optional => 1,
 104         type => 'boolean',
 105         description => "Specifies whether a VM will be started during system bootup.",
 106         default => 0,
 107     },
 108     startup => get_standard_option('pve-startup-order'),
 109     template => {
 110         optional => 1,
 111         type => 'boolean',
 112         description => "Enable/disable Template.",
 113         default => 0,
 114     },
 115     arch => {
 116         optional => 1,
 117         type => 'string',
 118         enum => ['amd64', 'i386'],
 119         description => "OS architecture type.",
 120         default => 'amd64',
 121     },
 122     ostype => {
 123         optional => 1,
 124         type => 'string',
 125         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux', 'alpine', 'unmanaged'],
 126         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 127     },
 128     console => {
 129         optional => 1,
 130         type => 'boolean',
 131         description => "Attach a console device (/dev/console) to the container.",
 132         default => 1,
 133     },
 134     tty => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Specify the number of tty available to the container",
 138         minimum => 0,
 139         maximum => 6,
 140         default => 2,
 141     },
 142     cpulimit => {
 143         optional => 1,
 144         type => 'number',
 145         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 146         minimum => 0,
 147         maximum => 128,
 148         default => 0,
 149     },
 150     cpuunits => {
 151         optional => 1,
 152         type => 'integer',
 153         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 154         minimum => 0,
 155         maximum => 500000,
 156         default => 1024,
 157     },
 158     memory => {
 159         optional => 1,
 160         type => 'integer',
 161         description => "Amount of RAM for the VM in MB.",
 162         minimum => 16,
 163         default => 512,
 164     },
 165     swap => {
 166         optional => 1,
 167         type => 'integer',
 168         description => "Amount of SWAP for the VM in MB.",
 169         minimum => 0,
 170         default => 512,
 171     },
 172     hostname => {
 173         optional => 1,
 174         description => "Set a host name for the container.",
 175         type => 'string', format => 'dns-name',
 176         maxLength => 255,
 177     },
 178     description => {
 179         optional => 1,
 180         type => 'string',
 181         description => "Container description. Only used on the configuration web interface.",
 182     },
 183     searchdomain => {
 184         optional => 1,
 185         type => 'string', format => 'dns-name-list',
 186         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 187     },
 188     nameserver => {
 189         optional => 1,
 190         type => 'string', format => 'address-list',
 191         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 192     },
 193     rootfs => get_standard_option('pve-ct-rootfs'),
 194     parent => {
 195         optional => 1,
 196         type => 'string', format => 'pve-configid',
 197         maxLength => 40,
 198         description => "Parent snapshot name. This is used internally, and should not be modified.",
 199     },
 200     snaptime => {
 201         optional => 1,
 202         description => "Timestamp for snapshots.",
 203         type => 'integer',
 204         minimum => 0,
 205     },
 206     cmode => {
 207         optional => 1,
 208         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 209         type => 'string',
 210         enum => ['shell', 'console', 'tty'],
 211         default => 'tty',
 212     },
 213     protection => {
 214         optional => 1,
 215         type => 'boolean',
 216         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 217         default => 0,
 218     },
 219     unprivileged => {
 220         optional => 1,
 221         type => 'boolean',
 222         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 223         default => 0,
 224     },
 225 };
 226
 227 my $valid_lxc_conf_keys = {
 228     'lxc.include' => 1,
 229     'lxc.arch' => 1,
 230     'lxc.utsname' => 1,
 231     'lxc.haltsignal' => 1,
 232     'lxc.rebootsignal' => 1,
 233     'lxc.stopsignal' => 1,
 234     'lxc.init_cmd' => 1,
 235     'lxc.network.type' => 1,
 236     'lxc.network.flags' => 1,
 237     'lxc.network.link' => 1,
 238     'lxc.network.mtu' => 1,
 239     'lxc.network.name' => 1,
 240     'lxc.network.hwaddr' => 1,
 241     'lxc.network.ipv4' => 1,
 242     'lxc.network.ipv4.gateway' => 1,
 243     'lxc.network.ipv6' => 1,
 244     'lxc.network.ipv6.gateway' => 1,
 245     'lxc.network.script.up' => 1,
 246     'lxc.network.script.down' => 1,
 247     'lxc.pts' => 1,
 248     'lxc.console.logfile' => 1,
 249     'lxc.console' => 1,
 250     'lxc.tty' => 1,
 251     'lxc.devttydir' => 1,
 252     'lxc.hook.autodev' => 1,
 253     'lxc.autodev' => 1,
 254     'lxc.kmsg' => 1,
 255     'lxc.mount' => 1,
 256     'lxc.mount.entry' => 1,
 257     'lxc.mount.auto' => 1,
 258     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 259     'lxc.rootfs.mount' => 1,
 260     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 261                             ', please use mountpoint options in the "rootfs" key',
 262     # lxc.cgroup.*
 263     'lxc.cap.drop' => 1,
 264     'lxc.cap.keep' => 1,
 265     'lxc.aa_profile' => 1,
 266     'lxc.aa_allow_incomplete' => 1,
 267     'lxc.se_context' => 1,
 268     'lxc.seccomp' => 1,
 269     'lxc.id_map' => 1,
 270     'lxc.hook.pre-start' => 1,
 271     'lxc.hook.pre-mount' => 1,
 272     'lxc.hook.mount' => 1,
 273     'lxc.hook.start' => 1,
 274     'lxc.hook.stop' => 1,
 275     'lxc.hook.post-stop' => 1,
 276     'lxc.hook.clone' => 1,
 277     'lxc.hook.destroy' => 1,
 278     'lxc.loglevel' => 1,
 279     'lxc.logfile' => 1,
 280     'lxc.start.auto' => 1,
 281     'lxc.start.delay' => 1,
 282     'lxc.start.order' => 1,
 283     'lxc.group' => 1,
 284     'lxc.environment' => 1,
 285 };
 286
 287 my $netconf_desc = {
 288     type => {
 289         type => 'string',
 290         optional => 1,
 291         description => "Network interface type.",
 292         enum => [qw(veth)],
 293     },
 294     name => {
 295         type => 'string',
 296         format_description => 'String',
 297         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 298         pattern => '[-_.\w\d]+',
 299     },
 300     bridge => {
 301         type => 'string',
 302         format_description => 'vmbr<Number>',
 303         description => 'Bridge to attach the network device to.',
 304         pattern => '[-_.\w\d]+',
 305         optional => 1,
 306     },
 307     hwaddr => {
 308         type => 'string',
 309         format_description => 'MAC',
 310         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 311         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 312         optional => 1,
 313     },
 314     mtu => {
 315         type => 'integer',
 316         format_description => 'Number',
 317         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 318         minimum => 64, # minimum ethernet frame is 64 bytes
 319         optional => 1,
 320     },
 321     ip => {
 322         type => 'string',
 323         format => 'pve-ipv4-config',
 324         format_description => 'IPv4Format/CIDR',
 325         description => 'IPv4 address in CIDR format.',
 326         optional => 1,
 327     },
 328     gw => {
 329         type => 'string',
 330         format => 'ipv4',
 331         format_description => 'GatewayIPv4',
 332         description => 'Default gateway for IPv4 traffic.',
 333         optional => 1,
 334     },
 335     ip6 => {
 336         type => 'string',
 337         format => 'pve-ipv6-config',
 338         format_description => 'IPv6Format/CIDR',
 339         description => 'IPv6 address in CIDR format.',
 340         optional => 1,
 341     },
 342     gw6 => {
 343         type => 'string',
 344         format => 'ipv6',
 345         format_description => 'GatewayIPv6',
 346         description => 'Default gateway for IPv6 traffic.',
 347         optional => 1,
 348     },
 349     firewall => {
 350         type => 'boolean',
 351         format_description => '[1|0]',
 352         description => "Controls whether this interface's firewall rules should be used.",
 353         optional => 1,
 354     },
 355     tag => {
 356         type => 'integer',
 357         format_description => 'VlanNo',
 358         minimum => '2',
 359         maximum => '4094',
 360         description => "VLAN tag for this interface.",
 361         optional => 1,
 362     },
 363     trunks => {
 364         type => 'string',
 365         pattern => qr/\d+(?:;\d+)*/,
 366         format_description => 'vlanid[;vlanid...]',
 367         description => "VLAN ids to pass through the interface",
 368         optional => 1,
 369     },
 370 };
 371 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 372
 373 my $MAX_LXC_NETWORKS = 10;
 374 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 375     $confdesc->{"net$i"} = {
 376         optional => 1,
 377         type => 'string', format => $netconf_desc,
 378         description => "Specifies network interfaces for the container.",
 379     };
 380 }
 381
 382 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 383 sub verify_lxc_mp_string{
 384     my ($mp, $noerr) = @_;
 385
 386     # do not allow:
 387     # /./ or /../
 388     # /. or /.. at the end
 389     # ../ at the beginning
 390
 391     if($mp =~ m@/\.\.?/@ ||
 392        $mp =~ m@/\.\.?$@ ||
 393        $mp =~ m@^\.\./@){
 394         return undef if $noerr;
 395         die "$mp contains illegal character sequences\n";
 396     }
 397     return $mp;
 398 }
 399
 400 my $mp_desc = {
 401     %$rootfs_desc,
 402     mp => {
 403         type => 'string',
 404         format => 'pve-lxc-mp-string',
 405         format_description => 'Path',
 406         description => 'Path to the mountpoint as seen from inside the container.',
 407     },
 408 };
 409 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 410
 411 my $unuseddesc = {
 412     optional => 1,
 413     type => 'string', format => 'pve-volume-id',
 414     description => "Reference to unused volumes.",
 415 };
 416
 417 my $MAX_MOUNT_POINTS = 10;
 418 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 419     $confdesc->{"mp$i"} = {
 420         optional => 1,
 421         type => 'string', format => $mp_desc,
 422         description => "Use volume as container mount point (experimental feature).",
 423         optional => 1,
 424     };
 425 }
 426
 427 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 428 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 429     $confdesc->{"unused$i"} = $unuseddesc;
 430 }
 431
 432 sub write_pct_config {
 433     my ($filename, $conf) = @_;
 434
 435     delete $conf->{snapstate}; # just to be sure
 436     my $volidlist = get_vm_volumes($conf);
 437     my $used_volids = {};
 438     foreach my $vid (@$volidlist) {
 439         $used_volids->{$vid} = 1;
 440     }
 441
 442     # remove 'unusedX' settings if the volume is still used
 443     foreach my $key (keys %$conf) {
 444         my $value = $conf->{$key};
 445         if ($key =~ m/^unused/ && $used_volids->{$value}) {
 446             delete $conf->{$key};
 447         }
 448     }
 449
 450     my $generate_raw_config = sub {
 451         my ($conf) = @_;
 452
 453         my $raw = '';
 454
 455         # add description as comment to top of file
 456         my $descr = $conf->{description} || '';
 457         foreach my $cl (split(/\n/, $descr)) {
 458             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 459         }
 460
 461         foreach my $key (sort keys %$conf) {
 462             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 463                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 464             my $value = $conf->{$key};
 465             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 466             $raw .= "$key: $value\n";
 467         }
 468
 469         if (my $lxcconf = $conf->{lxc}) {
 470             foreach my $entry (@$lxcconf) {
 471                 my ($k, $v) = @$entry;
 472                 $raw .= "$k: $v\n";
 473             }
 474         }
 475
 476         return $raw;
 477     };
 478
 479     my $raw = &$generate_raw_config($conf);
 480
 481     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 482         $raw .= "\n[$snapname]\n";
 483         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 484     }
 485
 486     return $raw;
 487 }
 488
 489 sub check_type {
 490     my ($key, $value) = @_;
 491
 492     die "unknown setting '$key'\n" if !$confdesc->{$key};
 493
 494     my $type = $confdesc->{$key}->{type};
 495
 496     if (!defined($value)) {
 497         die "got undefined value\n";
 498     }
 499
 500     if ($value =~ m/[\n\r]/) {
 501         die "property contains a line feed\n";
 502     }
 503
 504     if ($type eq 'boolean') {
 505         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 506         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 507         die "type check ('boolean') failed - got '$value'\n";
 508     } elsif ($type eq 'integer') {
 509         return int($1) if $value =~ m/^(\d+)$/;
 510         die "type check ('integer') failed - got '$value'\n";
 511     } elsif ($type eq 'number') {
 512         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 513         die "type check ('number') failed - got '$value'\n";
 514     } elsif ($type eq 'string') {
 515         if (my $fmt = $confdesc->{$key}->{format}) {
 516             PVE::JSONSchema::check_format($fmt, $value);
 517             return $value;
 518         }
 519         return $value;
 520     } else {
 521         die "internal error"
 522     }
 523 }
 524
 525 sub parse_pct_config {
 526     my ($filename, $raw) = @_;
 527
 528     return undef if !defined($raw);
 529
 530     my $res = {
 531         digest => Digest::SHA::sha1_hex($raw),
 532         snapshots => {},
 533     };
 534
 535     $filename =~ m|/lxc/(\d+).conf$|
 536         || die "got strange filename '$filename'";
 537
 538     my $vmid = $1;
 539
 540     my $conf = $res;
 541     my $descr = '';
 542     my $section = '';
 543
 544     my @lines = split(/\n/, $raw);
 545     foreach my $line (@lines) {
 546         next if $line =~ m/^\s*$/;
 547
 548         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 549             $section = $1;
 550             $conf->{description} = $descr if $descr;
 551             $descr = '';
 552             $conf = $res->{snapshots}->{$section} = {};
 553             next;
 554         }
 555
 556         if ($line =~ m/^\#(.*)\s*$/) {
 557             $descr .= PVE::Tools::decode_text($1) . "\n";
 558             next;
 559         }
 560
 561         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 562             my $key = $1;
 563             my $value = $3;
 564             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 565             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 566                 push @{$conf->{lxc}}, [$key, $value];
 567             } elsif (my $errmsg = $validity) {
 568                 warn "vm $vmid - $key: $errmsg\n";
 569             } else {
 570                 warn "vm $vmid - unable to parse config: $line\n";
 571             }
 572         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 573             $descr .= PVE::Tools::decode_text($2);
 574         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 575             $conf->{snapstate} = $1;
 576         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 577             my $key = $1;
 578             my $value = $2;
 579             eval { $value = check_type($key, $value); };
 580             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 581             $conf->{$key} = $value;
 582         } else {
 583             warn "vm $vmid - unable to parse config: $line\n";
 584         }
 585     }
 586
 587     $conf->{description} = $descr if $descr;
 588
 589     delete $res->{snapstate}; # just to be sure
 590
 591     return $res;
 592 }
 593
 594 sub config_list {
 595     my $vmlist = PVE::Cluster::get_vmlist();
 596     my $res = {};
 597     return $res if !$vmlist || !$vmlist->{ids};
 598     my $ids = $vmlist->{ids};
 599
 600     foreach my $vmid (keys %$ids) {
 601         next if !$vmid; # skip CT0
 602         my $d = $ids->{$vmid};
 603         next if !$d->{node} || $d->{node} ne $nodename;
 604         next if !$d->{type} || $d->{type} ne 'lxc';
 605         $res->{$vmid}->{type} = 'lxc';
 606     }
 607     return $res;
 608 }
 609
 610 sub destroy_config {
 611     my ($vmid) = @_;
 612
 613     unlink PVE::LXC::Config->config_file($vmid, $nodename);
 614 }
 615
 616 sub option_exists {
 617     my ($name) = @_;
 618
 619     return defined($confdesc->{$name});
 620 }
 621
 622 # add JSON properties for create and set function
 623 sub json_config_properties {
 624     my $prop = shift;
 625
 626     foreach my $opt (keys %$confdesc) {
 627         next if $opt eq 'parent' || $opt eq 'snaptime';
 628         next if $prop->{$opt};
 629         $prop->{$opt} = $confdesc->{$opt};
 630     }
 631
 632     return $prop;
 633 }
 634
 635 # container status helpers
 636
 637 sub list_active_containers {
 638
 639     my $filename = "/proc/net/unix";
 640
 641     # similar test is used by lcxcontainers.c: list_active_containers
 642     my $res = {};
 643
 644     my $fh = IO::File->new ($filename, "r");
 645     return $res if !$fh;
 646
 647     while (defined(my $line = <$fh>)) {
 648         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 649             my $path = $1;
 650             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 651                 $res->{$1} = 1;
 652             }
 653         }
 654     }
 655
 656     close($fh);
 657
 658     return $res;
 659 }
 660
 661 # warning: this is slow
 662 sub check_running {
 663     my ($vmid) = @_;
 664
 665     my $active_hash = list_active_containers();
 666
 667     return 1 if defined($active_hash->{$vmid});
 668
 669     return undef;
 670 }
 671
 672 sub get_container_disk_usage {
 673     my ($vmid, $pid) = @_;
 674
 675     return PVE::Tools::df("/proc/$pid/root/", 1);
 676 }
 677
 678 my $last_proc_vmid_stat;
 679
 680 my $parse_cpuacct_stat = sub {
 681     my ($vmid) = @_;
 682
 683     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 684
 685     my $stat = {};
 686
 687     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 688
 689         $stat->{utime} = $1;
 690         $stat->{stime} = $2;
 691
 692     }
 693
 694     return $stat;
 695 };
 696
 697 sub vmstatus {
 698     my ($opt_vmid) = @_;
 699
 700     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 701
 702     my $active_hash = list_active_containers();
 703
 704     my $cpucount = $cpuinfo->{cpus} || 1;
 705
 706     my $cdtime = gettimeofday;
 707
 708     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 709
 710     foreach my $vmid (keys %$list) {
 711         my $d = $list->{$vmid};
 712
 713         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 714         warn $@ if $@; # ignore errors (consider them stopped)
 715
 716         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 717
 718         my $cfspath = PVE::LXC::Config->cfs_config_path($vmid);
 719         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 720
 721         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 722         $d->{name} =~ s/[\s]//g;
 723
 724         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 725
 726         $d->{lock} = $conf->{lock} || '';
 727
 728         if ($d->{pid}) {
 729             my $res = get_container_disk_usage($vmid, $d->{pid});
 730             $d->{disk} = $res->{used};
 731             $d->{maxdisk} = $res->{total};
 732         } else {
 733             $d->{disk} = 0;
 734             # use 4GB by default ??
 735             if (my $rootfs = $conf->{rootfs}) {
 736                 my $rootinfo = parse_ct_rootfs($rootfs);
 737                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 738             } else {
 739                 $d->{maxdisk} = 4*1024*1024*1024;
 740             }
 741         }
 742
 743         $d->{mem} = 0;
 744         $d->{swap} = 0;
 745         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 746         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 747
 748         $d->{uptime} = 0;
 749         $d->{cpu} = 0;
 750
 751         $d->{netout} = 0;
 752         $d->{netin} = 0;
 753
 754         $d->{diskread} = 0;
 755         $d->{diskwrite} = 0;
 756
 757         $d->{template} = PVE::LXC::Config->is_template($conf);
 758     }
 759
 760     foreach my $vmid (keys %$list) {
 761         my $d = $list->{$vmid};
 762         my $pid = $d->{pid};
 763
 764         next if !$pid; # skip stopped CTs
 765
 766         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 767         $d->{uptime} = time - $ctime; # the method lxcfs uses
 768
 769         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 770         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 771
 772         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 773         my @bytes = split(/\n/, $blkio_bytes);
 774         foreach my $byte (@bytes) {
 775             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 776                 $d->{diskread} = $2 if $key eq 'Read';
 777                 $d->{diskwrite} = $2 if $key eq 'Write';
 778             }
 779         }
 780
 781         my $pstat = &$parse_cpuacct_stat($vmid);
 782
 783         my $used = $pstat->{utime} + $pstat->{stime};
 784
 785         my $old = $last_proc_vmid_stat->{$vmid};
 786         if (!$old) {
 787             $last_proc_vmid_stat->{$vmid} = {
 788                 time => $cdtime,
 789                 used => $used,
 790                 cpu => 0,
 791             };
 792             next;
 793         }
 794
 795         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 796
 797         if ($dtime > 1000) {
 798             my $dutime = $used -  $old->{used};
 799
 800             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 801             $last_proc_vmid_stat->{$vmid} = {
 802                 time => $cdtime,
 803                 used => $used,
 804                 cpu => $d->{cpu},
 805             };
 806         } else {
 807             $d->{cpu} = $old->{cpu};
 808         }
 809     }
 810
 811     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 812
 813     foreach my $dev (keys %$netdev) {
 814         next if $dev !~ m/^veth([1-9]\d*)i/;
 815         my $vmid = $1;
 816         my $d = $list->{$vmid};
 817
 818         next if !$d;
 819
 820         $d->{netout} += $netdev->{$dev}->{receive};
 821         $d->{netin} += $netdev->{$dev}->{transmit};
 822
 823     }
 824
 825     return $list;
 826 }
 827
 828 sub classify_mountpoint {
 829     my ($vol) = @_;
 830     if ($vol =~ m!^/!) {
 831         return 'device' if $vol =~ m!^/dev/!;
 832         return 'bind';
 833     }
 834     return 'volume';
 835 }
 836
 837 my $parse_ct_mountpoint_full = sub {
 838     my ($desc, $data, $noerr) = @_;
 839
 840     $data //= '';
 841
 842     my $res;
 843     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 844     if ($@) {
 845         return undef if $noerr;
 846         die $@;
 847     }
 848
 849     if (defined(my $size = $res->{size})) {
 850         $size = PVE::JSONSchema::parse_size($size);
 851         if (!defined($size)) {
 852             return undef if $noerr;
 853             die "invalid size: $size\n";
 854         }
 855         $res->{size} = $size;
 856     }
 857
 858     $res->{type} = classify_mountpoint($res->{volume});
 859
 860     return $res;
 861 };
 862
 863 sub parse_ct_rootfs {
 864     my ($data, $noerr) = @_;
 865
 866     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 867
 868     $res->{mp} = '/' if defined($res);
 869
 870     return $res;
 871 }
 872
 873 sub parse_ct_mountpoint {
 874     my ($data, $noerr) = @_;
 875
 876     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 877 }
 878
 879 sub print_ct_mountpoint {
 880     my ($info, $nomp) = @_;
 881     my $skip = [ 'type' ];
 882     push @$skip, 'mp' if $nomp;
 883     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 884 }
 885
 886 sub print_lxc_network {
 887     my $net = shift;
 888     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 889 }
 890
 891 sub parse_lxc_network {
 892     my ($data) = @_;
 893
 894     my $res = {};
 895
 896     return $res if !$data;
 897
 898     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 899
 900     $res->{type} = 'veth';
 901     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 902
 903     return $res;
 904 }
 905
 906 sub read_cgroup_value {
 907     my ($group, $vmid, $name, $full) = @_;
 908
 909     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 910
 911     return PVE::Tools::file_get_contents($path) if $full;
 912
 913     return PVE::Tools::file_read_firstline($path);
 914 }
 915
 916 sub write_cgroup_value {
 917    my ($group, $vmid, $name, $value) = @_;
 918
 919    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 920    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 921
 922 }
 923
 924 sub find_lxc_console_pids {
 925
 926     my $res = {};
 927
 928     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 929         my ($pid) = @_;
 930
 931         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 932         return if !$cmdline;
 933
 934         my @args = split(/\0/, $cmdline);
 935
 936         # search for lxc-console -n <vmid>
 937         return if scalar(@args) != 3;
 938         return if $args[1] ne '-n';
 939         return if $args[2] !~ m/^\d+$/;
 940         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 941
 942         my $vmid = $args[2];
 943
 944         push @{$res->{$vmid}}, $pid;
 945     });
 946
 947     return $res;
 948 }
 949
 950 sub find_lxc_pid {
 951     my ($vmid) = @_;
 952
 953     my $pid = undef;
 954     my $parser = sub {
 955         my $line = shift;
 956         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 957     };
 958     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
 959
 960     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 961
 962     return $pid;
 963 }
 964
 965 # Note: we cannot use Net:IP, because that only allows strict
 966 # CIDR networks
 967 sub parse_ipv4_cidr {
 968     my ($cidr, $noerr) = @_;
 969
 970     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
 971         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
 972     }
 973
 974     return undef if $noerr;
 975
 976     die "unable to parse ipv4 address/mask\n";
 977 }
 978
 979
 980 sub update_lxc_config {
 981     my ($storage_cfg, $vmid, $conf) = @_;
 982
 983     my $dir = "/var/lib/lxc/$vmid";
 984
 985     if ($conf->{template}) {
 986
 987         unlink "$dir/config";
 988
 989         return;
 990     }
 991
 992     my $raw = '';
 993
 994     die "missing 'arch' - internal error" if !$conf->{arch};
 995     $raw .= "lxc.arch = $conf->{arch}\n";
 996
 997     my $unprivileged = $conf->{unprivileged};
 998     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
 999
1000     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1001     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux | alpine | unmanaged)$/x) {
1002         my $inc ="/usr/share/lxc/config/$ostype.common.conf";
1003         $inc ="/usr/share/lxc/config/common.conf" if !-f $inc;
1004         $raw .= "lxc.include = $inc\n";
1005         if ($unprivileged || $custom_idmap) {
1006             $inc = "/usr/share/lxc/config/$ostype.userns.conf";
1007             $inc = "/usr/share/lxc/config/userns.conf" if !-f $inc;
1008             $raw .= "lxc.include = $inc\n"
1009         }
1010     } else {
1011         die "implement me (ostype $ostype)";
1012     }
1013
1014     # WARNING: DO NOT REMOVE this without making sure that loop device nodes
1015     # cannot be exposed to the container with r/w access (cgroup perms).
1016     # When this is enabled mounts will still remain in the monitor's namespace
1017     # after the container unmounted them and thus will not detach from their
1018     # files while the container is running!
1019     $raw .= "lxc.monitor.unshare = 1\n";
1020
1021     # Should we read them from /etc/subuid?
1022     if ($unprivileged && !$custom_idmap) {
1023         $raw .= "lxc.id_map = u 0 100000 65536\n";
1024         $raw .= "lxc.id_map = g 0 100000 65536\n";
1025     }
1026
1027     if (!has_dev_console($conf)) {
1028         $raw .= "lxc.console = none\n";
1029         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1030     }
1031
1032     my $ttycount = get_tty_count($conf);
1033     $raw .= "lxc.tty = $ttycount\n";
1034
1035     # some init scripts expect a linux terminal (turnkey).
1036     $raw .= "lxc.environment = TERM=linux\n";
1037
1038     my $utsname = $conf->{hostname} || "CT$vmid";
1039     $raw .= "lxc.utsname = $utsname\n";
1040
1041     my $memory = $conf->{memory} || 512;
1042     my $swap = $conf->{swap} // 0;
1043
1044     my $lxcmem = int($memory*1024*1024);
1045     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1046
1047     my $lxcswap = int(($memory + $swap)*1024*1024);
1048     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1049
1050     if (my $cpulimit = $conf->{cpulimit}) {
1051         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1052         my $value = int(100000*$cpulimit);
1053         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1054     }
1055
1056     my $shares = $conf->{cpuunits} || 1024;
1057     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1058
1059     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1060
1061     $raw .= "lxc.rootfs = $dir/rootfs\n";
1062
1063     my $netcount = 0;
1064     foreach my $k (keys %$conf) {
1065         next if $k !~ m/^net(\d+)$/;
1066         my $ind = $1;
1067         my $d = parse_lxc_network($conf->{$k});
1068         $netcount++;
1069         $raw .= "lxc.network.type = veth\n";
1070         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1071         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1072         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1073         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1074     }
1075
1076     if (my $lxcconf = $conf->{lxc}) {
1077         foreach my $entry (@$lxcconf) {
1078             my ($k, $v) = @$entry;
1079             $netcount++ if $k eq 'lxc.network.type';
1080             $raw .= "$k = $v\n";
1081         }
1082     }
1083
1084     $raw .= "lxc.network.type = empty\n" if !$netcount;
1085
1086     File::Path::mkpath("$dir/rootfs");
1087
1088     PVE::Tools::file_set_contents("$dir/config", $raw);
1089 }
1090
1091 # verify and cleanup nameserver list (replace \0 with ' ')
1092 sub verify_nameserver_list {
1093     my ($nameserver_list) = @_;
1094
1095     my @list = ();
1096     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1097         PVE::JSONSchema::pve_verify_ip($server);
1098         push @list, $server;
1099     }
1100
1101     return join(' ', @list);
1102 }
1103
1104 sub verify_searchdomain_list {
1105     my ($searchdomain_list) = @_;
1106
1107     my @list = ();
1108     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1109         # todo: should we add checks for valid dns domains?
1110         push @list, $server;
1111     }
1112
1113     return join(' ', @list);
1114 }
1115
1116 sub is_volume_in_use {
1117     my ($config, $volid, $include_snapshots) = @_;
1118     my $used = 0;
1119
1120     foreach_mountpoint($config, sub {
1121         my ($ms, $mountpoint) = @_;
1122         return if $used;
1123         if ($mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid) {
1124             $used = 1;
1125         }
1126     });
1127
1128     my $snapshots = $config->{snapshots};
1129     if ($include_snapshots && $snapshots) {
1130         foreach my $snap (keys %$snapshots) {
1131             $used ||= is_volume_in_use($snapshots->{$snap}, $volid);
1132         }
1133     }
1134
1135     return $used;
1136 }
1137
1138 sub add_unused_volume {
1139     my ($config, $volid) = @_;
1140
1141     my $key;
1142     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1143         my $test = "unused$ind";
1144         if (my $vid = $config->{$test}) {
1145             return if $vid eq $volid; # do not add duplicates
1146         } else {
1147             $key = $test;
1148         }
1149     }
1150
1151     die "Too many unused volumes - please delete them first.\n" if !$key;
1152
1153     $config->{$key} = $volid;
1154
1155     return $key;
1156 }
1157
1158 sub update_pct_config {
1159     my ($vmid, $conf, $running, $param, $delete) = @_;
1160
1161     my @nohotplug;
1162
1163     my $new_disks = 0;
1164     my @deleted_volumes;
1165
1166     my $rootdir;
1167     if ($running) {
1168         my $pid = find_lxc_pid($vmid);
1169         $rootdir = "/proc/$pid/root";
1170     }
1171
1172     my $hotplug_error = sub {
1173         if ($running) {
1174             push @nohotplug, @_;
1175             return 1;
1176         } else {
1177             return 0;
1178         }
1179     };
1180
1181     if (defined($delete)) {
1182         foreach my $opt (@$delete) {
1183             if (!exists($conf->{$opt})) {
1184                 warn "no such option: $opt\n";
1185                 next;
1186             }
1187
1188             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1189                 die "unable to delete required option '$opt'\n";
1190             } elsif ($opt eq 'swap') {
1191                 delete $conf->{$opt};
1192                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1193             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1194                 delete $conf->{$opt};
1195             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1196                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1197                 next if $hotplug_error->($opt);
1198                 delete $conf->{$opt};
1199             } elsif ($opt =~ m/^net(\d)$/) {
1200                 delete $conf->{$opt};
1201                 next if !$running;
1202                 my $netid = $1;
1203                 PVE::Network::veth_delete("veth${vmid}i$netid");
1204             } elsif ($opt eq 'protection') {
1205                 delete $conf->{$opt};
1206             } elsif ($opt =~ m/^unused(\d+)$/) {
1207                 next if $hotplug_error->($opt);
1208                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1209                 push @deleted_volumes, $conf->{$opt};
1210                 delete $conf->{$opt};
1211             } elsif ($opt =~ m/^mp(\d+)$/) {
1212                 next if $hotplug_error->($opt);
1213                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1214                 my $mp = parse_ct_mountpoint($conf->{$opt});
1215                 delete $conf->{$opt};
1216                 if ($mp->{type} eq 'volume') {
1217                     add_unused_volume($conf, $mp->{volume});
1218                 }
1219             } elsif ($opt eq 'unprivileged') {
1220                 die "unable to delete read-only option: '$opt'\n";
1221             } else {
1222                 die "implement me (delete: $opt)"
1223             }
1224             PVE::LXC::Config->write_config($vmid, $conf) if $running;
1225         }
1226     }
1227
1228     # There's no separate swap size to configure, there's memory and "total"
1229     # memory (iow. memory+swap). This means we have to change them together.
1230     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1231     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1232     if (defined($wanted_memory) || defined($wanted_swap)) {
1233
1234         my $old_memory = ($conf->{memory} || 512);
1235         my $old_swap = ($conf->{swap} || 0);
1236
1237         $wanted_memory //= $old_memory;
1238         $wanted_swap //= $old_swap;
1239
1240         my $total = $wanted_memory + $wanted_swap;
1241         if ($running) {
1242             my $old_total = $old_memory + $old_swap;
1243             if ($total > $old_total) {
1244                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1245                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1246             } else {
1247                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1248                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1249             }
1250         }
1251         $conf->{memory} = $wanted_memory;
1252         $conf->{swap} = $wanted_swap;
1253
1254         PVE::LXC::Config->write_config($vmid, $conf) if $running;
1255     }
1256
1257     my $used_volids = {};
1258
1259     foreach my $opt (keys %$param) {
1260         my $value = $param->{$opt};
1261         if ($opt eq 'hostname') {
1262             $conf->{$opt} = $value;
1263         } elsif ($opt eq 'onboot') {
1264             $conf->{$opt} = $value ? 1 : 0;
1265         } elsif ($opt eq 'startup') {
1266             $conf->{$opt} = $value;
1267         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1268             next if $hotplug_error->($opt);
1269             $conf->{$opt} = $value;
1270         } elsif ($opt eq 'nameserver') {
1271             next if $hotplug_error->($opt);
1272             my $list = verify_nameserver_list($value);
1273             $conf->{$opt} = $list;
1274         } elsif ($opt eq 'searchdomain') {
1275             next if $hotplug_error->($opt);
1276             my $list = verify_searchdomain_list($value);
1277             $conf->{$opt} = $list;
1278         } elsif ($opt eq 'cpulimit') {
1279             next if $hotplug_error->($opt); # FIXME: hotplug
1280             $conf->{$opt} = $value;
1281         } elsif ($opt eq 'cpuunits') {
1282             $conf->{$opt} = $value;
1283             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1284         } elsif ($opt eq 'description') {
1285             $conf->{$opt} = PVE::Tools::encode_text($value);
1286         } elsif ($opt =~ m/^net(\d+)$/) {
1287             my $netid = $1;
1288             my $net = parse_lxc_network($value);
1289             if (!$running) {
1290                 $conf->{$opt} = print_lxc_network($net);
1291             } else {
1292                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1293             }
1294         } elsif ($opt eq 'protection') {
1295             $conf->{$opt} = $value ? 1 : 0;
1296         } elsif ($opt =~ m/^mp(\d+)$/) {
1297             next if $hotplug_error->($opt);
1298             check_protection($conf, "can't update CT $vmid drive '$opt'");
1299             my $old = $conf->{$opt};
1300             $conf->{$opt} = $value;
1301             if (defined($old)) {
1302                 my $mp = parse_ct_mountpoint($old);
1303                 if ($mp->{type} eq 'volume') {
1304                     add_unused_volume($conf, $mp->{volume});
1305                 }
1306             }
1307             $new_disks = 1;
1308             my $mp = parse_ct_mountpoint($value);
1309             $used_volids->{$mp->{volume}} = 1;
1310         } elsif ($opt eq 'rootfs') {
1311             next if $hotplug_error->($opt);
1312             check_protection($conf, "can't update CT $vmid drive '$opt'");
1313             my $old = $conf->{$opt};
1314             $conf->{$opt} = $value;
1315             if (defined($old)) {
1316                 my $mp = parse_ct_rootfs($old);
1317                 if ($mp->{type} eq 'volume') {
1318                     add_unused_volume($conf, $mp->{volume});
1319                 }
1320             }
1321             my $mp = parse_ct_rootfs($value);
1322             $used_volids->{$mp->{volume}} = 1;
1323         } elsif ($opt eq 'unprivileged') {
1324             die "unable to modify read-only option: '$opt'\n";
1325         } elsif ($opt eq 'ostype') {
1326             next if $hotplug_error->($opt);
1327             $conf->{$opt} = $value;
1328         } else {
1329             die "implement me: $opt";
1330         }
1331         PVE::LXC::Config->write_config($vmid, $conf) if $running;
1332     }
1333
1334     # Apply deletions and creations of new volumes
1335     if (@deleted_volumes) {
1336         my $storage_cfg = PVE::Storage::config();
1337         foreach my $volume (@deleted_volumes) {
1338             next if $used_volids->{$volume}; # could have been re-added, too
1339             # also check for references in snapshots
1340             next if is_volume_in_use($conf, $volume, 1);
1341             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1342         }
1343     }
1344
1345     if ($new_disks) {
1346         my $storage_cfg = PVE::Storage::config();
1347         create_disks($storage_cfg, $vmid, $conf, $conf);
1348     }
1349
1350     # This should be the last thing we do here
1351     if ($running && scalar(@nohotplug)) {
1352         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1353     }
1354 }
1355
1356 sub has_dev_console {
1357     my ($conf) = @_;
1358
1359     return !(defined($conf->{console}) && !$conf->{console});
1360 }
1361
1362 sub get_tty_count {
1363     my ($conf) = @_;
1364
1365     return $conf->{tty} // $confdesc->{tty}->{default};
1366 }
1367
1368 sub get_cmode {
1369     my ($conf) = @_;
1370
1371     return $conf->{cmode} // $confdesc->{cmode}->{default};
1372 }
1373
1374 sub get_console_command {
1375     my ($vmid, $conf) = @_;
1376
1377     my $cmode = get_cmode($conf);
1378
1379     if ($cmode eq 'console') {
1380         return ['lxc-console', '-n',  $vmid, '-t', 0];
1381     } elsif ($cmode eq 'tty') {
1382         return ['lxc-console', '-n',  $vmid];
1383     } elsif ($cmode eq 'shell') {
1384         return ['lxc-attach', '--clear-env', '-n', $vmid];
1385     } else {
1386         die "internal error";
1387     }
1388 }
1389
1390 sub get_primary_ips {
1391     my ($conf) = @_;
1392
1393     # return data from net0
1394
1395     return undef if !defined($conf->{net0});
1396     my $net = parse_lxc_network($conf->{net0});
1397
1398     my $ipv4 = $net->{ip};
1399     if ($ipv4) {
1400         if ($ipv4 =~ /^(dhcp|manual)$/) {
1401             $ipv4 = undef
1402         } else {
1403             $ipv4 =~ s!/\d+$!!;
1404         }
1405     }
1406     my $ipv6 = $net->{ip6};
1407     if ($ipv6) {
1408         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1409             $ipv6 = undef;
1410         } else {
1411             $ipv6 =~ s!/\d+$!!;
1412         }
1413     }
1414
1415     return ($ipv4, $ipv6);
1416 }
1417
1418 sub delete_mountpoint_volume {
1419     my ($storage_cfg, $vmid, $volume) = @_;
1420
1421     return if classify_mountpoint($volume) ne 'volume';
1422
1423     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1424     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1425 }
1426
1427 sub destroy_lxc_container {
1428     my ($storage_cfg, $vmid, $conf) = @_;
1429
1430     foreach_mountpoint($conf, sub {
1431         my ($ms, $mountpoint) = @_;
1432         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1433     });
1434
1435     rmdir "/var/lib/lxc/$vmid/rootfs";
1436     unlink "/var/lib/lxc/$vmid/config";
1437     rmdir "/var/lib/lxc/$vmid";
1438     destroy_config($vmid);
1439
1440     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1441     #PVE::Tools::run_command($cmd);
1442 }
1443
1444 sub vm_stop_cleanup {
1445     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1446
1447     eval {
1448         if (!$keepActive) {
1449
1450             my $vollist = get_vm_volumes($conf);
1451             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1452         }
1453     };
1454     warn $@ if $@; # avoid errors - just warn
1455 }
1456
1457 my $safe_num_ne = sub {
1458     my ($a, $b) = @_;
1459
1460     return 0 if !defined($a) && !defined($b);
1461     return 1 if !defined($a);
1462     return 1 if !defined($b);
1463
1464     return $a != $b;
1465 };
1466
1467 my $safe_string_ne = sub {
1468     my ($a, $b) = @_;
1469
1470     return 0 if !defined($a) && !defined($b);
1471     return 1 if !defined($a);
1472     return 1 if !defined($b);
1473
1474     return $a ne $b;
1475 };
1476
1477 sub update_net {
1478     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1479
1480     if ($newnet->{type} ne 'veth') {
1481         # for when there are physical interfaces
1482         die "cannot update interface of type $newnet->{type}";
1483     }
1484
1485     my $veth = "veth${vmid}i${netid}";
1486     my $eth = $newnet->{name};
1487
1488     if (my $oldnetcfg = $conf->{$opt}) {
1489         my $oldnet = parse_lxc_network($oldnetcfg);
1490
1491         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1492             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1493
1494             PVE::Network::veth_delete($veth);
1495             delete $conf->{$opt};
1496             PVE::LXC::Config->write_config($vmid, $conf);
1497
1498             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1499
1500         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1501                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1502                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1503
1504                 if ($oldnet->{bridge}) {
1505                     PVE::Network::tap_unplug($veth);
1506                     foreach (qw(bridge tag firewall)) {
1507                         delete $oldnet->{$_};
1508                     }
1509                     $conf->{$opt} = print_lxc_network($oldnet);
1510                     PVE::LXC::Config->write_config($vmid, $conf);
1511                 }
1512
1513                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1514                 foreach (qw(bridge tag firewall)) {
1515                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1516                 }
1517                 $conf->{$opt} = print_lxc_network($oldnet);
1518                 PVE::LXC::Config->write_config($vmid, $conf);
1519         }
1520     } else {
1521         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1522     }
1523
1524     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1525 }
1526
1527 sub hotplug_net {
1528     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1529
1530     my $veth = "veth${vmid}i${netid}";
1531     my $vethpeer = $veth . "p";
1532     my $eth = $newnet->{name};
1533
1534     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1535     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1536
1537     # attach peer in container
1538     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1539     PVE::Tools::run_command($cmd);
1540
1541     # link up peer in container
1542     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1543     PVE::Tools::run_command($cmd);
1544
1545     my $done = { type => 'veth' };
1546     foreach (qw(bridge tag firewall hwaddr name)) {
1547         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1548     }
1549     $conf->{$opt} = print_lxc_network($done);
1550
1551     PVE::LXC::Config->write_config($vmid, $conf);
1552 }
1553
1554 sub update_ipconfig {
1555     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1556
1557     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1558
1559     my $optdata = parse_lxc_network($conf->{$opt});
1560     my $deleted = [];
1561     my $added = [];
1562     my $nscmd = sub {
1563         my $cmdargs = shift;
1564         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1565     };
1566     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1567
1568     my $change_ip_config = sub {
1569         my ($ipversion) = @_;
1570
1571         my $family_opt = "-$ipversion";
1572         my $suffix = $ipversion == 4 ? '' : $ipversion;
1573         my $gw= "gw$suffix";
1574         my $ip= "ip$suffix";
1575
1576         my $newip = $newnet->{$ip};
1577         my $newgw = $newnet->{$gw};
1578         my $oldip = $optdata->{$ip};
1579
1580         my $change_ip = &$safe_string_ne($oldip, $newip);
1581         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1582
1583         return if !$change_ip && !$change_gw;
1584
1585         # step 1: add new IP, if this fails we cancel
1586         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1587         if ($change_ip && $is_real_ip) {
1588             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1589             if (my $err = $@) {
1590                 warn $err;
1591                 return;
1592             }
1593         }
1594
1595         # step 2: replace gateway
1596         #   If this fails we delete the added IP and cancel.
1597         #   If it succeeds we save the config and delete the old IP, ignoring
1598         #   errors. The config is then saved.
1599         # Note: 'ip route replace' can add
1600         if ($change_gw) {
1601             if ($newgw) {
1602                 eval {
1603                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1604                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1605                     }
1606                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1607                 };
1608                 if (my $err = $@) {
1609                     warn $err;
1610                     # the route was not replaced, the old IP is still available
1611                     # rollback (delete new IP) and cancel
1612                     if ($change_ip) {
1613                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1614                         warn $@ if $@; # no need to die here
1615                     }
1616                     return;
1617                 }
1618             } else {
1619                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1620                 # if the route was not deleted, the guest might have deleted it manually
1621                 # warn and continue
1622                 warn $@ if $@;
1623             }
1624         }
1625
1626         # from this point on we save the configuration
1627         # step 3: delete old IP ignoring errors
1628         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1629             # We need to enable promote_secondaries, otherwise our newly added
1630             # address will be removed along with the old one.
1631             my $promote = 0;
1632             eval {
1633                 if ($ipversion == 4) {
1634                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1635                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1636                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1637                 }
1638                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1639             };
1640             warn $@ if $@; # no need to die here
1641
1642             if ($ipversion == 4) {
1643                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1644             }
1645         }
1646
1647         foreach my $property ($ip, $gw) {
1648             if ($newnet->{$property}) {
1649                 $optdata->{$property} = $newnet->{$property};
1650             } else {
1651                 delete $optdata->{$property};
1652             }
1653         }
1654         $conf->{$opt} = print_lxc_network($optdata);
1655         PVE::LXC::Config->write_config($vmid, $conf);
1656         $lxc_setup->setup_network($conf);
1657     };
1658
1659     &$change_ip_config(4);
1660     &$change_ip_config(6);
1661
1662 }
1663
1664 my $enter_namespace = sub {
1665     my ($vmid, $pid, $which, $type) = @_;
1666     sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY
1667         or die "failed to open $which namespace of container $vmid: $!\n";
1668     PVE::Tools::setns(fileno($fd), $type)
1669         or die "failed to enter $which namespace of container $vmid: $!\n";
1670     close $fd;
1671 };
1672
1673 my $do_syncfs = sub {
1674     my ($vmid, $pid, $socket) = @_;
1675
1676     &$enter_namespace($vmid, $pid, 'mnt', PVE::Tools::CLONE_NEWNS);
1677
1678     # Tell the parent process to start reading our /proc/mounts
1679     print {$socket} "go\n";
1680     $socket->flush();
1681
1682     # Receive /proc/self/mounts
1683     my $mountdata = do { local $/ = undef; <$socket> };
1684     close $socket;
1685
1686     # Now sync all mountpoints...
1687     my $mounts = PVE::ProcFSTools::parse_mounts($mountdata);
1688     foreach my $mp (@$mounts) {
1689         my ($what, $dir, $fs) = @$mp;
1690         next if $fs eq 'fuse.lxcfs';
1691         eval { PVE::Tools::sync_mountpoint($dir); };
1692         warn $@ if $@;
1693     }
1694 };
1695
1696 sub sync_container_namespace {
1697     my ($vmid) = @_;
1698     my $pid = find_lxc_pid($vmid);
1699
1700     # SOCK_DGRAM is nicer for barriers but cannot be slurped
1701     socketpair my $pfd, my $cfd, AF_UNIX, SOCK_STREAM, PF_UNSPEC
1702         or die "failed to create socketpair: $!\n";
1703
1704     my $child = fork();
1705     die "fork failed: $!\n" if !defined($child);
1706
1707     if (!$child) {
1708         eval {
1709             close $pfd;
1710             &$do_syncfs($vmid, $pid, $cfd);
1711         };
1712         if (my $err = $@) {
1713             warn $err;
1714             POSIX::_exit(1);
1715         }
1716         POSIX::_exit(0);
1717     }
1718     close $cfd;
1719     my $go = <$pfd>;
1720     die "failed to enter container namespace\n" if $go ne "go\n";
1721
1722     open my $mounts, '<', "/proc/$child/mounts"
1723         or die "failed to open container's /proc/mounts: $!\n";
1724     my $mountdata = do { local $/ = undef; <$mounts> };
1725     close $mounts;
1726     print {$pfd} $mountdata;
1727     close $pfd;
1728
1729     while (waitpid($child, 0) != $child) {}
1730     die "failed to sync container namespace\n" if $? != 0;
1731 }
1732
1733 sub template_create {
1734     my ($vmid, $conf) = @_;
1735
1736     my $storecfg = PVE::Storage::config();
1737
1738     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1739     my $volid = $rootinfo->{volume};
1740
1741     die "Template feature is not available for '$volid'\n"
1742         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1743
1744     PVE::Storage::activate_volumes($storecfg, [$volid]);
1745
1746     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1747     $rootinfo->{volume} = $template_volid;
1748     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1749
1750     PVE::LXC::Config->write_config($vmid, $conf);
1751 }
1752
1753 sub mountpoint_names {
1754     my ($reverse) = @_;
1755
1756     my @names = ('rootfs');
1757
1758     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1759         push @names, "mp$i";
1760     }
1761
1762     return $reverse ? reverse @names : @names;
1763 }
1764
1765
1766 sub foreach_mountpoint_full {
1767     my ($conf, $reverse, $func) = @_;
1768
1769     foreach my $key (mountpoint_names($reverse)) {
1770         my $value = $conf->{$key};
1771         next if !defined($value);
1772         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
1773         next if !defined($mountpoint);
1774
1775         &$func($key, $mountpoint);
1776     }
1777 }
1778
1779 sub foreach_mountpoint {
1780     my ($conf, $func) = @_;
1781
1782     foreach_mountpoint_full($conf, 0, $func);
1783 }
1784
1785 sub foreach_mountpoint_reverse {
1786     my ($conf, $func) = @_;
1787
1788     foreach_mountpoint_full($conf, 1, $func);
1789 }
1790
1791 sub check_ct_modify_config_perm {
1792     my ($rpcenv, $authuser, $vmid, $pool, $newconf, $delete) = @_;
1793
1794     return 1 if $authuser eq 'root@pam';
1795
1796     my $check = sub {
1797         my ($opt, $delete) = @_;
1798         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1799             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1800         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1801             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1802             return if $delete;
1803             my $data = $opt eq 'rootfs' ? parse_ct_rootfs($newconf->{$opt})
1804                                         : parse_ct_mountpoint($newconf->{$opt});
1805             raise_perm_exc("mountpoint type $data->{type}") if $data->{type} ne 'volume';
1806         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1807             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1808         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1809                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1810             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1811         } else {
1812             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1813         }
1814     };
1815
1816     foreach my $opt (keys %$newconf) {
1817         &$check($opt, 0);
1818     }
1819     foreach my $opt (@$delete) {
1820         &$check($opt, 1);
1821     }
1822
1823     return 1;
1824 }
1825
1826 sub umount_all {
1827     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
1828
1829     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1830     my $volid_list = get_vm_volumes($conf);
1831
1832     foreach_mountpoint_reverse($conf, sub {
1833         my ($ms, $mountpoint) = @_;
1834
1835         my $volid = $mountpoint->{volume};
1836         my $mount = $mountpoint->{mp};
1837
1838         return if !$volid || !$mount;
1839
1840         my $mount_path = "$rootdir/$mount";
1841         $mount_path =~ s!/+!/!g;
1842
1843         return if !PVE::ProcFSTools::is_mounted($mount_path);
1844
1845         eval {
1846             PVE::Tools::run_command(['umount', '-d', $mount_path]);
1847         };
1848         if (my $err = $@) {
1849             if ($noerr) {
1850                 warn $err;
1851             } else {
1852                 die $err;
1853             }
1854         }
1855     });
1856 }
1857
1858 sub mount_all {
1859     my ($vmid, $storage_cfg, $conf) = @_;
1860
1861     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
1862     File::Path::make_path($rootdir);
1863
1864     my $volid_list = get_vm_volumes($conf);
1865     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
1866
1867     eval {
1868         foreach_mountpoint($conf, sub {
1869             my ($ms, $mountpoint) = @_;
1870
1871             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
1872         });
1873     };
1874     if (my $err = $@) {
1875         warn "mounting container failed\n";
1876         umount_all($vmid, $storage_cfg, $conf, 1);
1877         die $err;
1878     }
1879
1880     return $rootdir;
1881 }
1882
1883
1884 sub mountpoint_mount_path {
1885     my ($mountpoint, $storage_cfg, $snapname) = @_;
1886
1887     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
1888 }
1889
1890 my $check_mount_path = sub {
1891     my ($path) = @_;
1892     $path = File::Spec->canonpath($path);
1893     my $real = Cwd::realpath($path);
1894     if ($real ne $path) {
1895         die "mount path modified by symlink: $path != $real";
1896     }
1897 };
1898
1899 sub query_loopdev {
1900     my ($path) = @_;
1901     my $found;
1902     my $parser = sub {
1903         my $line = shift;
1904         if ($line =~ m@^(/dev/loop\d+):@) {
1905             $found = $1;
1906         }
1907     };
1908     my $cmd = ['losetup', '--associated', $path];
1909     PVE::Tools::run_command($cmd, outfunc => $parser);
1910     return $found;
1911 }
1912
1913 # Run a function with a file attached to a loop device.
1914 # The loop device is always detached afterwards (or set to autoclear).
1915 # Returns the loop device.
1916 sub run_with_loopdev {
1917     my ($func, $file) = @_;
1918     my $device = query_loopdev($file);
1919     # Try to reuse an existing device
1920     if ($device) {
1921         # We assume that whoever setup the loop device is responsible for
1922         # detaching it.
1923         &$func($device);
1924         return $device;
1925     }
1926
1927     my $parser = sub {
1928         my $line = shift;
1929         if ($line =~ m@^(/dev/loop\d+)$@) {
1930             $device = $1;
1931         }
1932     };
1933     PVE::Tools::run_command(['losetup', '--show', '-f', $file], outfunc => $parser);
1934     die "failed to setup loop device for $file\n" if !$device;
1935     eval { &$func($device); };
1936     my $err = $@;
1937     PVE::Tools::run_command(['losetup', '-d', $device]);
1938     die $err if $err;
1939     return $device;
1940 }
1941
1942 sub bindmount {
1943     my ($dir, $dest, $ro, @extra_opts) = @_;
1944     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $dir, $dest]);
1945     if ($ro) {
1946         eval { PVE::Tools::run_command(['mount', '-o', 'bind,remount,ro', $dest]); };
1947         if (my $err = $@) {
1948             warn "bindmount error\n";
1949             # don't leave writable bind-mounts behind...
1950             PVE::Tools::run_command(['umount', $dest]);
1951             die $err;
1952         }
1953     }
1954 }
1955
1956 # use $rootdir = undef to just return the corresponding mount path
1957 sub mountpoint_mount {
1958     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
1959
1960     my $volid = $mountpoint->{volume};
1961     my $mount = $mountpoint->{mp};
1962     my $type = $mountpoint->{type};
1963     my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota};
1964     my $mounted_dev;
1965
1966     return if !$volid || !$mount;
1967
1968     my $mount_path;
1969
1970     if (defined($rootdir)) {
1971         $rootdir =~ s!/+$!!;
1972         $mount_path = "$rootdir/$mount";
1973         $mount_path =~ s!/+!/!g;
1974         &$check_mount_path($mount_path);
1975         File::Path::mkpath($mount_path);
1976     }
1977
1978     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1979
1980     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
1981
1982     my $optstring = '';
1983     if (defined($mountpoint->{acl})) {
1984         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
1985     }
1986     my $readonly = $mountpoint->{ro};
1987
1988     my @extra_opts = ('-o', $optstring);
1989
1990     if ($storage) {
1991
1992         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1993         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
1994
1995         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1996             PVE::Storage::parse_volname($storage_cfg, $volid);
1997
1998         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
1999
2000         if ($format eq 'subvol') {
2001             if ($mount_path) {
2002                 if ($snapname) {
2003                     if ($scfg->{type} eq 'zfspool') {
2004                         my $path_arg = $path;
2005                         $path_arg =~ s!^/+!!;
2006                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
2007                     } else {
2008                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2009                     }
2010                 } else {
2011                     bindmount($path, $mount_path, $readonly, @extra_opts);
2012                     warn "cannot enable quota control for bind mounted subvolumes\n" if $quota;
2013                 }
2014             }
2015             return wantarray ? ($path, 0, $mounted_dev) : $path;
2016         } elsif ($format eq 'raw' || $format eq 'iso') {
2017             my $domount = sub {
2018                 my ($path) = @_;
2019                 if ($mount_path) {
2020                     if ($format eq 'iso') {
2021                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2022                     } elsif ($isBase || defined($snapname)) {
2023                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2024                     } else {
2025                         if ($quota) {
2026                             push @extra_opts, '-o', 'usrjquota=aquota.user,grpjquota=aquota.group,jqfmt=vfsv0';
2027                         }
2028                         push @extra_opts, '-o', 'ro' if $readonly;
2029                         PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2030                     }
2031                 }
2032             };
2033             my $use_loopdev = 0;
2034             if ($scfg->{path}) {
2035                 $mounted_dev = run_with_loopdev($domount, $path);
2036                 $use_loopdev = 1;
2037             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2038                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2039                 $mounted_dev = $path;
2040                 &$domount($path);
2041             } else {
2042                 die "unsupported storage type '$scfg->{type}'\n";
2043             }
2044             return wantarray ? ($path, $use_loopdev, $mounted_dev) : $path;
2045         } else {
2046             die "unsupported image format '$format'\n";
2047         }
2048     } elsif ($type eq 'device') {
2049                         push @extra_opts, '-o', 'ro' if $readonly;
2050         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
2051         return wantarray ? ($volid, 0, $volid) : $volid;
2052     } elsif ($type eq 'bind') {
2053         die "directory '$volid' does not exist\n" if ! -d $volid;
2054         &$check_mount_path($volid);
2055         bindmount($volid, $mount_path, $readonly, @extra_opts) if $mount_path;
2056         warn "cannot enable quota control for bind mounts\n" if $quota;
2057         return wantarray ? ($volid, 0, undef) : $volid;
2058     }
2059
2060     die "unsupported storage";
2061 }
2062
2063 sub get_vm_volumes {
2064     my ($conf, $excludes) = @_;
2065
2066     my $vollist = [];
2067
2068     foreach_mountpoint($conf, sub {
2069         my ($ms, $mountpoint) = @_;
2070
2071         return if $excludes && $ms eq $excludes;
2072
2073         my $volid = $mountpoint->{volume};
2074
2075         return if !$volid || $mountpoint->{type} ne 'volume';
2076
2077         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2078         return if !$sid;
2079
2080         push @$vollist, $volid;
2081     });
2082
2083     return $vollist;
2084 }
2085
2086 sub mkfs {
2087     my ($dev, $rootuid, $rootgid) = @_;
2088
2089     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2090                              '-E', "root_owner=$rootuid:$rootgid",
2091                              $dev]);
2092 }
2093
2094 sub format_disk {
2095     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2096
2097     if ($volid =~ m!^/dev/.+!) {
2098         mkfs($volid);
2099         return;
2100     }
2101
2102     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2103
2104     die "cannot format volume '$volid' with no storage\n" if !$storage;
2105
2106     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2107
2108     my $path = PVE::Storage::path($storage_cfg, $volid);
2109
2110     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2111         PVE::Storage::parse_volname($storage_cfg, $volid);
2112
2113     die "cannot format volume '$volid' (format == $format)\n"
2114         if $format ne 'raw';
2115
2116     mkfs($path, $rootuid, $rootgid);
2117 }
2118
2119 sub destroy_disks {
2120     my ($storecfg, $vollist) = @_;
2121
2122     foreach my $volid (@$vollist) {
2123         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2124         warn $@ if $@;
2125     }
2126 }
2127
2128 sub create_disks {
2129     my ($storecfg, $vmid, $settings, $conf) = @_;
2130
2131     my $vollist = [];
2132
2133     eval {
2134         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2135         my $chown_vollist = [];
2136
2137         foreach_mountpoint($settings, sub {
2138             my ($ms, $mountpoint) = @_;
2139
2140             my $volid = $mountpoint->{volume};
2141             my $mp = $mountpoint->{mp};
2142
2143             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2144
2145             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2146                 my ($storeid, $size_gb) = ($1, $2);
2147
2148                 my $size_kb = int(${size_gb}*1024) * 1024;
2149
2150                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2151                 # fixme: use better naming ct-$vmid-disk-X.raw?
2152
2153                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2154                     if ($size_kb > 0) {
2155                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2156                                                            undef, $size_kb);
2157                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2158                     } else {
2159                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2160                                                            undef, 0);
2161                         push @$chown_vollist, $volid;
2162                     }
2163                 } elsif ($scfg->{type} eq 'zfspool') {
2164
2165                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2166                                                        undef, $size_kb);
2167                     push @$chown_vollist, $volid;
2168                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2169
2170                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2171                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2172
2173                 } elsif ($scfg->{type} eq 'rbd') {
2174
2175                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2176                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2177                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2178                 } else {
2179                     die "unable to create containers on storage type '$scfg->{type}'\n";
2180                 }
2181                 push @$vollist, $volid;
2182                 $mountpoint->{volume} = $volid;
2183                 $mountpoint->{size} = $size_kb * 1024;
2184                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2185             } else {
2186                 # use specified/existing volid/dir/device
2187                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2188             }
2189         });
2190
2191         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2192         foreach my $volid (@$chown_vollist) {
2193             my $path = PVE::Storage::path($storecfg, $volid, undef);
2194             chown($rootuid, $rootgid, $path);
2195         }
2196         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2197     };
2198     # free allocated images on error
2199     if (my $err = $@) {
2200         destroy_disks($storecfg, $vollist);
2201         die $err;
2202     }
2203     return $vollist;
2204 }
2205
2206 # bash completion helper
2207
2208 sub complete_os_templates {
2209     my ($cmdname, $pname, $cvalue) = @_;
2210
2211     my $cfg = PVE::Storage::config();
2212
2213     my $storeid;
2214
2215     if ($cvalue =~ m/^([^:]+):/) {
2216         $storeid = $1;
2217     }
2218
2219     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2220     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2221
2222     my $res = [];
2223     foreach my $id (keys %$data) {
2224         foreach my $item (@{$data->{$id}}) {
2225             push @$res, $item->{volid} if defined($item->{volid});
2226         }
2227     }
2228
2229     return $res;
2230 }
2231
2232 my $complete_ctid_full = sub {
2233     my ($running) = @_;
2234
2235     my $idlist = vmstatus();
2236
2237     my $active_hash = list_active_containers();
2238
2239     my $res = [];
2240
2241     foreach my $id (keys %$idlist) {
2242         my $d = $idlist->{$id};
2243         if (defined($running)) {
2244             next if $d->{template};
2245             next if $running && !$active_hash->{$id};
2246             next if !$running && $active_hash->{$id};
2247         }
2248         push @$res, $id;
2249
2250     }
2251     return $res;
2252 };
2253
2254 sub complete_ctid {
2255     return &$complete_ctid_full();
2256 }
2257
2258 sub complete_ctid_stopped {
2259     return &$complete_ctid_full(0);
2260 }
2261
2262 sub complete_ctid_running {
2263     return &$complete_ctid_full(1);
2264 }
2265
2266 sub parse_id_maps {
2267     my ($conf) = @_;
2268
2269     my $id_map = [];
2270     my $rootuid = 0;
2271     my $rootgid = 0;
2272
2273     my $lxc = $conf->{lxc};
2274     foreach my $entry (@$lxc) {
2275         my ($key, $value) = @$entry;
2276         next if $key ne 'lxc.id_map';
2277         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2278             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2279             push @$id_map, [$type, $ct, $host, $length];
2280             if ($ct == 0) {
2281                 $rootuid = $host if $type eq 'u';
2282                 $rootgid = $host if $type eq 'g';
2283             }
2284         } else {
2285             die "failed to parse id_map: $value\n";
2286         }
2287     }
2288
2289     if (!@$id_map && $conf->{unprivileged}) {
2290         # Should we read them from /etc/subuid?
2291         $id_map = [ ['u', '0', '100000', '65536'],
2292                     ['g', '0', '100000', '65536'] ];
2293         $rootuid = $rootgid = 100000;
2294     }
2295
2296     return ($id_map, $rootuid, $rootgid);
2297 }
2298
2299 sub userns_command {
2300     my ($id_map) = @_;
2301     if (@$id_map) {
2302         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2303     }
2304     return [];
2305 }
2306
2307
2308 1;