src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use Socket;
   8
   9 use File::Path;
  10 use File::Spec;
  11 use Cwd qw();
  12 use Fcntl qw(O_RDONLY);
  13
  14 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  15 use PVE::Exception qw(raise_perm_exc);
  16 use PVE::Storage;
  17 use PVE::SafeSyslog;
  18 use PVE::INotify;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full);
  21 use PVE::Network;
  22 use PVE::AccessControl;
  23 use PVE::ProcFSTools;
  24 use Time::HiRes qw (gettimeofday);
  25
  26 use Data::Dumper;
  27
  28 my $nodename = PVE::INotify::nodename();
  29
  30 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  31
  32 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  33                           '--xattrs',
  34                           '--xattrs-include=user.*',
  35                           '--xattrs-include=security.capability',
  36                           '--warning=no-xattr-write' ];
  37
  38 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  39
  40 my $rootfs_desc = {
  41     volume => {
  42         type => 'string',
  43         default_key => 1,
  44         format => 'pve-lxc-mp-string',
  45         format_description => 'volume',
  46         description => 'Volume, device or directory to mount into the container.',
  47     },
  48     backup => {
  49         type => 'boolean',
  50         format_description => '[1|0]',
  51         description => 'Whether to include the mountpoint in backups.',
  52         optional => 1,
  53     },
  54     size => {
  55         type => 'string',
  56         format => 'disk-size',
  57         format_description => 'DiskSize',
  58         description => 'Volume size (read only value).',
  59         optional => 1,
  60     },
  61     acl => {
  62         type => 'boolean',
  63         format_description => 'acl',
  64         description => 'Explicitly enable or disable ACL support.',
  65         optional => 1,
  66     },
  67     ro => {
  68         type => 'boolean',
  69         format_description => 'ro',
  70         description => 'Read-only mountpoint (not supported with bind mounts)',
  71         optional => 1,
  72     },
  73     quota => {
  74         type => 'boolean',
  75         format_description => '[0|1]',
  76         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
  77         optional => 1,
  78     },
  79 };
  80
  81 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  82     type => 'string', format => $rootfs_desc,
  83     description => "Use volume as container root.",
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  88     description => "The name of the snapshot.",
  89     type => 'string', format => 'pve-configid',
  90     maxLength => 40,
  91 });
  92
  93 my $confdesc = {
  94     lock => {
  95         optional => 1,
  96         type => 'string',
  97         description => "Lock/unlock the VM.",
  98         enum => [qw(migrate backup snapshot rollback)],
  99     },
 100     onboot => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Specifies whether a VM will be started during system bootup.",
 104         default => 0,
 105     },
 106     startup => get_standard_option('pve-startup-order'),
 107     template => {
 108         optional => 1,
 109         type => 'boolean',
 110         description => "Enable/disable Template.",
 111         default => 0,
 112     },
 113     arch => {
 114         optional => 1,
 115         type => 'string',
 116         enum => ['amd64', 'i386'],
 117         description => "OS architecture type.",
 118         default => 'amd64',
 119     },
 120     ostype => {
 121         optional => 1,
 122         type => 'string',
 123         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux', 'alpine', 'unmanaged'],
 124         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 125     },
 126     console => {
 127         optional => 1,
 128         type => 'boolean',
 129         description => "Attach a console device (/dev/console) to the container.",
 130         default => 1,
 131     },
 132     tty => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "Specify the number of tty available to the container",
 136         minimum => 0,
 137         maximum => 6,
 138         default => 2,
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1024,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     swap => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of SWAP for the VM in MB.",
 167         minimum => 0,
 168         default => 512,
 169     },
 170     hostname => {
 171         optional => 1,
 172         description => "Set a host name for the container.",
 173         type => 'string', format => 'dns-name',
 174         maxLength => 255,
 175     },
 176     description => {
 177         optional => 1,
 178         type => 'string',
 179         description => "Container description. Only used on the configuration web interface.",
 180     },
 181     searchdomain => {
 182         optional => 1,
 183         type => 'string', format => 'dns-name-list',
 184         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 185     },
 186     nameserver => {
 187         optional => 1,
 188         type => 'string', format => 'address-list',
 189         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 190     },
 191     rootfs => get_standard_option('pve-ct-rootfs'),
 192     parent => {
 193         optional => 1,
 194         type => 'string', format => 'pve-configid',
 195         maxLength => 40,
 196         description => "Parent snapshot name. This is used internally, and should not be modified.",
 197     },
 198     snaptime => {
 199         optional => 1,
 200         description => "Timestamp for snapshots.",
 201         type => 'integer',
 202         minimum => 0,
 203     },
 204     cmode => {
 205         optional => 1,
 206         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 207         type => 'string',
 208         enum => ['shell', 'console', 'tty'],
 209         default => 'tty',
 210     },
 211     protection => {
 212         optional => 1,
 213         type => 'boolean',
 214         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 215         default => 0,
 216     },
 217     unprivileged => {
 218         optional => 1,
 219         type => 'boolean',
 220         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 221         default => 0,
 222     },
 223 };
 224
 225 my $valid_lxc_conf_keys = {
 226     'lxc.include' => 1,
 227     'lxc.arch' => 1,
 228     'lxc.utsname' => 1,
 229     'lxc.haltsignal' => 1,
 230     'lxc.rebootsignal' => 1,
 231     'lxc.stopsignal' => 1,
 232     'lxc.init_cmd' => 1,
 233     'lxc.network.type' => 1,
 234     'lxc.network.flags' => 1,
 235     'lxc.network.link' => 1,
 236     'lxc.network.mtu' => 1,
 237     'lxc.network.name' => 1,
 238     'lxc.network.hwaddr' => 1,
 239     'lxc.network.ipv4' => 1,
 240     'lxc.network.ipv4.gateway' => 1,
 241     'lxc.network.ipv6' => 1,
 242     'lxc.network.ipv6.gateway' => 1,
 243     'lxc.network.script.up' => 1,
 244     'lxc.network.script.down' => 1,
 245     'lxc.pts' => 1,
 246     'lxc.console.logfile' => 1,
 247     'lxc.console' => 1,
 248     'lxc.tty' => 1,
 249     'lxc.devttydir' => 1,
 250     'lxc.hook.autodev' => 1,
 251     'lxc.autodev' => 1,
 252     'lxc.kmsg' => 1,
 253     'lxc.mount' => 1,
 254     'lxc.mount.entry' => 1,
 255     'lxc.mount.auto' => 1,
 256     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 257     'lxc.rootfs.mount' => 1,
 258     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 259                             ', please use mountpoint options in the "rootfs" key',
 260     # lxc.cgroup.*
 261     'lxc.cap.drop' => 1,
 262     'lxc.cap.keep' => 1,
 263     'lxc.aa_profile' => 1,
 264     'lxc.aa_allow_incomplete' => 1,
 265     'lxc.se_context' => 1,
 266     'lxc.seccomp' => 1,
 267     'lxc.id_map' => 1,
 268     'lxc.hook.pre-start' => 1,
 269     'lxc.hook.pre-mount' => 1,
 270     'lxc.hook.mount' => 1,
 271     'lxc.hook.start' => 1,
 272     'lxc.hook.stop' => 1,
 273     'lxc.hook.post-stop' => 1,
 274     'lxc.hook.clone' => 1,
 275     'lxc.hook.destroy' => 1,
 276     'lxc.loglevel' => 1,
 277     'lxc.logfile' => 1,
 278     'lxc.start.auto' => 1,
 279     'lxc.start.delay' => 1,
 280     'lxc.start.order' => 1,
 281     'lxc.group' => 1,
 282     'lxc.environment' => 1,
 283 };
 284
 285 my $netconf_desc = {
 286     type => {
 287         type => 'string',
 288         optional => 1,
 289         description => "Network interface type.",
 290         enum => [qw(veth)],
 291     },
 292     name => {
 293         type => 'string',
 294         format_description => 'String',
 295         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 296         pattern => '[-_.\w\d]+',
 297     },
 298     bridge => {
 299         type => 'string',
 300         format_description => 'vmbr<Number>',
 301         description => 'Bridge to attach the network device to.',
 302         pattern => '[-_.\w\d]+',
 303         optional => 1,
 304     },
 305     hwaddr => {
 306         type => 'string',
 307         format_description => 'MAC',
 308         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 309         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 310         optional => 1,
 311     },
 312     mtu => {
 313         type => 'integer',
 314         format_description => 'Number',
 315         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 316         minimum => 64, # minimum ethernet frame is 64 bytes
 317         optional => 1,
 318     },
 319     ip => {
 320         type => 'string',
 321         format => 'pve-ipv4-config',
 322         format_description => 'IPv4Format/CIDR',
 323         description => 'IPv4 address in CIDR format.',
 324         optional => 1,
 325     },
 326     gw => {
 327         type => 'string',
 328         format => 'ipv4',
 329         format_description => 'GatewayIPv4',
 330         description => 'Default gateway for IPv4 traffic.',
 331         optional => 1,
 332     },
 333     ip6 => {
 334         type => 'string',
 335         format => 'pve-ipv6-config',
 336         format_description => 'IPv6Format/CIDR',
 337         description => 'IPv6 address in CIDR format.',
 338         optional => 1,
 339     },
 340     gw6 => {
 341         type => 'string',
 342         format => 'ipv6',
 343         format_description => 'GatewayIPv6',
 344         description => 'Default gateway for IPv6 traffic.',
 345         optional => 1,
 346     },
 347     firewall => {
 348         type => 'boolean',
 349         format_description => '[1|0]',
 350         description => "Controls whether this interface's firewall rules should be used.",
 351         optional => 1,
 352     },
 353     tag => {
 354         type => 'integer',
 355         format_description => 'VlanNo',
 356         minimum => '2',
 357         maximum => '4094',
 358         description => "VLAN tag for this interface.",
 359         optional => 1,
 360     },
 361     trunks => {
 362         type => 'string',
 363         pattern => qr/\d+(?:;\d+)*/,
 364         format_description => 'vlanid[;vlanid...]',
 365         description => "VLAN ids to pass through the interface",
 366         optional => 1,
 367     },
 368 };
 369 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 370
 371 my $MAX_LXC_NETWORKS = 10;
 372 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 373     $confdesc->{"net$i"} = {
 374         optional => 1,
 375         type => 'string', format => $netconf_desc,
 376         description => "Specifies network interfaces for the container.",
 377     };
 378 }
 379
 380 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 381 sub verify_lxc_mp_string{
 382     my ($mp, $noerr) = @_;
 383
 384     # do not allow:
 385     # /./ or /../
 386     # /. or /.. at the end
 387     # ../ at the beginning
 388
 389     if($mp =~ m@/\.\.?/@ ||
 390        $mp =~ m@/\.\.?$@ ||
 391        $mp =~ m@^\.\./@){
 392         return undef if $noerr;
 393         die "$mp contains illegal character sequences\n";
 394     }
 395     return $mp;
 396 }
 397
 398 my $mp_desc = {
 399     %$rootfs_desc,
 400     mp => {
 401         type => 'string',
 402         format => 'pve-lxc-mp-string',
 403         format_description => 'Path',
 404         description => 'Path to the mountpoint as seen from inside the container.',
 405     },
 406 };
 407 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 408
 409 my $unuseddesc = {
 410     optional => 1,
 411     type => 'string', format => 'pve-volume-id',
 412     description => "Reference to unused volumes.",
 413 };
 414
 415 my $MAX_MOUNT_POINTS = 10;
 416 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 417     $confdesc->{"mp$i"} = {
 418         optional => 1,
 419         type => 'string', format => $mp_desc,
 420         description => "Use volume as container mount point (experimental feature).",
 421         optional => 1,
 422     };
 423 }
 424
 425 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 426 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 427     $confdesc->{"unused$i"} = $unuseddesc;
 428 }
 429
 430 sub write_pct_config {
 431     my ($filename, $conf) = @_;
 432
 433     delete $conf->{snapstate}; # just to be sure
 434
 435     my $generate_raw_config = sub {
 436         my ($conf) = @_;
 437
 438         my $raw = '';
 439
 440         # add description as comment to top of file
 441         my $descr = $conf->{description} || '';
 442         foreach my $cl (split(/\n/, $descr)) {
 443             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 444         }
 445
 446         foreach my $key (sort keys %$conf) {
 447             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 448                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 449             my $value = $conf->{$key};
 450             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 451             $raw .= "$key: $value\n";
 452         }
 453
 454         if (my $lxcconf = $conf->{lxc}) {
 455             foreach my $entry (@$lxcconf) {
 456                 my ($k, $v) = @$entry;
 457                 $raw .= "$k: $v\n";
 458             }
 459         }
 460
 461         return $raw;
 462     };
 463
 464     my $raw = &$generate_raw_config($conf);
 465
 466     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 467         $raw .= "\n[$snapname]\n";
 468         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 469     }
 470
 471     return $raw;
 472 }
 473
 474 sub check_type {
 475     my ($key, $value) = @_;
 476
 477     die "unknown setting '$key'\n" if !$confdesc->{$key};
 478
 479     my $type = $confdesc->{$key}->{type};
 480
 481     if (!defined($value)) {
 482         die "got undefined value\n";
 483     }
 484
 485     if ($value =~ m/[\n\r]/) {
 486         die "property contains a line feed\n";
 487     }
 488
 489     if ($type eq 'boolean') {
 490         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 491         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 492         die "type check ('boolean') failed - got '$value'\n";
 493     } elsif ($type eq 'integer') {
 494         return int($1) if $value =~ m/^(\d+)$/;
 495         die "type check ('integer') failed - got '$value'\n";
 496     } elsif ($type eq 'number') {
 497         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 498         die "type check ('number') failed - got '$value'\n";
 499     } elsif ($type eq 'string') {
 500         if (my $fmt = $confdesc->{$key}->{format}) {
 501             PVE::JSONSchema::check_format($fmt, $value);
 502             return $value;
 503         }
 504         return $value;
 505     } else {
 506         die "internal error"
 507     }
 508 }
 509
 510 sub parse_pct_config {
 511     my ($filename, $raw) = @_;
 512
 513     return undef if !defined($raw);
 514
 515     my $res = {
 516         digest => Digest::SHA::sha1_hex($raw),
 517         snapshots => {},
 518     };
 519
 520     $filename =~ m|/lxc/(\d+).conf$|
 521         || die "got strange filename '$filename'";
 522
 523     my $vmid = $1;
 524
 525     my $conf = $res;
 526     my $descr = '';
 527     my $section = '';
 528
 529     my @lines = split(/\n/, $raw);
 530     foreach my $line (@lines) {
 531         next if $line =~ m/^\s*$/;
 532
 533         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 534             $section = $1;
 535             $conf->{description} = $descr if $descr;
 536             $descr = '';
 537             $conf = $res->{snapshots}->{$section} = {};
 538             next;
 539         }
 540
 541         if ($line =~ m/^\#(.*)\s*$/) {
 542             $descr .= PVE::Tools::decode_text($1) . "\n";
 543             next;
 544         }
 545
 546         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 547             my $key = $1;
 548             my $value = $3;
 549             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 550             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 551                 push @{$conf->{lxc}}, [$key, $value];
 552             } elsif (my $errmsg = $validity) {
 553                 warn "vm $vmid - $key: $errmsg\n";
 554             } else {
 555                 warn "vm $vmid - unable to parse config: $line\n";
 556             }
 557         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 558             $descr .= PVE::Tools::decode_text($2);
 559         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 560             $conf->{snapstate} = $1;
 561         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 562             my $key = $1;
 563             my $value = $2;
 564             eval { $value = check_type($key, $value); };
 565             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 566             $conf->{$key} = $value;
 567         } else {
 568             warn "vm $vmid - unable to parse config: $line\n";
 569         }
 570     }
 571
 572     $conf->{description} = $descr if $descr;
 573
 574     delete $res->{snapstate}; # just to be sure
 575
 576     return $res;
 577 }
 578
 579 sub config_list {
 580     my $vmlist = PVE::Cluster::get_vmlist();
 581     my $res = {};
 582     return $res if !$vmlist || !$vmlist->{ids};
 583     my $ids = $vmlist->{ids};
 584
 585     foreach my $vmid (keys %$ids) {
 586         next if !$vmid; # skip CT0
 587         my $d = $ids->{$vmid};
 588         next if !$d->{node} || $d->{node} ne $nodename;
 589         next if !$d->{type} || $d->{type} ne 'lxc';
 590         $res->{$vmid}->{type} = 'lxc';
 591     }
 592     return $res;
 593 }
 594
 595 sub cfs_config_path {
 596     my ($vmid, $node) = @_;
 597
 598     $node = $nodename if !$node;
 599     return "nodes/$node/lxc/$vmid.conf";
 600 }
 601
 602 sub config_file {
 603     my ($vmid, $node) = @_;
 604
 605     my $cfspath = cfs_config_path($vmid, $node);
 606     return "/etc/pve/$cfspath";
 607 }
 608
 609 sub load_config {
 610     my ($vmid, $node) = @_;
 611
 612     $node = $nodename if !$node;
 613     my $cfspath = cfs_config_path($vmid, $node);
 614
 615     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 616     die "container $vmid does not exist\n" if !defined($conf);
 617
 618     return $conf;
 619 }
 620
 621 sub create_config {
 622     my ($vmid, $conf) = @_;
 623
 624     my $dir = "/etc/pve/nodes/$nodename/lxc";
 625     mkdir $dir;
 626
 627     write_config($vmid, $conf);
 628 }
 629
 630 sub destroy_config {
 631     my ($vmid) = @_;
 632
 633     unlink config_file($vmid, $nodename);
 634 }
 635
 636 sub write_config {
 637     my ($vmid, $conf) = @_;
 638
 639     my $cfspath = cfs_config_path($vmid);
 640
 641     PVE::Cluster::cfs_write_file($cfspath, $conf);
 642 }
 643
 644 # flock: we use one file handle per process, so lock file
 645 # can be called multiple times and will succeed for the same process.
 646
 647 my $lock_handles =  {};
 648 my $lockdir = "/run/lock/lxc";
 649
 650 sub config_file_lock {
 651     my ($vmid) = @_;
 652
 653     return "$lockdir/pve-config-${vmid}.lock";
 654 }
 655
 656 sub lock_config_full {
 657     my ($vmid, $timeout, $code, @param) = @_;
 658
 659     my $filename = config_file_lock($vmid);
 660
 661     mkdir $lockdir if !-d $lockdir;
 662
 663     my $res = lock_file($filename, $timeout, $code, @param);
 664
 665     die $@ if $@;
 666
 667     return $res;
 668 }
 669
 670 sub lock_config_mode {
 671     my ($vmid, $timeout, $shared, $code, @param) = @_;
 672
 673     my $filename = config_file_lock($vmid);
 674
 675     mkdir $lockdir if !-d $lockdir;
 676
 677     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
 678
 679     die $@ if $@;
 680
 681     return $res;
 682 }
 683
 684 sub lock_config {
 685     my ($vmid, $code, @param) = @_;
 686
 687     return lock_config_full($vmid, 10, $code, @param);
 688 }
 689
 690 sub option_exists {
 691     my ($name) = @_;
 692
 693     return defined($confdesc->{$name});
 694 }
 695
 696 # add JSON properties for create and set function
 697 sub json_config_properties {
 698     my $prop = shift;
 699
 700     foreach my $opt (keys %$confdesc) {
 701         next if $opt eq 'parent' || $opt eq 'snaptime';
 702         next if $prop->{$opt};
 703         $prop->{$opt} = $confdesc->{$opt};
 704     }
 705
 706     return $prop;
 707 }
 708
 709 # container status helpers
 710
 711 sub list_active_containers {
 712
 713     my $filename = "/proc/net/unix";
 714
 715     # similar test is used by lcxcontainers.c: list_active_containers
 716     my $res = {};
 717
 718     my $fh = IO::File->new ($filename, "r");
 719     return $res if !$fh;
 720
 721     while (defined(my $line = <$fh>)) {
 722         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 723             my $path = $1;
 724             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 725                 $res->{$1} = 1;
 726             }
 727         }
 728     }
 729
 730     close($fh);
 731
 732     return $res;
 733 }
 734
 735 # warning: this is slow
 736 sub check_running {
 737     my ($vmid) = @_;
 738
 739     my $active_hash = list_active_containers();
 740
 741     return 1 if defined($active_hash->{$vmid});
 742
 743     return undef;
 744 }
 745
 746 sub get_container_disk_usage {
 747     my ($vmid, $pid) = @_;
 748
 749     return PVE::Tools::df("/proc/$pid/root/", 1);
 750 }
 751
 752 my $last_proc_vmid_stat;
 753
 754 my $parse_cpuacct_stat = sub {
 755     my ($vmid) = @_;
 756
 757     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 758
 759     my $stat = {};
 760
 761     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 762
 763         $stat->{utime} = $1;
 764         $stat->{stime} = $2;
 765
 766     }
 767
 768     return $stat;
 769 };
 770
 771 sub vmstatus {
 772     my ($opt_vmid) = @_;
 773
 774     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 775
 776     my $active_hash = list_active_containers();
 777
 778     my $cpucount = $cpuinfo->{cpus} || 1;
 779
 780     my $cdtime = gettimeofday;
 781
 782     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 783
 784     foreach my $vmid (keys %$list) {
 785         my $d = $list->{$vmid};
 786
 787         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 788         warn $@ if $@; # ignore errors (consider them stopped)
 789
 790         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 791
 792         my $cfspath = cfs_config_path($vmid);
 793         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 794
 795         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 796         $d->{name} =~ s/[\s]//g;
 797
 798         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 799
 800         if ($d->{pid}) {
 801             my $res = get_container_disk_usage($vmid, $d->{pid});
 802             $d->{disk} = $res->{used};
 803             $d->{maxdisk} = $res->{total};
 804         } else {
 805             $d->{disk} = 0;
 806             # use 4GB by default ??
 807             if (my $rootfs = $conf->{rootfs}) {
 808                 my $rootinfo = parse_ct_rootfs($rootfs);
 809                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 810             } else {
 811                 $d->{maxdisk} = 4*1024*1024*1024;
 812             }
 813         }
 814
 815         $d->{mem} = 0;
 816         $d->{swap} = 0;
 817         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 818         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 819
 820         $d->{uptime} = 0;
 821         $d->{cpu} = 0;
 822
 823         $d->{netout} = 0;
 824         $d->{netin} = 0;
 825
 826         $d->{diskread} = 0;
 827         $d->{diskwrite} = 0;
 828
 829         $d->{template} = is_template($conf);
 830     }
 831
 832     foreach my $vmid (keys %$list) {
 833         my $d = $list->{$vmid};
 834         my $pid = $d->{pid};
 835
 836         next if !$pid; # skip stopped CTs
 837
 838         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 839         $d->{uptime} = time - $ctime; # the method lxcfs uses
 840
 841         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 842         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 843
 844         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 845         my @bytes = split(/\n/, $blkio_bytes);
 846         foreach my $byte (@bytes) {
 847             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 848                 $d->{diskread} = $2 if $key eq 'Read';
 849                 $d->{diskwrite} = $2 if $key eq 'Write';
 850             }
 851         }
 852
 853         my $pstat = &$parse_cpuacct_stat($vmid);
 854
 855         my $used = $pstat->{utime} + $pstat->{stime};
 856
 857         my $old = $last_proc_vmid_stat->{$vmid};
 858         if (!$old) {
 859             $last_proc_vmid_stat->{$vmid} = {
 860                 time => $cdtime,
 861                 used => $used,
 862                 cpu => 0,
 863             };
 864             next;
 865         }
 866
 867         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 868
 869         if ($dtime > 1000) {
 870             my $dutime = $used -  $old->{used};
 871
 872             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 873             $last_proc_vmid_stat->{$vmid} = {
 874                 time => $cdtime,
 875                 used => $used,
 876                 cpu => $d->{cpu},
 877             };
 878         } else {
 879             $d->{cpu} = $old->{cpu};
 880         }
 881     }
 882
 883     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 884
 885     foreach my $dev (keys %$netdev) {
 886         next if $dev !~ m/^veth([1-9]\d*)i/;
 887         my $vmid = $1;
 888         my $d = $list->{$vmid};
 889
 890         next if !$d;
 891
 892         $d->{netout} += $netdev->{$dev}->{receive};
 893         $d->{netin} += $netdev->{$dev}->{transmit};
 894
 895     }
 896
 897     return $list;
 898 }
 899
 900 sub classify_mountpoint {
 901     my ($vol) = @_;
 902     if ($vol =~ m!^/!) {
 903         return 'device' if $vol =~ m!^/dev/!;
 904         return 'bind';
 905     }
 906     return 'volume';
 907 }
 908
 909 my $parse_ct_mountpoint_full = sub {
 910     my ($desc, $data, $noerr) = @_;
 911
 912     $data //= '';
 913
 914     my $res;
 915     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 916     if ($@) {
 917         return undef if $noerr;
 918         die $@;
 919     }
 920
 921     if (defined(my $size = $res->{size})) {
 922         $size = PVE::JSONSchema::parse_size($size);
 923         if (!defined($size)) {
 924             return undef if $noerr;
 925             die "invalid size: $size\n";
 926         }
 927         $res->{size} = $size;
 928     }
 929
 930     $res->{type} = classify_mountpoint($res->{volume});
 931
 932     return $res;
 933 };
 934
 935 sub parse_ct_rootfs {
 936     my ($data, $noerr) = @_;
 937
 938     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 939
 940     $res->{mp} = '/' if defined($res);
 941
 942     return $res;
 943 }
 944
 945 sub parse_ct_mountpoint {
 946     my ($data, $noerr) = @_;
 947
 948     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 949 }
 950
 951 sub print_ct_mountpoint {
 952     my ($info, $nomp) = @_;
 953     my $skip = [ 'type' ];
 954     push @$skip, 'mp' if $nomp;
 955     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 956 }
 957
 958 sub print_lxc_network {
 959     my $net = shift;
 960     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 961 }
 962
 963 sub parse_lxc_network {
 964     my ($data) = @_;
 965
 966     my $res = {};
 967
 968     return $res if !$data;
 969
 970     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 971
 972     $res->{type} = 'veth';
 973     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 974
 975     return $res;
 976 }
 977
 978 sub read_cgroup_value {
 979     my ($group, $vmid, $name, $full) = @_;
 980
 981     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 982
 983     return PVE::Tools::file_get_contents($path) if $full;
 984
 985     return PVE::Tools::file_read_firstline($path);
 986 }
 987
 988 sub write_cgroup_value {
 989    my ($group, $vmid, $name, $value) = @_;
 990
 991    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 992    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 993
 994 }
 995
 996 sub find_lxc_console_pids {
 997
 998     my $res = {};
 999
1000     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
1001         my ($pid) = @_;
1002
1003         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
1004         return if !$cmdline;
1005
1006         my @args = split(/\0/, $cmdline);
1007
1008         # search for lxc-console -n <vmid>
1009         return if scalar(@args) != 3;
1010         return if $args[1] ne '-n';
1011         return if $args[2] !~ m/^\d+$/;
1012         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1013
1014         my $vmid = $args[2];
1015
1016         push @{$res->{$vmid}}, $pid;
1017     });
1018
1019     return $res;
1020 }
1021
1022 sub find_lxc_pid {
1023     my ($vmid) = @_;
1024
1025     my $pid = undef;
1026     my $parser = sub {
1027         my $line = shift;
1028         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1029     };
1030     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1031
1032     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1033
1034     return $pid;
1035 }
1036
1037 # Note: we cannot use Net:IP, because that only allows strict
1038 # CIDR networks
1039 sub parse_ipv4_cidr {
1040     my ($cidr, $noerr) = @_;
1041
1042     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1043         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1044     }
1045
1046     return undef if $noerr;
1047
1048     die "unable to parse ipv4 address/mask\n";
1049 }
1050
1051 sub check_lock {
1052     my ($conf) = @_;
1053
1054     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1055 }
1056
1057 sub check_protection {
1058     my ($vm_conf, $err_msg) = @_;
1059
1060     if ($vm_conf->{protection}) {
1061         die "$err_msg - protection mode enabled\n";
1062     }
1063 }
1064
1065 sub update_lxc_config {
1066     my ($storage_cfg, $vmid, $conf) = @_;
1067
1068     my $dir = "/var/lib/lxc/$vmid";
1069
1070     if ($conf->{template}) {
1071
1072         unlink "$dir/config";
1073
1074         return;
1075     }
1076
1077     my $raw = '';
1078
1079     die "missing 'arch' - internal error" if !$conf->{arch};
1080     $raw .= "lxc.arch = $conf->{arch}\n";
1081
1082     my $unprivileged = $conf->{unprivileged};
1083     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1084
1085     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1086     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux | alpine | unmanaged)$/x) {
1087         my $inc ="/usr/share/lxc/config/$ostype.common.conf";
1088         $inc ="/usr/share/lxc/config/common.conf" if !-f $inc;
1089         $raw .= "lxc.include = $inc\n";
1090         if ($unprivileged || $custom_idmap) {
1091             $inc = "/usr/share/lxc/config/$ostype.userns.conf";
1092             $inc = "/usr/share/lxc/config/userns.conf" if !-f $inc;
1093             $raw .= "lxc.include = $inc\n"
1094         }
1095     } else {
1096         die "implement me (ostype $ostype)";
1097     }
1098
1099     # WARNING: DO NOT REMOVE this without making sure that loop device nodes
1100     # cannot be exposed to the container with r/w access (cgroup perms).
1101     # When this is enabled mounts will still remain in the monitor's namespace
1102     # after the container unmounted them and thus will not detach from their
1103     # files while the container is running!
1104     $raw .= "lxc.monitor.unshare = 1\n";
1105
1106     # Should we read them from /etc/subuid?
1107     if ($unprivileged && !$custom_idmap) {
1108         $raw .= "lxc.id_map = u 0 100000 65536\n";
1109         $raw .= "lxc.id_map = g 0 100000 65536\n";
1110     }
1111
1112     if (!has_dev_console($conf)) {
1113         $raw .= "lxc.console = none\n";
1114         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1115     }
1116
1117     my $ttycount = get_tty_count($conf);
1118     $raw .= "lxc.tty = $ttycount\n";
1119
1120     # some init scripts expect a linux terminal (turnkey).
1121     $raw .= "lxc.environment = TERM=linux\n";
1122
1123     my $utsname = $conf->{hostname} || "CT$vmid";
1124     $raw .= "lxc.utsname = $utsname\n";
1125
1126     my $memory = $conf->{memory} || 512;
1127     my $swap = $conf->{swap} // 0;
1128
1129     my $lxcmem = int($memory*1024*1024);
1130     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1131
1132     my $lxcswap = int(($memory + $swap)*1024*1024);
1133     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1134
1135     if (my $cpulimit = $conf->{cpulimit}) {
1136         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1137         my $value = int(100000*$cpulimit);
1138         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1139     }
1140
1141     my $shares = $conf->{cpuunits} || 1024;
1142     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1143
1144     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1145
1146     $raw .= "lxc.rootfs = $dir/rootfs\n";
1147
1148     my $netcount = 0;
1149     foreach my $k (keys %$conf) {
1150         next if $k !~ m/^net(\d+)$/;
1151         my $ind = $1;
1152         my $d = parse_lxc_network($conf->{$k});
1153         $netcount++;
1154         $raw .= "lxc.network.type = veth\n";
1155         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1156         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1157         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1158         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1159     }
1160
1161     if (my $lxcconf = $conf->{lxc}) {
1162         foreach my $entry (@$lxcconf) {
1163             my ($k, $v) = @$entry;
1164             $netcount++ if $k eq 'lxc.network.type';
1165             $raw .= "$k = $v\n";
1166         }
1167     }
1168
1169     $raw .= "lxc.network.type = empty\n" if !$netcount;
1170
1171     File::Path::mkpath("$dir/rootfs");
1172
1173     PVE::Tools::file_set_contents("$dir/config", $raw);
1174 }
1175
1176 # verify and cleanup nameserver list (replace \0 with ' ')
1177 sub verify_nameserver_list {
1178     my ($nameserver_list) = @_;
1179
1180     my @list = ();
1181     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1182         PVE::JSONSchema::pve_verify_ip($server);
1183         push @list, $server;
1184     }
1185
1186     return join(' ', @list);
1187 }
1188
1189 sub verify_searchdomain_list {
1190     my ($searchdomain_list) = @_;
1191
1192     my @list = ();
1193     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1194         # todo: should we add checks for valid dns domains?
1195         push @list, $server;
1196     }
1197
1198     return join(' ', @list);
1199 }
1200
1201 sub is_volume_in_use {
1202     my ($config, $volid, $include_snapshots) = @_;
1203     my $used = 0;
1204
1205     foreach_mountpoint($config, sub {
1206         my ($ms, $mountpoint) = @_;
1207         return if $used;
1208         if ($mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid) {
1209             $used = 1;
1210         }
1211     });
1212
1213     my $snapshots = $config->{snapshots};
1214     if ($include_snapshots && $snapshots) {
1215         foreach my $snap (keys %$snapshots) {
1216             $used ||= is_volume_in_use($snapshots->{$snap}, $volid);
1217         }
1218     }
1219
1220     return $used;
1221 }
1222
1223 sub add_unused_volume {
1224     my ($config, $volid) = @_;
1225
1226     my $key;
1227     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1228         my $test = "unused$ind";
1229         if (my $vid = $config->{$test}) {
1230             return if $vid eq $volid; # do not add duplicates
1231         } else {
1232             $key = $test;
1233         }
1234     }
1235
1236     die "Too many unused volumes - please delete them first.\n" if !$key;
1237
1238     $config->{$key} = $volid;
1239
1240     return $key;
1241 }
1242
1243 sub update_pct_config {
1244     my ($vmid, $conf, $running, $param, $delete) = @_;
1245
1246     my @nohotplug;
1247
1248     my $new_disks = 0;
1249     my @deleted_volumes;
1250
1251     my $rootdir;
1252     if ($running) {
1253         my $pid = find_lxc_pid($vmid);
1254         $rootdir = "/proc/$pid/root";
1255     }
1256
1257     my $hotplug_error = sub {
1258         if ($running) {
1259             push @nohotplug, @_;
1260             return 1;
1261         } else {
1262             return 0;
1263         }
1264     };
1265
1266     if (defined($delete)) {
1267         foreach my $opt (@$delete) {
1268             if (!exists($conf->{$opt})) {
1269                 warn "no such option: $opt\n";
1270                 next;
1271             }
1272
1273             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1274                 die "unable to delete required option '$opt'\n";
1275             } elsif ($opt eq 'swap') {
1276                 delete $conf->{$opt};
1277                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1278             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1279                 delete $conf->{$opt};
1280             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1281                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1282                 next if $hotplug_error->($opt);
1283                 delete $conf->{$opt};
1284             } elsif ($opt =~ m/^net(\d)$/) {
1285                 delete $conf->{$opt};
1286                 next if !$running;
1287                 my $netid = $1;
1288                 PVE::Network::veth_delete("veth${vmid}i$netid");
1289             } elsif ($opt eq 'protection') {
1290                 delete $conf->{$opt};
1291             } elsif ($opt =~ m/^unused(\d+)$/) {
1292                 next if $hotplug_error->($opt);
1293                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1294                 push @deleted_volumes, $conf->{$opt};
1295                 delete $conf->{$opt};
1296             } elsif ($opt =~ m/^mp(\d+)$/) {
1297                 next if $hotplug_error->($opt);
1298                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1299                 my $mp = parse_ct_mountpoint($conf->{$opt});
1300                 delete $conf->{$opt};
1301                 if ($mp->{type} eq 'volume' && !is_volume_in_use($conf, $mp->{volume})) {
1302                     add_unused_volume($conf, $mp->{volume});
1303                 }
1304             } elsif ($opt eq 'unprivileged') {
1305                 die "unable to delete read-only option: '$opt'\n";
1306             } else {
1307                 die "implement me (delete: $opt)"
1308             }
1309             write_config($vmid, $conf) if $running;
1310         }
1311     }
1312
1313     # There's no separate swap size to configure, there's memory and "total"
1314     # memory (iow. memory+swap). This means we have to change them together.
1315     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1316     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1317     if (defined($wanted_memory) || defined($wanted_swap)) {
1318
1319         my $old_memory = ($conf->{memory} || 512);
1320         my $old_swap = ($conf->{swap} || 0);
1321
1322         $wanted_memory //= $old_memory;
1323         $wanted_swap //= $old_swap;
1324
1325         my $total = $wanted_memory + $wanted_swap;
1326         if ($running) {
1327             my $old_total = $old_memory + $old_swap;
1328             if ($total > $old_total) {
1329                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1330                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1331             } else {
1332                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1333                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1334             }
1335         }
1336         $conf->{memory} = $wanted_memory;
1337         $conf->{swap} = $wanted_swap;
1338
1339         write_config($vmid, $conf) if $running;
1340     }
1341
1342     my $used_volids = {};
1343
1344     foreach my $opt (keys %$param) {
1345         my $value = $param->{$opt};
1346         if ($opt eq 'hostname') {
1347             $conf->{$opt} = $value;
1348         } elsif ($opt eq 'onboot') {
1349             $conf->{$opt} = $value ? 1 : 0;
1350         } elsif ($opt eq 'startup') {
1351             $conf->{$opt} = $value;
1352         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1353             next if $hotplug_error->($opt);
1354             $conf->{$opt} = $value;
1355         } elsif ($opt eq 'nameserver') {
1356             next if $hotplug_error->($opt);
1357             my $list = verify_nameserver_list($value);
1358             $conf->{$opt} = $list;
1359         } elsif ($opt eq 'searchdomain') {
1360             next if $hotplug_error->($opt);
1361             my $list = verify_searchdomain_list($value);
1362             $conf->{$opt} = $list;
1363         } elsif ($opt eq 'cpulimit') {
1364             next if $hotplug_error->($opt); # FIXME: hotplug
1365             $conf->{$opt} = $value;
1366         } elsif ($opt eq 'cpuunits') {
1367             $conf->{$opt} = $value;
1368             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1369         } elsif ($opt eq 'description') {
1370             $conf->{$opt} = PVE::Tools::encode_text($value);
1371         } elsif ($opt =~ m/^net(\d+)$/) {
1372             my $netid = $1;
1373             my $net = parse_lxc_network($value);
1374             if (!$running) {
1375                 $conf->{$opt} = print_lxc_network($net);
1376             } else {
1377                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1378             }
1379         } elsif ($opt eq 'protection') {
1380             $conf->{$opt} = $value ? 1 : 0;
1381         } elsif ($opt =~ m/^mp(\d+)$/) {
1382             next if $hotplug_error->($opt);
1383             check_protection($conf, "can't update CT $vmid drive '$opt'");
1384             my $old = $conf->{$opt};
1385             $conf->{$opt} = $value;
1386             if (defined($old)) {
1387                 my $mp = parse_ct_mountpoint($old);
1388                 if ($mp->{type} eq 'volume' && !is_volume_in_use($conf, $mp->{volume})) {
1389                     add_unused_volume($conf, $mp->{volume});
1390                 }
1391             }
1392             $new_disks = 1;
1393             my $mp = parse_ct_mountpoint($value);
1394             $used_volids->{$mp->{volume}} = 1;
1395         } elsif ($opt eq 'rootfs') {
1396             next if $hotplug_error->($opt);
1397             check_protection($conf, "can't update CT $vmid drive '$opt'");
1398             my $old = $conf->{$opt};
1399             $conf->{$opt} = $value;
1400             if (defined($old)) {
1401                 my $mp = parse_ct_rootfs($old);
1402                 if ($mp->{type} eq 'volume' && !is_volume_in_use($conf, $mp->{volume})) {
1403                     add_unused_volume($conf, $mp->{volume});
1404                 }
1405             }
1406             my $mp = parse_ct_rootfs($value);
1407             $used_volids->{$mp->{volume}} = 1;
1408         } elsif ($opt eq 'unprivileged') {
1409             die "unable to modify read-only option: '$opt'\n";
1410         } elsif ($opt eq 'ostype') {
1411             next if $hotplug_error->($opt);
1412             $conf->{$opt} = $value;
1413         } else {
1414             die "implement me: $opt";
1415         }
1416         write_config($vmid, $conf) if $running;
1417     }
1418
1419     # Cleanup config:
1420
1421     # Remove unused disks after re-adding
1422     foreach my $key (keys %$conf) {
1423         next if $key !~ /^unused\d+/;
1424         my $volid = $conf->{$key};
1425         if ($used_volids->{$volid}) {
1426             delete $conf->{$key};
1427         }
1428     }
1429
1430     # Apply deletions and creations of new volumes
1431     if (@deleted_volumes) {
1432         my $storage_cfg = PVE::Storage::config();
1433         foreach my $volume (@deleted_volumes) {
1434             next if $used_volids->{$volume}; # could have been re-added, too
1435             # also check for references in snapshots
1436             next if is_volume_in_use($conf, $volume, 1);
1437             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1438         }
1439     }
1440
1441     if ($new_disks) {
1442         my $storage_cfg = PVE::Storage::config();
1443         create_disks($storage_cfg, $vmid, $conf, $conf);
1444     }
1445
1446     # This should be the last thing we do here
1447     if ($running && scalar(@nohotplug)) {
1448         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1449     }
1450 }
1451
1452 sub has_dev_console {
1453     my ($conf) = @_;
1454
1455     return !(defined($conf->{console}) && !$conf->{console});
1456 }
1457
1458 sub get_tty_count {
1459     my ($conf) = @_;
1460
1461     return $conf->{tty} // $confdesc->{tty}->{default};
1462 }
1463
1464 sub get_cmode {
1465     my ($conf) = @_;
1466
1467     return $conf->{cmode} // $confdesc->{cmode}->{default};
1468 }
1469
1470 sub get_console_command {
1471     my ($vmid, $conf) = @_;
1472
1473     my $cmode = get_cmode($conf);
1474
1475     if ($cmode eq 'console') {
1476         return ['lxc-console', '-n',  $vmid, '-t', 0];
1477     } elsif ($cmode eq 'tty') {
1478         return ['lxc-console', '-n',  $vmid];
1479     } elsif ($cmode eq 'shell') {
1480         return ['lxc-attach', '--clear-env', '-n', $vmid];
1481     } else {
1482         die "internal error";
1483     }
1484 }
1485
1486 sub get_primary_ips {
1487     my ($conf) = @_;
1488
1489     # return data from net0
1490
1491     return undef if !defined($conf->{net0});
1492     my $net = parse_lxc_network($conf->{net0});
1493
1494     my $ipv4 = $net->{ip};
1495     if ($ipv4) {
1496         if ($ipv4 =~ /^(dhcp|manual)$/) {
1497             $ipv4 = undef
1498         } else {
1499             $ipv4 =~ s!/\d+$!!;
1500         }
1501     }
1502     my $ipv6 = $net->{ip6};
1503     if ($ipv6) {
1504         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1505             $ipv6 = undef;
1506         } else {
1507             $ipv6 =~ s!/\d+$!!;
1508         }
1509     }
1510
1511     return ($ipv4, $ipv6);
1512 }
1513
1514 sub delete_mountpoint_volume {
1515     my ($storage_cfg, $vmid, $volume) = @_;
1516
1517     return if classify_mountpoint($volume) ne 'volume';
1518
1519     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1520     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1521 }
1522
1523 sub destroy_lxc_container {
1524     my ($storage_cfg, $vmid, $conf) = @_;
1525
1526     foreach_mountpoint($conf, sub {
1527         my ($ms, $mountpoint) = @_;
1528         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1529     });
1530
1531     rmdir "/var/lib/lxc/$vmid/rootfs";
1532     unlink "/var/lib/lxc/$vmid/config";
1533     rmdir "/var/lib/lxc/$vmid";
1534     destroy_config($vmid);
1535
1536     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1537     #PVE::Tools::run_command($cmd);
1538 }
1539
1540 sub vm_stop_cleanup {
1541     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1542
1543     eval {
1544         if (!$keepActive) {
1545
1546             my $vollist = get_vm_volumes($conf);
1547             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1548         }
1549     };
1550     warn $@ if $@; # avoid errors - just warn
1551 }
1552
1553 my $safe_num_ne = sub {
1554     my ($a, $b) = @_;
1555
1556     return 0 if !defined($a) && !defined($b);
1557     return 1 if !defined($a);
1558     return 1 if !defined($b);
1559
1560     return $a != $b;
1561 };
1562
1563 my $safe_string_ne = sub {
1564     my ($a, $b) = @_;
1565
1566     return 0 if !defined($a) && !defined($b);
1567     return 1 if !defined($a);
1568     return 1 if !defined($b);
1569
1570     return $a ne $b;
1571 };
1572
1573 sub update_net {
1574     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1575
1576     if ($newnet->{type} ne 'veth') {
1577         # for when there are physical interfaces
1578         die "cannot update interface of type $newnet->{type}";
1579     }
1580
1581     my $veth = "veth${vmid}i${netid}";
1582     my $eth = $newnet->{name};
1583
1584     if (my $oldnetcfg = $conf->{$opt}) {
1585         my $oldnet = parse_lxc_network($oldnetcfg);
1586
1587         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1588             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1589
1590             PVE::Network::veth_delete($veth);
1591             delete $conf->{$opt};
1592             write_config($vmid, $conf);
1593
1594             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1595
1596         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1597                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1598                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1599
1600                 if ($oldnet->{bridge}) {
1601                     PVE::Network::tap_unplug($veth);
1602                     foreach (qw(bridge tag firewall)) {
1603                         delete $oldnet->{$_};
1604                     }
1605                     $conf->{$opt} = print_lxc_network($oldnet);
1606                     write_config($vmid, $conf);
1607                 }
1608
1609                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1610                 foreach (qw(bridge tag firewall)) {
1611                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1612                 }
1613                 $conf->{$opt} = print_lxc_network($oldnet);
1614                 write_config($vmid, $conf);
1615         }
1616     } else {
1617         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1618     }
1619
1620     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1621 }
1622
1623 sub hotplug_net {
1624     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1625
1626     my $veth = "veth${vmid}i${netid}";
1627     my $vethpeer = $veth . "p";
1628     my $eth = $newnet->{name};
1629
1630     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1631     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1632
1633     # attach peer in container
1634     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1635     PVE::Tools::run_command($cmd);
1636
1637     # link up peer in container
1638     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1639     PVE::Tools::run_command($cmd);
1640
1641     my $done = { type => 'veth' };
1642     foreach (qw(bridge tag firewall hwaddr name)) {
1643         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1644     }
1645     $conf->{$opt} = print_lxc_network($done);
1646
1647     write_config($vmid, $conf);
1648 }
1649
1650 sub update_ipconfig {
1651     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1652
1653     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1654
1655     my $optdata = parse_lxc_network($conf->{$opt});
1656     my $deleted = [];
1657     my $added = [];
1658     my $nscmd = sub {
1659         my $cmdargs = shift;
1660         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1661     };
1662     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1663
1664     my $change_ip_config = sub {
1665         my ($ipversion) = @_;
1666
1667         my $family_opt = "-$ipversion";
1668         my $suffix = $ipversion == 4 ? '' : $ipversion;
1669         my $gw= "gw$suffix";
1670         my $ip= "ip$suffix";
1671
1672         my $newip = $newnet->{$ip};
1673         my $newgw = $newnet->{$gw};
1674         my $oldip = $optdata->{$ip};
1675
1676         my $change_ip = &$safe_string_ne($oldip, $newip);
1677         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1678
1679         return if !$change_ip && !$change_gw;
1680
1681         # step 1: add new IP, if this fails we cancel
1682         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1683         if ($change_ip && $is_real_ip) {
1684             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1685             if (my $err = $@) {
1686                 warn $err;
1687                 return;
1688             }
1689         }
1690
1691         # step 2: replace gateway
1692         #   If this fails we delete the added IP and cancel.
1693         #   If it succeeds we save the config and delete the old IP, ignoring
1694         #   errors. The config is then saved.
1695         # Note: 'ip route replace' can add
1696         if ($change_gw) {
1697             if ($newgw) {
1698                 eval {
1699                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1700                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1701                     }
1702                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1703                 };
1704                 if (my $err = $@) {
1705                     warn $err;
1706                     # the route was not replaced, the old IP is still available
1707                     # rollback (delete new IP) and cancel
1708                     if ($change_ip) {
1709                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1710                         warn $@ if $@; # no need to die here
1711                     }
1712                     return;
1713                 }
1714             } else {
1715                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1716                 # if the route was not deleted, the guest might have deleted it manually
1717                 # warn and continue
1718                 warn $@ if $@;
1719             }
1720         }
1721
1722         # from this point on we save the configuration
1723         # step 3: delete old IP ignoring errors
1724         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1725             # We need to enable promote_secondaries, otherwise our newly added
1726             # address will be removed along with the old one.
1727             my $promote = 0;
1728             eval {
1729                 if ($ipversion == 4) {
1730                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1731                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1732                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1733                 }
1734                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1735             };
1736             warn $@ if $@; # no need to die here
1737
1738             if ($ipversion == 4) {
1739                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1740             }
1741         }
1742
1743         foreach my $property ($ip, $gw) {
1744             if ($newnet->{$property}) {
1745                 $optdata->{$property} = $newnet->{$property};
1746             } else {
1747                 delete $optdata->{$property};
1748             }
1749         }
1750         $conf->{$opt} = print_lxc_network($optdata);
1751         write_config($vmid, $conf);
1752         $lxc_setup->setup_network($conf);
1753     };
1754
1755     &$change_ip_config(4);
1756     &$change_ip_config(6);
1757
1758 }
1759
1760 # Internal snapshots
1761
1762 # NOTE: Snapshot create/delete involves several non-atomic
1763 # actions, and can take a long time.
1764 # So we try to avoid locking the file and use the 'lock' variable
1765 # inside the config file instead.
1766
1767 my $snapshot_copy_config = sub {
1768     my ($source, $dest) = @_;
1769
1770     foreach my $k (keys %$source) {
1771         next if $k eq 'snapshots';
1772         next if $k eq 'snapstate';
1773         next if $k eq 'snaptime';
1774         next if $k eq 'vmstate';
1775         next if $k eq 'lock';
1776         next if $k eq 'digest';
1777         next if $k eq 'description';
1778         next if $k =~ m/^unused\d+$/;
1779
1780         $dest->{$k} = $source->{$k};
1781     }
1782 };
1783
1784 my $snapshot_apply_config = sub {
1785     my ($conf, $snap) = @_;
1786
1787     # copy snapshot list
1788     my $newconf = {
1789         snapshots => $conf->{snapshots},
1790     };
1791
1792     # keep description and list of unused disks
1793     foreach my $k (keys %$conf) {
1794         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
1795         $newconf->{$k} = $conf->{$k};
1796     }
1797
1798     &$snapshot_copy_config($snap, $newconf);
1799
1800     return $newconf;
1801 };
1802
1803 sub snapshot_save_vmstate {
1804     die "implement me - snapshot_save_vmstate\n";
1805 }
1806
1807 sub snapshot_prepare {
1808     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1809
1810     my $snap;
1811
1812     my $updatefn =  sub {
1813
1814         my $conf = load_config($vmid);
1815
1816         die "you can't take a snapshot if it's a template\n"
1817             if is_template($conf);
1818
1819         check_lock($conf);
1820
1821         $conf->{lock} = 'snapshot';
1822
1823         die "snapshot name '$snapname' already used\n"
1824             if defined($conf->{snapshots}->{$snapname});
1825
1826         my $storecfg = PVE::Storage::config();
1827
1828         # workaround until mp snapshots are implemented
1829         my $feature = $snapname eq 'vzdump' ? 'vzdump' : 'snapshot';
1830         die "snapshot feature is not available\n" if !has_feature($feature, $conf, $storecfg);
1831
1832         $snap = $conf->{snapshots}->{$snapname} = {};
1833
1834         if ($save_vmstate && check_running($vmid)) {
1835             snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
1836         }
1837
1838         &$snapshot_copy_config($conf, $snap);
1839
1840         $snap->{snapstate} = "prepare";
1841         $snap->{snaptime} = time();
1842         $snap->{description} = $comment if $comment;
1843
1844         write_config($vmid, $conf);
1845     };
1846
1847     lock_config($vmid, $updatefn);
1848
1849     return $snap;
1850 }
1851
1852 sub snapshot_commit {
1853     my ($vmid, $snapname) = @_;
1854
1855     my $updatefn = sub {
1856
1857         my $conf = load_config($vmid);
1858
1859         die "missing snapshot lock\n"
1860             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1861
1862         my $snap = $conf->{snapshots}->{$snapname};
1863         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1864
1865         die "wrong snapshot state\n"
1866             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
1867
1868         delete $snap->{snapstate};
1869         delete $conf->{lock};
1870
1871         my $newconf = &$snapshot_apply_config($conf, $snap);
1872
1873         $newconf->{parent} = $snapname;
1874
1875         write_config($vmid, $newconf);
1876     };
1877
1878     lock_config($vmid, $updatefn);
1879 }
1880
1881 sub has_feature {
1882     my ($feature, $conf, $storecfg, $snapname) = @_;
1883
1884     my $err;
1885     my $vzdump = $feature eq 'vzdump';
1886     $feature = 'snapshot' if $vzdump;
1887
1888     foreach_mountpoint($conf, sub {
1889         my ($ms, $mountpoint) = @_;
1890
1891         return if $err; # skip further test
1892         return if $vzdump && $ms ne 'rootfs' && !$mountpoint->{backup};
1893
1894         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1895     });
1896
1897     return $err ? 0 : 1;
1898 }
1899
1900 my $enter_namespace = sub {
1901     my ($vmid, $pid, $which, $type) = @_;
1902     sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY
1903         or die "failed to open $which namespace of container $vmid: $!\n";
1904     PVE::Tools::setns(fileno($fd), $type)
1905         or die "failed to enter $which namespace of container $vmid: $!\n";
1906     close $fd;
1907 };
1908
1909 my $do_syncfs = sub {
1910     my ($vmid, $pid, $socket) = @_;
1911
1912     &$enter_namespace($vmid, $pid, 'mnt', PVE::Tools::CLONE_NEWNS);
1913
1914     # Tell the parent process to start reading our /proc/mounts
1915     print {$socket} "go\n";
1916     $socket->flush();
1917
1918     # Receive /proc/self/mounts
1919     my $mountdata = do { local $/ = undef; <$socket> };
1920     close $socket;
1921
1922     # Now sync all mountpoints...
1923     my $mounts = PVE::ProcFSTools::parse_mounts($mountdata);
1924     foreach my $mp (@$mounts) {
1925         my ($what, $dir, $fs) = @$mp;
1926         next if $fs eq 'fuse.lxcfs';
1927         eval { PVE::Tools::sync_mountpoint($dir); };
1928         warn $@ if $@;
1929     }
1930 };
1931
1932 sub sync_container_namespace {
1933     my ($vmid) = @_;
1934     my $pid = find_lxc_pid($vmid);
1935
1936     # SOCK_DGRAM is nicer for barriers but cannot be slurped
1937     socketpair my $pfd, my $cfd, AF_UNIX, SOCK_STREAM, PF_UNSPEC
1938         or die "failed to create socketpair: $!\n";
1939
1940     my $child = fork();
1941     die "fork failed: $!\n" if !defined($child);
1942
1943     if (!$child) {
1944         eval {
1945             close $pfd;
1946             &$do_syncfs($vmid, $pid, $cfd);
1947         };
1948         if (my $err = $@) {
1949             warn $err;
1950             POSIX::_exit(1);
1951         }
1952         POSIX::_exit(0);
1953     }
1954     close $cfd;
1955     my $go = <$pfd>;
1956     die "failed to enter container namespace\n" if $go ne "go\n";
1957
1958     open my $mounts, '<', "/proc/$child/mounts"
1959         or die "failed to open container's /proc/mounts: $!\n";
1960     my $mountdata = do { local $/ = undef; <$mounts> };
1961     close $mounts;
1962     print {$pfd} $mountdata;
1963     close $pfd;
1964
1965     while (waitpid($child, 0) != $child) {}
1966     die "failed to sync container namespace\n" if $? != 0;
1967 }
1968
1969 sub check_freeze_needed {
1970     my ($vmid, $config, $save_vmstate) = @_;
1971
1972     my $ret = check_running($vmid);
1973     return ($ret, $ret);
1974 }
1975
1976 sub snapshot_create {
1977     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1978
1979     my $snap = snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
1980
1981     $save_vmstate = 0 if !$snap->{vmstate};
1982
1983     my $conf = load_config($vmid);
1984
1985     my ($running, $freezefs) = check_freeze_needed($vmid, $conf, $snap->{vmstate});
1986
1987     my $drivehash = {};
1988
1989     eval {
1990         if ($freezefs) {
1991             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1992             sync_container_namespace($vmid);
1993         }
1994
1995         my $storecfg = PVE::Storage::config();
1996         foreach_mountpoint($conf, sub {
1997             my ($ms, $mountpoint) = @_;
1998
1999             return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
2000             PVE::Storage::volume_snapshot($storecfg, $mountpoint->{volume}, $snapname);
2001             $drivehash->{$ms} = 1;
2002         });
2003     };
2004     my $err = $@;
2005
2006     if ($running) {
2007         if ($freezefs) {
2008             eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
2009             warn $@ if $@;
2010         }
2011     }
2012
2013     if ($err) {
2014         warn "snapshot create failed: starting cleanup\n";
2015         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
2016         warn "$@" if $@;
2017         die "$err\n";
2018     }
2019
2020     snapshot_commit($vmid, $snapname);
2021 }
2022
2023 # Note: $drivehash is only set when called from snapshot_create.
2024 sub snapshot_delete {
2025     my ($vmid, $snapname, $force, $drivehash) = @_;
2026
2027     my $prepare = 1;
2028
2029     my $snap;
2030     my $unused = [];
2031
2032     my $unlink_parent = sub {
2033         my ($confref, $new_parent) = @_;
2034
2035         if ($confref->{parent} && $confref->{parent} eq $snapname) {
2036             if ($new_parent) {
2037                 $confref->{parent} = $new_parent;
2038             } else {
2039                 delete $confref->{parent};
2040             }
2041         }
2042     };
2043
2044     my $updatefn =  sub {
2045         my ($remove_drive) = @_;
2046
2047         my $conf = load_config($vmid);
2048
2049         if (!$drivehash) {
2050             check_lock($conf);
2051             die "you can't delete a snapshot if vm is a template\n"
2052                 if is_template($conf);
2053         }
2054
2055         $snap = $conf->{snapshots}->{$snapname};
2056
2057         die "snapshot '$snapname' does not exist\n" if !defined($snap);
2058
2059         # remove parent refs
2060         if (!$prepare) {
2061             &$unlink_parent($conf, $snap->{parent});
2062             foreach my $sn (keys %{$conf->{snapshots}}) {
2063                 next if $sn eq $snapname;
2064                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
2065             }
2066         }
2067
2068         if ($remove_drive) {
2069             if ($remove_drive eq 'vmstate') {
2070                 die "implement me - saving vmstate\n";
2071             } else {
2072                 my $value = $snap->{$remove_drive};
2073                 my $mountpoint = $remove_drive eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2074                 delete $snap->{$remove_drive};
2075                 add_unused_volume($snap, $mountpoint->{volume})
2076                     if (!is_volume_in_use($snap, $mountpoint->{volume}));
2077             }
2078         }
2079
2080         if ($prepare) {
2081             $snap->{snapstate} = 'delete';
2082         } else {
2083             delete $conf->{snapshots}->{$snapname};
2084             delete $conf->{lock} if $drivehash;
2085             foreach my $volid (@$unused) {
2086                 add_unused_volume($conf, $volid)
2087                     if (!is_volume_in_use($conf, $volid));
2088             }
2089         }
2090
2091         write_config($vmid, $conf);
2092     };
2093
2094     lock_config($vmid, $updatefn);
2095
2096     # now remove vmstate file
2097     # never set for LXC!
2098     my $storecfg = PVE::Storage::config();
2099
2100     if ($snap->{vmstate}) {
2101         die "implement me - saving vmstate\n";
2102     };
2103
2104     # now remove all volume snapshots
2105     foreach_mountpoint($snap, sub {
2106         my ($ms, $mountpoint) = @_;
2107
2108         return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
2109         if (!$drivehash || $drivehash->{$ms}) {
2110             eval { PVE::Storage::volume_snapshot_delete($storecfg, $mountpoint->{volume}, $snapname); };
2111             if (my $err = $@) {
2112                 die $err if !$force;
2113                 warn $err;
2114             }
2115         }
2116
2117         # save changes (remove mp from snapshot)
2118         lock_config($vmid, $updatefn, $ms) if !$force;
2119         push @$unused, $mountpoint->{volume};
2120     });
2121
2122     # now cleanup config
2123     $prepare = 0;
2124     lock_config($vmid, $updatefn);
2125 }
2126
2127 sub snapshot_rollback {
2128     my ($vmid, $snapname) = @_;
2129
2130     my $prepare = 1;
2131
2132     my $storecfg = PVE::Storage::config();
2133
2134     my $conf = load_config($vmid);
2135
2136     my $get_snapshot_config = sub {
2137
2138         die "you can't rollback if vm is a template\n" if is_template($conf);
2139
2140         my $res = $conf->{snapshots}->{$snapname};
2141
2142         die "snapshot '$snapname' does not exist\n" if !defined($res);
2143
2144         return $res;
2145     };
2146
2147     my $snap = &$get_snapshot_config();
2148
2149     foreach_mountpoint($snap, sub {
2150         my ($ms, $mountpoint) = @_;
2151
2152         PVE::Storage::volume_rollback_is_possible($storecfg, $mountpoint->{volume}, $snapname);
2153     });
2154
2155     my $updatefn = sub {
2156
2157         $conf = load_config($vmid);
2158
2159         $snap = &$get_snapshot_config();
2160
2161         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
2162             if $snap->{snapstate};
2163
2164         if ($prepare) {
2165             check_lock($conf);
2166             PVE::Tools::run_command(['/usr/bin/lxc-stop', '-n', $vmid, '--kill'])
2167                 if check_running($vmid);
2168         }
2169
2170         die "unable to rollback vm $vmid: vm is running\n"
2171             if check_running($vmid);
2172
2173         if ($prepare) {
2174             $conf->{lock} = 'rollback';
2175         } else {
2176             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
2177             delete $conf->{lock};
2178         }
2179
2180         my $forcemachine;
2181
2182         if (!$prepare) {
2183             # copy snapshot config to current config
2184             $conf = &$snapshot_apply_config($conf, $snap);
2185             $conf->{parent} = $snapname;
2186         }
2187
2188         write_config($vmid, $conf);
2189
2190         if (!$prepare && $snap->{vmstate}) {
2191             die "implement me - save vmstate\n";
2192         }
2193     };
2194
2195     lock_config($vmid, $updatefn);
2196
2197     foreach_mountpoint($snap, sub {
2198         my ($ms, $mountpoint) = @_;
2199
2200         PVE::Storage::volume_snapshot_rollback($storecfg, $mountpoint->{volume}, $snapname);
2201     });
2202
2203     $prepare = 0;
2204     lock_config($vmid, $updatefn);
2205 }
2206
2207 sub template_create {
2208     my ($vmid, $conf) = @_;
2209
2210     my $storecfg = PVE::Storage::config();
2211
2212     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
2213     my $volid = $rootinfo->{volume};
2214
2215     die "Template feature is not available for '$volid'\n"
2216         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
2217
2218     PVE::Storage::activate_volumes($storecfg, [$volid]);
2219
2220     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
2221     $rootinfo->{volume} = $template_volid;
2222     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
2223
2224     write_config($vmid, $conf);
2225 }
2226
2227 sub is_template {
2228     my ($conf) = @_;
2229
2230     return 1 if defined $conf->{template} && $conf->{template} == 1;
2231 }
2232
2233 sub mountpoint_names {
2234     my ($reverse) = @_;
2235
2236     my @names = ('rootfs');
2237
2238     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
2239         push @names, "mp$i";
2240     }
2241
2242     return $reverse ? reverse @names : @names;
2243 }
2244
2245
2246 sub foreach_mountpoint_full {
2247     my ($conf, $reverse, $func) = @_;
2248
2249     foreach my $key (mountpoint_names($reverse)) {
2250         my $value = $conf->{$key};
2251         next if !defined($value);
2252         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2253         next if !defined($mountpoint);
2254
2255         &$func($key, $mountpoint);
2256     }
2257 }
2258
2259 sub foreach_mountpoint {
2260     my ($conf, $func) = @_;
2261
2262     foreach_mountpoint_full($conf, 0, $func);
2263 }
2264
2265 sub foreach_mountpoint_reverse {
2266     my ($conf, $func) = @_;
2267
2268     foreach_mountpoint_full($conf, 1, $func);
2269 }
2270
2271 sub check_ct_modify_config_perm {
2272     my ($rpcenv, $authuser, $vmid, $pool, $newconf, $delete) = @_;
2273
2274     return 1 if $authuser eq 'root@pam';
2275
2276     my $check = sub {
2277         my ($opt, $delete) = @_;
2278         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2279             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2280         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2281             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2282             return if $delete;
2283             my $data = $opt eq 'rootfs' ? parse_ct_rootfs($newconf->{$opt})
2284                                         : parse_ct_mountpoint($newconf->{$opt});
2285             raise_perm_exc("mountpoint type $data->{type}") if $data->{type} ne 'volume';
2286         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2287             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2288         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2289                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2290             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2291         } else {
2292             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2293         }
2294     };
2295
2296     foreach my $opt (keys %$newconf) {
2297         &$check($opt, 0);
2298     }
2299     foreach my $opt (@$delete) {
2300         &$check($opt, 1);
2301     }
2302
2303     return 1;
2304 }
2305
2306 sub umount_all {
2307     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2308
2309     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2310     my $volid_list = get_vm_volumes($conf);
2311
2312     foreach_mountpoint_reverse($conf, sub {
2313         my ($ms, $mountpoint) = @_;
2314
2315         my $volid = $mountpoint->{volume};
2316         my $mount = $mountpoint->{mp};
2317
2318         return if !$volid || !$mount;
2319
2320         my $mount_path = "$rootdir/$mount";
2321         $mount_path =~ s!/+!/!g;
2322
2323         return if !PVE::ProcFSTools::is_mounted($mount_path);
2324
2325         eval {
2326             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2327         };
2328         if (my $err = $@) {
2329             if ($noerr) {
2330                 warn $err;
2331             } else {
2332                 die $err;
2333             }
2334         }
2335     });
2336 }
2337
2338 sub mount_all {
2339     my ($vmid, $storage_cfg, $conf) = @_;
2340
2341     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2342     File::Path::make_path($rootdir);
2343
2344     my $volid_list = get_vm_volumes($conf);
2345     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2346
2347     eval {
2348         foreach_mountpoint($conf, sub {
2349             my ($ms, $mountpoint) = @_;
2350
2351             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2352         });
2353     };
2354     if (my $err = $@) {
2355         warn "mounting container failed\n";
2356         umount_all($vmid, $storage_cfg, $conf, 1);
2357         die $err;
2358     }
2359
2360     return $rootdir;
2361 }
2362
2363
2364 sub mountpoint_mount_path {
2365     my ($mountpoint, $storage_cfg, $snapname) = @_;
2366
2367     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2368 }
2369
2370 my $check_mount_path = sub {
2371     my ($path) = @_;
2372     $path = File::Spec->canonpath($path);
2373     my $real = Cwd::realpath($path);
2374     if ($real ne $path) {
2375         die "mount path modified by symlink: $path != $real";
2376     }
2377 };
2378
2379 sub query_loopdev {
2380     my ($path) = @_;
2381     my $found;
2382     my $parser = sub {
2383         my $line = shift;
2384         if ($line =~ m@^(/dev/loop\d+):@) {
2385             $found = $1;
2386         }
2387     };
2388     my $cmd = ['losetup', '--associated', $path];
2389     PVE::Tools::run_command($cmd, outfunc => $parser);
2390     return $found;
2391 }
2392
2393 # Run a function with a file attached to a loop device.
2394 # The loop device is always detached afterwards (or set to autoclear).
2395 # Returns the loop device.
2396 sub run_with_loopdev {
2397     my ($func, $file) = @_;
2398     my $device;
2399     my $parser = sub {
2400         my $line = shift;
2401         if ($line =~ m@^(/dev/loop\d+)$@) {
2402             $device = $1;
2403         }
2404     };
2405     PVE::Tools::run_command(['losetup', '--show', '-f', $file], outfunc => $parser);
2406     die "failed to setup loop device for $file\n" if !$device;
2407     eval { &$func($device); };
2408     my $err = $@;
2409     PVE::Tools::run_command(['losetup', '-d', $device]);
2410     die $err if $err;
2411     return $device;
2412 }
2413
2414 sub bindmount {
2415     my ($dir, $dest, $ro, @extra_opts) = @_;
2416     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $dir, $dest]);
2417     if ($ro) {
2418         eval { PVE::Tools::run_command(['mount', '-o', 'bind,remount,ro', $dest]); };
2419         if (my $err = $@) {
2420             warn "bindmount error\n";
2421             # don't leave writable bind-mounts behind...
2422             PVE::Tools::run_command(['umount', $dest]);
2423             die $err;
2424         }
2425     }
2426 }
2427
2428 # use $rootdir = undef to just return the corresponding mount path
2429 sub mountpoint_mount {
2430     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2431
2432     my $volid = $mountpoint->{volume};
2433     my $mount = $mountpoint->{mp};
2434     my $type = $mountpoint->{type};
2435     my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota};
2436     my $mounted_dev;
2437
2438     return if !$volid || !$mount;
2439
2440     my $mount_path;
2441
2442     if (defined($rootdir)) {
2443         $rootdir =~ s!/+$!!;
2444         $mount_path = "$rootdir/$mount";
2445         $mount_path =~ s!/+!/!g;
2446         &$check_mount_path($mount_path);
2447         File::Path::mkpath($mount_path);
2448     }
2449
2450     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2451
2452     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2453
2454     my $optstring = '';
2455     if (defined($mountpoint->{acl})) {
2456         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
2457     }
2458     my $readonly = $mountpoint->{ro};
2459
2460     my @extra_opts = ('-o', $optstring);
2461
2462     if ($storage) {
2463
2464         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2465         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2466
2467         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2468             PVE::Storage::parse_volname($storage_cfg, $volid);
2469
2470         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2471
2472         if ($format eq 'subvol') {
2473             if ($mount_path) {
2474                 if ($snapname) {
2475                     if ($scfg->{type} eq 'zfspool') {
2476                         my $path_arg = $path;
2477                         $path_arg =~ s!^/+!!;
2478                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
2479                     } else {
2480                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2481                     }
2482                 } else {
2483                     bindmount($path, $mount_path, $readonly, @extra_opts);
2484                     warn "cannot enable quota control for bind mounted subvolumes\n" if $quota;
2485                 }
2486             }
2487             return wantarray ? ($path, 0, $mounted_dev) : $path;
2488         } elsif ($format eq 'raw' || $format eq 'iso') {
2489             my $domount = sub {
2490                 my ($path) = @_;
2491                 if ($mount_path) {
2492                     if ($format eq 'iso') {
2493                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2494                     } elsif ($isBase || defined($snapname)) {
2495                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2496                     } else {
2497                         if ($quota) {
2498                             push @extra_opts, '-o', 'usrjquota=aquota.user,grpjquota=aquota.group,jqfmt=vfsv0';
2499                         }
2500                         push @extra_opts, '-o', 'ro' if $readonly;
2501                         PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2502                     }
2503                 }
2504             };
2505             my $use_loopdev = 0;
2506             if ($scfg->{path}) {
2507                 $mounted_dev = run_with_loopdev($domount, $path);
2508                 $use_loopdev = 1;
2509             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2510                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2511                 $mounted_dev = $path;
2512                 &$domount($path);
2513             } else {
2514                 die "unsupported storage type '$scfg->{type}'\n";
2515             }
2516             return wantarray ? ($path, $use_loopdev, $mounted_dev) : $path;
2517         } else {
2518             die "unsupported image format '$format'\n";
2519         }
2520     } elsif ($type eq 'device') {
2521                         push @extra_opts, '-o', 'ro' if $readonly;
2522         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
2523         return wantarray ? ($volid, 0, $volid) : $volid;
2524     } elsif ($type eq 'bind') {
2525         die "directory '$volid' does not exist\n" if ! -d $volid;
2526         &$check_mount_path($volid);
2527         bindmount($volid, $mount_path, $readonly, @extra_opts) if $mount_path;
2528         warn "cannot enable quota control for bind mounts\n" if $quota;
2529         return wantarray ? ($volid, 0, undef) : $volid;
2530     }
2531
2532     die "unsupported storage";
2533 }
2534
2535 sub get_vm_volumes {
2536     my ($conf, $excludes) = @_;
2537
2538     my $vollist = [];
2539
2540     foreach_mountpoint($conf, sub {
2541         my ($ms, $mountpoint) = @_;
2542
2543         return if $excludes && $ms eq $excludes;
2544
2545         my $volid = $mountpoint->{volume};
2546
2547         return if !$volid || $mountpoint->{type} ne 'volume';
2548
2549         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2550         return if !$sid;
2551
2552         push @$vollist, $volid;
2553     });
2554
2555     return $vollist;
2556 }
2557
2558 sub mkfs {
2559     my ($dev, $rootuid, $rootgid) = @_;
2560
2561     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2562                              '-E', "root_owner=$rootuid:$rootgid",
2563                              $dev]);
2564 }
2565
2566 sub format_disk {
2567     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2568
2569     if ($volid =~ m!^/dev/.+!) {
2570         mkfs($volid);
2571         return;
2572     }
2573
2574     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2575
2576     die "cannot format volume '$volid' with no storage\n" if !$storage;
2577
2578     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2579
2580     my $path = PVE::Storage::path($storage_cfg, $volid);
2581
2582     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2583         PVE::Storage::parse_volname($storage_cfg, $volid);
2584
2585     die "cannot format volume '$volid' (format == $format)\n"
2586         if $format ne 'raw';
2587
2588     mkfs($path, $rootuid, $rootgid);
2589 }
2590
2591 sub destroy_disks {
2592     my ($storecfg, $vollist) = @_;
2593
2594     foreach my $volid (@$vollist) {
2595         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2596         warn $@ if $@;
2597     }
2598 }
2599
2600 sub create_disks {
2601     my ($storecfg, $vmid, $settings, $conf) = @_;
2602
2603     my $vollist = [];
2604
2605     eval {
2606         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2607         my $chown_vollist = [];
2608
2609         foreach_mountpoint($settings, sub {
2610             my ($ms, $mountpoint) = @_;
2611
2612             my $volid = $mountpoint->{volume};
2613             my $mp = $mountpoint->{mp};
2614
2615             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2616
2617             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2618                 my ($storeid, $size_gb) = ($1, $2);
2619
2620                 my $size_kb = int(${size_gb}*1024) * 1024;
2621
2622                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2623                 # fixme: use better naming ct-$vmid-disk-X.raw?
2624
2625                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2626                     if ($size_kb > 0) {
2627                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2628                                                            undef, $size_kb);
2629                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2630                     } else {
2631                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2632                                                            undef, 0);
2633                         push @$chown_vollist, $volid;
2634                     }
2635                 } elsif ($scfg->{type} eq 'zfspool') {
2636
2637                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2638                                                        undef, $size_kb);
2639                     push @$chown_vollist, $volid;
2640                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2641
2642                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2643                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2644
2645                 } elsif ($scfg->{type} eq 'rbd') {
2646
2647                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2648                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2649                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2650                 } else {
2651                     die "unable to create containers on storage type '$scfg->{type}'\n";
2652                 }
2653                 push @$vollist, $volid;
2654                 $mountpoint->{volume} = $volid;
2655                 $mountpoint->{size} = $size_kb * 1024;
2656                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2657             } else {
2658                 # use specified/existing volid/dir/device
2659                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2660             }
2661         });
2662
2663         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2664         foreach my $volid (@$chown_vollist) {
2665             my $path = PVE::Storage::path($storecfg, $volid, undef);
2666             chown($rootuid, $rootgid, $path);
2667         }
2668         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2669     };
2670     # free allocated images on error
2671     if (my $err = $@) {
2672         destroy_disks($storecfg, $vollist);
2673         die $err;
2674     }
2675     return $vollist;
2676 }
2677
2678 # bash completion helper
2679
2680 sub complete_os_templates {
2681     my ($cmdname, $pname, $cvalue) = @_;
2682
2683     my $cfg = PVE::Storage::config();
2684
2685     my $storeid;
2686
2687     if ($cvalue =~ m/^([^:]+):/) {
2688         $storeid = $1;
2689     }
2690
2691     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2692     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2693
2694     my $res = [];
2695     foreach my $id (keys %$data) {
2696         foreach my $item (@{$data->{$id}}) {
2697             push @$res, $item->{volid} if defined($item->{volid});
2698         }
2699     }
2700
2701     return $res;
2702 }
2703
2704 my $complete_ctid_full = sub {
2705     my ($running) = @_;
2706
2707     my $idlist = vmstatus();
2708
2709     my $active_hash = list_active_containers();
2710
2711     my $res = [];
2712
2713     foreach my $id (keys %$idlist) {
2714         my $d = $idlist->{$id};
2715         if (defined($running)) {
2716             next if $d->{template};
2717             next if $running && !$active_hash->{$id};
2718             next if !$running && $active_hash->{$id};
2719         }
2720         push @$res, $id;
2721
2722     }
2723     return $res;
2724 };
2725
2726 sub complete_ctid {
2727     return &$complete_ctid_full();
2728 }
2729
2730 sub complete_ctid_stopped {
2731     return &$complete_ctid_full(0);
2732 }
2733
2734 sub complete_ctid_running {
2735     return &$complete_ctid_full(1);
2736 }
2737
2738 sub parse_id_maps {
2739     my ($conf) = @_;
2740
2741     my $id_map = [];
2742     my $rootuid = 0;
2743     my $rootgid = 0;
2744
2745     my $lxc = $conf->{lxc};
2746     foreach my $entry (@$lxc) {
2747         my ($key, $value) = @$entry;
2748         next if $key ne 'lxc.id_map';
2749         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2750             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2751             push @$id_map, [$type, $ct, $host, $length];
2752             if ($ct == 0) {
2753                 $rootuid = $host if $type eq 'u';
2754                 $rootgid = $host if $type eq 'g';
2755             }
2756         } else {
2757             die "failed to parse id_map: $value\n";
2758         }
2759     }
2760
2761     if (!@$id_map && $conf->{unprivileged}) {
2762         # Should we read them from /etc/subuid?
2763         $id_map = [ ['u', '0', '100000', '65536'],
2764                     ['g', '0', '100000', '65536'] ];
2765         $rootuid = $rootgid = 100000;
2766     }
2767
2768     return ($id_map, $rootuid, $rootgid);
2769 }
2770
2771 sub userns_command {
2772     my ($id_map) = @_;
2773     if (@$id_map) {
2774         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2775     }
2776     return [];
2777 }
2778
2779 1;