src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use Socket;
   8
   9 use File::Path;
  10 use File::Spec;
  11 use Cwd qw();
  12 use Fcntl qw(O_RDONLY);
  13
  14 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  15 use PVE::Exception qw(raise_perm_exc);
  16 use PVE::Storage;
  17 use PVE::SafeSyslog;
  18 use PVE::INotify;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full);
  21 use PVE::Network;
  22 use PVE::AccessControl;
  23 use PVE::ProcFSTools;
  24 use Time::HiRes qw (gettimeofday);
  25
  26 use Data::Dumper;
  27
  28 my $nodename = PVE::INotify::nodename();
  29
  30 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  31
  32 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  33                           '--xattrs',
  34                           '--xattrs-include=user.*',
  35                           '--xattrs-include=security.capability',
  36                           '--warning=no-xattr-write' ];
  37
  38 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  39
  40 my $rootfs_desc = {
  41     volume => {
  42         type => 'string',
  43         default_key => 1,
  44         format => 'pve-lxc-mp-string',
  45         format_description => 'volume',
  46         description => 'Volume, device or directory to mount into the container.',
  47     },
  48     backup => {
  49         type => 'boolean',
  50         format_description => '[1|0]',
  51         description => 'Whether to include the mountpoint in backups.',
  52         optional => 1,
  53     },
  54     size => {
  55         type => 'string',
  56         format => 'disk-size',
  57         format_description => 'DiskSize',
  58         description => 'Volume size (read only value).',
  59         optional => 1,
  60     },
  61     acl => {
  62         type => 'boolean',
  63         format_description => 'acl',
  64         description => 'Explicitly enable or disable ACL support.',
  65         optional => 1,
  66     },
  67     ro => {
  68         type => 'boolean',
  69         format_description => 'ro',
  70         description => 'Read-only mountpoint (not supported with bind mounts)',
  71         optional => 1,
  72     },
  73     quota => {
  74         type => 'boolean',
  75         format_description => '[0|1]',
  76         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
  77         optional => 1,
  78     },
  79 };
  80
  81 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  82     type => 'string', format => $rootfs_desc,
  83     description => "Use volume as container root.",
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  88     description => "The name of the snapshot.",
  89     type => 'string', format => 'pve-configid',
  90     maxLength => 40,
  91 });
  92
  93 my $confdesc = {
  94     lock => {
  95         optional => 1,
  96         type => 'string',
  97         description => "Lock/unlock the VM.",
  98         enum => [qw(migrate backup snapshot rollback)],
  99     },
 100     onboot => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Specifies whether a VM will be started during system bootup.",
 104         default => 0,
 105     },
 106     startup => get_standard_option('pve-startup-order'),
 107     template => {
 108         optional => 1,
 109         type => 'boolean',
 110         description => "Enable/disable Template.",
 111         default => 0,
 112     },
 113     arch => {
 114         optional => 1,
 115         type => 'string',
 116         enum => ['amd64', 'i386'],
 117         description => "OS architecture type.",
 118         default => 'amd64',
 119     },
 120     ostype => {
 121         optional => 1,
 122         type => 'string',
 123         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux', 'alpine', 'unmanaged'],
 124         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 125     },
 126     console => {
 127         optional => 1,
 128         type => 'boolean',
 129         description => "Attach a console device (/dev/console) to the container.",
 130         default => 1,
 131     },
 132     tty => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "Specify the number of tty available to the container",
 136         minimum => 0,
 137         maximum => 6,
 138         default => 2,
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1024,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     swap => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of SWAP for the VM in MB.",
 167         minimum => 0,
 168         default => 512,
 169     },
 170     hostname => {
 171         optional => 1,
 172         description => "Set a host name for the container.",
 173         type => 'string', format => 'dns-name',
 174         maxLength => 255,
 175     },
 176     description => {
 177         optional => 1,
 178         type => 'string',
 179         description => "Container description. Only used on the configuration web interface.",
 180     },
 181     searchdomain => {
 182         optional => 1,
 183         type => 'string', format => 'dns-name-list',
 184         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 185     },
 186     nameserver => {
 187         optional => 1,
 188         type => 'string', format => 'address-list',
 189         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 190     },
 191     rootfs => get_standard_option('pve-ct-rootfs'),
 192     parent => {
 193         optional => 1,
 194         type => 'string', format => 'pve-configid',
 195         maxLength => 40,
 196         description => "Parent snapshot name. This is used internally, and should not be modified.",
 197     },
 198     snaptime => {
 199         optional => 1,
 200         description => "Timestamp for snapshots.",
 201         type => 'integer',
 202         minimum => 0,
 203     },
 204     cmode => {
 205         optional => 1,
 206         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 207         type => 'string',
 208         enum => ['shell', 'console', 'tty'],
 209         default => 'tty',
 210     },
 211     protection => {
 212         optional => 1,
 213         type => 'boolean',
 214         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 215         default => 0,
 216     },
 217     unprivileged => {
 218         optional => 1,
 219         type => 'boolean',
 220         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 221         default => 0,
 222     },
 223 };
 224
 225 my $valid_lxc_conf_keys = {
 226     'lxc.include' => 1,
 227     'lxc.arch' => 1,
 228     'lxc.utsname' => 1,
 229     'lxc.haltsignal' => 1,
 230     'lxc.rebootsignal' => 1,
 231     'lxc.stopsignal' => 1,
 232     'lxc.init_cmd' => 1,
 233     'lxc.network.type' => 1,
 234     'lxc.network.flags' => 1,
 235     'lxc.network.link' => 1,
 236     'lxc.network.mtu' => 1,
 237     'lxc.network.name' => 1,
 238     'lxc.network.hwaddr' => 1,
 239     'lxc.network.ipv4' => 1,
 240     'lxc.network.ipv4.gateway' => 1,
 241     'lxc.network.ipv6' => 1,
 242     'lxc.network.ipv6.gateway' => 1,
 243     'lxc.network.script.up' => 1,
 244     'lxc.network.script.down' => 1,
 245     'lxc.pts' => 1,
 246     'lxc.console.logfile' => 1,
 247     'lxc.console' => 1,
 248     'lxc.tty' => 1,
 249     'lxc.devttydir' => 1,
 250     'lxc.hook.autodev' => 1,
 251     'lxc.autodev' => 1,
 252     'lxc.kmsg' => 1,
 253     'lxc.mount' => 1,
 254     'lxc.mount.entry' => 1,
 255     'lxc.mount.auto' => 1,
 256     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 257     'lxc.rootfs.mount' => 1,
 258     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 259                             ', please use mountpoint options in the "rootfs" key',
 260     # lxc.cgroup.*
 261     'lxc.cap.drop' => 1,
 262     'lxc.cap.keep' => 1,
 263     'lxc.aa_profile' => 1,
 264     'lxc.aa_allow_incomplete' => 1,
 265     'lxc.se_context' => 1,
 266     'lxc.seccomp' => 1,
 267     'lxc.id_map' => 1,
 268     'lxc.hook.pre-start' => 1,
 269     'lxc.hook.pre-mount' => 1,
 270     'lxc.hook.mount' => 1,
 271     'lxc.hook.start' => 1,
 272     'lxc.hook.stop' => 1,
 273     'lxc.hook.post-stop' => 1,
 274     'lxc.hook.clone' => 1,
 275     'lxc.hook.destroy' => 1,
 276     'lxc.loglevel' => 1,
 277     'lxc.logfile' => 1,
 278     'lxc.start.auto' => 1,
 279     'lxc.start.delay' => 1,
 280     'lxc.start.order' => 1,
 281     'lxc.group' => 1,
 282     'lxc.environment' => 1,
 283 };
 284
 285 my $netconf_desc = {
 286     type => {
 287         type => 'string',
 288         optional => 1,
 289         description => "Network interface type.",
 290         enum => [qw(veth)],
 291     },
 292     name => {
 293         type => 'string',
 294         format_description => 'String',
 295         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 296         pattern => '[-_.\w\d]+',
 297     },
 298     bridge => {
 299         type => 'string',
 300         format_description => 'vmbr<Number>',
 301         description => 'Bridge to attach the network device to.',
 302         pattern => '[-_.\w\d]+',
 303         optional => 1,
 304     },
 305     hwaddr => {
 306         type => 'string',
 307         format_description => 'MAC',
 308         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 309         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 310         optional => 1,
 311     },
 312     mtu => {
 313         type => 'integer',
 314         format_description => 'Number',
 315         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 316         minimum => 64, # minimum ethernet frame is 64 bytes
 317         optional => 1,
 318     },
 319     ip => {
 320         type => 'string',
 321         format => 'pve-ipv4-config',
 322         format_description => 'IPv4Format/CIDR',
 323         description => 'IPv4 address in CIDR format.',
 324         optional => 1,
 325     },
 326     gw => {
 327         type => 'string',
 328         format => 'ipv4',
 329         format_description => 'GatewayIPv4',
 330         description => 'Default gateway for IPv4 traffic.',
 331         optional => 1,
 332     },
 333     ip6 => {
 334         type => 'string',
 335         format => 'pve-ipv6-config',
 336         format_description => 'IPv6Format/CIDR',
 337         description => 'IPv6 address in CIDR format.',
 338         optional => 1,
 339     },
 340     gw6 => {
 341         type => 'string',
 342         format => 'ipv6',
 343         format_description => 'GatewayIPv6',
 344         description => 'Default gateway for IPv6 traffic.',
 345         optional => 1,
 346     },
 347     firewall => {
 348         type => 'boolean',
 349         format_description => '[1|0]',
 350         description => "Controls whether this interface's firewall rules should be used.",
 351         optional => 1,
 352     },
 353     tag => {
 354         type => 'integer',
 355         format_description => 'VlanNo',
 356         minimum => '2',
 357         maximum => '4094',
 358         description => "VLAN tag for this interface.",
 359         optional => 1,
 360     },
 361     trunks => {
 362         type => 'string',
 363         pattern => qr/\d+(?:;\d+)*/,
 364         format_description => 'vlanid[;vlanid...]',
 365         description => "VLAN ids to pass through the interface",
 366         optional => 1,
 367     },
 368 };
 369 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 370
 371 my $MAX_LXC_NETWORKS = 10;
 372 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 373     $confdesc->{"net$i"} = {
 374         optional => 1,
 375         type => 'string', format => $netconf_desc,
 376         description => "Specifies network interfaces for the container.",
 377     };
 378 }
 379
 380 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 381 sub verify_lxc_mp_string{
 382     my ($mp, $noerr) = @_;
 383
 384     # do not allow:
 385     # /./ or /../
 386     # /. or /.. at the end
 387     # ../ at the beginning
 388
 389     if($mp =~ m@/\.\.?/@ ||
 390        $mp =~ m@/\.\.?$@ ||
 391        $mp =~ m@^\.\./@){
 392         return undef if $noerr;
 393         die "$mp contains illegal character sequences\n";
 394     }
 395     return $mp;
 396 }
 397
 398 my $mp_desc = {
 399     %$rootfs_desc,
 400     mp => {
 401         type => 'string',
 402         format => 'pve-lxc-mp-string',
 403         format_description => 'Path',
 404         description => 'Path to the mountpoint as seen from inside the container.',
 405     },
 406 };
 407 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 408
 409 my $unuseddesc = {
 410     optional => 1,
 411     type => 'string', format => 'pve-volume-id',
 412     description => "Reference to unused volumes.",
 413 };
 414
 415 my $MAX_MOUNT_POINTS = 10;
 416 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 417     $confdesc->{"mp$i"} = {
 418         optional => 1,
 419         type => 'string', format => $mp_desc,
 420         description => "Use volume as container mount point (experimental feature).",
 421         optional => 1,
 422     };
 423 }
 424
 425 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 426 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 427     $confdesc->{"unused$i"} = $unuseddesc;
 428 }
 429
 430 sub write_pct_config {
 431     my ($filename, $conf) = @_;
 432
 433     delete $conf->{snapstate}; # just to be sure
 434
 435     my $generate_raw_config = sub {
 436         my ($conf) = @_;
 437
 438         my $raw = '';
 439
 440         # add description as comment to top of file
 441         my $descr = $conf->{description} || '';
 442         foreach my $cl (split(/\n/, $descr)) {
 443             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 444         }
 445
 446         foreach my $key (sort keys %$conf) {
 447             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 448                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 449             my $value = $conf->{$key};
 450             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 451             $raw .= "$key: $value\n";
 452         }
 453
 454         if (my $lxcconf = $conf->{lxc}) {
 455             foreach my $entry (@$lxcconf) {
 456                 my ($k, $v) = @$entry;
 457                 $raw .= "$k: $v\n";
 458             }
 459         }
 460
 461         return $raw;
 462     };
 463
 464     my $raw = &$generate_raw_config($conf);
 465
 466     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 467         $raw .= "\n[$snapname]\n";
 468         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 469     }
 470
 471     return $raw;
 472 }
 473
 474 sub check_type {
 475     my ($key, $value) = @_;
 476
 477     die "unknown setting '$key'\n" if !$confdesc->{$key};
 478
 479     my $type = $confdesc->{$key}->{type};
 480
 481     if (!defined($value)) {
 482         die "got undefined value\n";
 483     }
 484
 485     if ($value =~ m/[\n\r]/) {
 486         die "property contains a line feed\n";
 487     }
 488
 489     if ($type eq 'boolean') {
 490         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 491         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 492         die "type check ('boolean') failed - got '$value'\n";
 493     } elsif ($type eq 'integer') {
 494         return int($1) if $value =~ m/^(\d+)$/;
 495         die "type check ('integer') failed - got '$value'\n";
 496     } elsif ($type eq 'number') {
 497         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 498         die "type check ('number') failed - got '$value'\n";
 499     } elsif ($type eq 'string') {
 500         if (my $fmt = $confdesc->{$key}->{format}) {
 501             PVE::JSONSchema::check_format($fmt, $value);
 502             return $value;
 503         }
 504         return $value;
 505     } else {
 506         die "internal error"
 507     }
 508 }
 509
 510 sub parse_pct_config {
 511     my ($filename, $raw) = @_;
 512
 513     return undef if !defined($raw);
 514
 515     my $res = {
 516         digest => Digest::SHA::sha1_hex($raw),
 517         snapshots => {},
 518     };
 519
 520     $filename =~ m|/lxc/(\d+).conf$|
 521         || die "got strange filename '$filename'";
 522
 523     my $vmid = $1;
 524
 525     my $conf = $res;
 526     my $descr = '';
 527     my $section = '';
 528
 529     my @lines = split(/\n/, $raw);
 530     foreach my $line (@lines) {
 531         next if $line =~ m/^\s*$/;
 532
 533         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 534             $section = $1;
 535             $conf->{description} = $descr if $descr;
 536             $descr = '';
 537             $conf = $res->{snapshots}->{$section} = {};
 538             next;
 539         }
 540
 541         if ($line =~ m/^\#(.*)\s*$/) {
 542             $descr .= PVE::Tools::decode_text($1) . "\n";
 543             next;
 544         }
 545
 546         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 547             my $key = $1;
 548             my $value = $3;
 549             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 550             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 551                 push @{$conf->{lxc}}, [$key, $value];
 552             } elsif (my $errmsg = $validity) {
 553                 warn "vm $vmid - $key: $errmsg\n";
 554             } else {
 555                 warn "vm $vmid - unable to parse config: $line\n";
 556             }
 557         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 558             $descr .= PVE::Tools::decode_text($2);
 559         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 560             $conf->{snapstate} = $1;
 561         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 562             my $key = $1;
 563             my $value = $2;
 564             eval { $value = check_type($key, $value); };
 565             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 566             $conf->{$key} = $value;
 567         } else {
 568             warn "vm $vmid - unable to parse config: $line\n";
 569         }
 570     }
 571
 572     $conf->{description} = $descr if $descr;
 573
 574     delete $res->{snapstate}; # just to be sure
 575
 576     return $res;
 577 }
 578
 579 sub config_list {
 580     my $vmlist = PVE::Cluster::get_vmlist();
 581     my $res = {};
 582     return $res if !$vmlist || !$vmlist->{ids};
 583     my $ids = $vmlist->{ids};
 584
 585     foreach my $vmid (keys %$ids) {
 586         next if !$vmid; # skip CT0
 587         my $d = $ids->{$vmid};
 588         next if !$d->{node} || $d->{node} ne $nodename;
 589         next if !$d->{type} || $d->{type} ne 'lxc';
 590         $res->{$vmid}->{type} = 'lxc';
 591     }
 592     return $res;
 593 }
 594
 595 sub cfs_config_path {
 596     my ($vmid, $node) = @_;
 597
 598     $node = $nodename if !$node;
 599     return "nodes/$node/lxc/$vmid.conf";
 600 }
 601
 602 sub config_file {
 603     my ($vmid, $node) = @_;
 604
 605     my $cfspath = cfs_config_path($vmid, $node);
 606     return "/etc/pve/$cfspath";
 607 }
 608
 609 sub load_config {
 610     my ($vmid, $node) = @_;
 611
 612     $node = $nodename if !$node;
 613     my $cfspath = cfs_config_path($vmid, $node);
 614
 615     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 616     die "container $vmid does not exist\n" if !defined($conf);
 617
 618     return $conf;
 619 }
 620
 621 sub create_config {
 622     my ($vmid, $conf) = @_;
 623
 624     my $dir = "/etc/pve/nodes/$nodename/lxc";
 625     mkdir $dir;
 626
 627     write_config($vmid, $conf);
 628 }
 629
 630 sub destroy_config {
 631     my ($vmid) = @_;
 632
 633     unlink config_file($vmid, $nodename);
 634 }
 635
 636 sub write_config {
 637     my ($vmid, $conf) = @_;
 638
 639     my $cfspath = cfs_config_path($vmid);
 640
 641     PVE::Cluster::cfs_write_file($cfspath, $conf);
 642 }
 643
 644 # flock: we use one file handle per process, so lock file
 645 # can be called multiple times and will succeed for the same process.
 646
 647 my $lock_handles =  {};
 648 my $lockdir = "/run/lock/lxc";
 649
 650 sub config_file_lock {
 651     my ($vmid) = @_;
 652
 653     return "$lockdir/pve-config-${vmid}.lock";
 654 }
 655
 656 sub lock_config_full {
 657     my ($vmid, $timeout, $code, @param) = @_;
 658
 659     my $filename = config_file_lock($vmid);
 660
 661     mkdir $lockdir if !-d $lockdir;
 662
 663     my $res = lock_file($filename, $timeout, $code, @param);
 664
 665     die $@ if $@;
 666
 667     return $res;
 668 }
 669
 670 sub lock_config_mode {
 671     my ($vmid, $timeout, $shared, $code, @param) = @_;
 672
 673     my $filename = config_file_lock($vmid);
 674
 675     mkdir $lockdir if !-d $lockdir;
 676
 677     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
 678
 679     die $@ if $@;
 680
 681     return $res;
 682 }
 683
 684 sub lock_config {
 685     my ($vmid, $code, @param) = @_;
 686
 687     return lock_config_full($vmid, 10, $code, @param);
 688 }
 689
 690 sub option_exists {
 691     my ($name) = @_;
 692
 693     return defined($confdesc->{$name});
 694 }
 695
 696 # add JSON properties for create and set function
 697 sub json_config_properties {
 698     my $prop = shift;
 699
 700     foreach my $opt (keys %$confdesc) {
 701         next if $opt eq 'parent' || $opt eq 'snaptime';
 702         next if $prop->{$opt};
 703         $prop->{$opt} = $confdesc->{$opt};
 704     }
 705
 706     return $prop;
 707 }
 708
 709 # container status helpers
 710
 711 sub list_active_containers {
 712
 713     my $filename = "/proc/net/unix";
 714
 715     # similar test is used by lcxcontainers.c: list_active_containers
 716     my $res = {};
 717
 718     my $fh = IO::File->new ($filename, "r");
 719     return $res if !$fh;
 720
 721     while (defined(my $line = <$fh>)) {
 722         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 723             my $path = $1;
 724             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 725                 $res->{$1} = 1;
 726             }
 727         }
 728     }
 729
 730     close($fh);
 731
 732     return $res;
 733 }
 734
 735 # warning: this is slow
 736 sub check_running {
 737     my ($vmid) = @_;
 738
 739     my $active_hash = list_active_containers();
 740
 741     return 1 if defined($active_hash->{$vmid});
 742
 743     return undef;
 744 }
 745
 746 sub get_container_disk_usage {
 747     my ($vmid, $pid) = @_;
 748
 749     return PVE::Tools::df("/proc/$pid/root/", 1);
 750 }
 751
 752 my $last_proc_vmid_stat;
 753
 754 my $parse_cpuacct_stat = sub {
 755     my ($vmid) = @_;
 756
 757     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 758
 759     my $stat = {};
 760
 761     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 762
 763         $stat->{utime} = $1;
 764         $stat->{stime} = $2;
 765
 766     }
 767
 768     return $stat;
 769 };
 770
 771 sub vmstatus {
 772     my ($opt_vmid) = @_;
 773
 774     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 775
 776     my $active_hash = list_active_containers();
 777
 778     my $cpucount = $cpuinfo->{cpus} || 1;
 779
 780     my $cdtime = gettimeofday;
 781
 782     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 783
 784     foreach my $vmid (keys %$list) {
 785         my $d = $list->{$vmid};
 786
 787         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 788         warn $@ if $@; # ignore errors (consider them stopped)
 789
 790         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 791
 792         my $cfspath = cfs_config_path($vmid);
 793         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 794
 795         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 796         $d->{name} =~ s/[\s]//g;
 797
 798         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 799
 800         if ($d->{pid}) {
 801             my $res = get_container_disk_usage($vmid, $d->{pid});
 802             $d->{disk} = $res->{used};
 803             $d->{maxdisk} = $res->{total};
 804         } else {
 805             $d->{disk} = 0;
 806             # use 4GB by default ??
 807             if (my $rootfs = $conf->{rootfs}) {
 808                 my $rootinfo = parse_ct_rootfs($rootfs);
 809                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 810             } else {
 811                 $d->{maxdisk} = 4*1024*1024*1024;
 812             }
 813         }
 814
 815         $d->{mem} = 0;
 816         $d->{swap} = 0;
 817         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 818         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 819
 820         $d->{uptime} = 0;
 821         $d->{cpu} = 0;
 822
 823         $d->{netout} = 0;
 824         $d->{netin} = 0;
 825
 826         $d->{diskread} = 0;
 827         $d->{diskwrite} = 0;
 828
 829         $d->{template} = is_template($conf);
 830     }
 831
 832     foreach my $vmid (keys %$list) {
 833         my $d = $list->{$vmid};
 834         my $pid = $d->{pid};
 835
 836         next if !$pid; # skip stopped CTs
 837
 838         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 839         $d->{uptime} = time - $ctime; # the method lxcfs uses
 840
 841         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 842         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 843
 844         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 845         my @bytes = split(/\n/, $blkio_bytes);
 846         foreach my $byte (@bytes) {
 847             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 848                 $d->{diskread} = $2 if $key eq 'Read';
 849                 $d->{diskwrite} = $2 if $key eq 'Write';
 850             }
 851         }
 852
 853         my $pstat = &$parse_cpuacct_stat($vmid);
 854
 855         my $used = $pstat->{utime} + $pstat->{stime};
 856
 857         my $old = $last_proc_vmid_stat->{$vmid};
 858         if (!$old) {
 859             $last_proc_vmid_stat->{$vmid} = {
 860                 time => $cdtime,
 861                 used => $used,
 862                 cpu => 0,
 863             };
 864             next;
 865         }
 866
 867         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 868
 869         if ($dtime > 1000) {
 870             my $dutime = $used -  $old->{used};
 871
 872             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 873             $last_proc_vmid_stat->{$vmid} = {
 874                 time => $cdtime,
 875                 used => $used,
 876                 cpu => $d->{cpu},
 877             };
 878         } else {
 879             $d->{cpu} = $old->{cpu};
 880         }
 881     }
 882
 883     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 884
 885     foreach my $dev (keys %$netdev) {
 886         next if $dev !~ m/^veth([1-9]\d*)i/;
 887         my $vmid = $1;
 888         my $d = $list->{$vmid};
 889
 890         next if !$d;
 891
 892         $d->{netout} += $netdev->{$dev}->{receive};
 893         $d->{netin} += $netdev->{$dev}->{transmit};
 894
 895     }
 896
 897     return $list;
 898 }
 899
 900 sub classify_mountpoint {
 901     my ($vol) = @_;
 902     if ($vol =~ m!^/!) {
 903         return 'device' if $vol =~ m!^/dev/!;
 904         return 'bind';
 905     }
 906     return 'volume';
 907 }
 908
 909 my $parse_ct_mountpoint_full = sub {
 910     my ($desc, $data, $noerr) = @_;
 911
 912     $data //= '';
 913
 914     my $res;
 915     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 916     if ($@) {
 917         return undef if $noerr;
 918         die $@;
 919     }
 920
 921     if (defined(my $size = $res->{size})) {
 922         $size = PVE::JSONSchema::parse_size($size);
 923         if (!defined($size)) {
 924             return undef if $noerr;
 925             die "invalid size: $size\n";
 926         }
 927         $res->{size} = $size;
 928     }
 929
 930     $res->{type} = classify_mountpoint($res->{volume});
 931
 932     return $res;
 933 };
 934
 935 sub parse_ct_rootfs {
 936     my ($data, $noerr) = @_;
 937
 938     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 939
 940     $res->{mp} = '/' if defined($res);
 941
 942     return $res;
 943 }
 944
 945 sub parse_ct_mountpoint {
 946     my ($data, $noerr) = @_;
 947
 948     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 949 }
 950
 951 sub print_ct_mountpoint {
 952     my ($info, $nomp) = @_;
 953     my $skip = [ 'type' ];
 954     push @$skip, 'mp' if $nomp;
 955     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 956 }
 957
 958 sub print_lxc_network {
 959     my $net = shift;
 960     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 961 }
 962
 963 sub parse_lxc_network {
 964     my ($data) = @_;
 965
 966     my $res = {};
 967
 968     return $res if !$data;
 969
 970     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 971
 972     $res->{type} = 'veth';
 973     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 974
 975     return $res;
 976 }
 977
 978 sub read_cgroup_value {
 979     my ($group, $vmid, $name, $full) = @_;
 980
 981     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 982
 983     return PVE::Tools::file_get_contents($path) if $full;
 984
 985     return PVE::Tools::file_read_firstline($path);
 986 }
 987
 988 sub write_cgroup_value {
 989    my ($group, $vmid, $name, $value) = @_;
 990
 991    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 992    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 993
 994 }
 995
 996 sub find_lxc_console_pids {
 997
 998     my $res = {};
 999
1000     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
1001         my ($pid) = @_;
1002
1003         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
1004         return if !$cmdline;
1005
1006         my @args = split(/\0/, $cmdline);
1007
1008         # search for lxc-console -n <vmid>
1009         return if scalar(@args) != 3;
1010         return if $args[1] ne '-n';
1011         return if $args[2] !~ m/^\d+$/;
1012         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1013
1014         my $vmid = $args[2];
1015
1016         push @{$res->{$vmid}}, $pid;
1017     });
1018
1019     return $res;
1020 }
1021
1022 sub find_lxc_pid {
1023     my ($vmid) = @_;
1024
1025     my $pid = undef;
1026     my $parser = sub {
1027         my $line = shift;
1028         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1029     };
1030     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1031
1032     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1033
1034     return $pid;
1035 }
1036
1037 # Note: we cannot use Net:IP, because that only allows strict
1038 # CIDR networks
1039 sub parse_ipv4_cidr {
1040     my ($cidr, $noerr) = @_;
1041
1042     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1043         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1044     }
1045
1046     return undef if $noerr;
1047
1048     die "unable to parse ipv4 address/mask\n";
1049 }
1050
1051 sub check_lock {
1052     my ($conf) = @_;
1053
1054     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1055 }
1056
1057 sub check_protection {
1058     my ($vm_conf, $err_msg) = @_;
1059
1060     if ($vm_conf->{protection}) {
1061         die "$err_msg - protection mode enabled\n";
1062     }
1063 }
1064
1065 sub update_lxc_config {
1066     my ($storage_cfg, $vmid, $conf) = @_;
1067
1068     my $dir = "/var/lib/lxc/$vmid";
1069
1070     if ($conf->{template}) {
1071
1072         unlink "$dir/config";
1073
1074         return;
1075     }
1076
1077     my $raw = '';
1078
1079     die "missing 'arch' - internal error" if !$conf->{arch};
1080     $raw .= "lxc.arch = $conf->{arch}\n";
1081
1082     my $unprivileged = $conf->{unprivileged};
1083     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1084
1085     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1086     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux | alpine | unmanaged)$/x) {
1087         my $inc ="/usr/share/lxc/config/$ostype.common.conf";
1088         $inc ="/usr/share/lxc/config/common.conf" if !-f $inc;
1089         $raw .= "lxc.include = $inc\n";
1090         if ($unprivileged || $custom_idmap) {
1091             $inc = "/usr/share/lxc/config/$ostype.userns.conf";
1092             $inc = "/usr/share/lxc/config/userns.conf" if !-f $inc;
1093             $raw .= "lxc.include = $inc\n"
1094         }
1095     } else {
1096         die "implement me (ostype $ostype)";
1097     }
1098
1099     # WARNING: DO NOT REMOVE this without making sure that loop device nodes
1100     # cannot be exposed to the container with r/w access (cgroup perms).
1101     # When this is enabled mounts will still remain in the monitor's namespace
1102     # after the container unmounted them and thus will not detach from their
1103     # files while the container is running!
1104     $raw .= "lxc.monitor.unshare = 1\n";
1105
1106     # Should we read them from /etc/subuid?
1107     if ($unprivileged && !$custom_idmap) {
1108         $raw .= "lxc.id_map = u 0 100000 65536\n";
1109         $raw .= "lxc.id_map = g 0 100000 65536\n";
1110     }
1111
1112     if (!has_dev_console($conf)) {
1113         $raw .= "lxc.console = none\n";
1114         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1115     }
1116
1117     my $ttycount = get_tty_count($conf);
1118     $raw .= "lxc.tty = $ttycount\n";
1119
1120     # some init scripts expect a linux terminal (turnkey).
1121     $raw .= "lxc.environment = TERM=linux\n";
1122
1123     my $utsname = $conf->{hostname} || "CT$vmid";
1124     $raw .= "lxc.utsname = $utsname\n";
1125
1126     my $memory = $conf->{memory} || 512;
1127     my $swap = $conf->{swap} // 0;
1128
1129     my $lxcmem = int($memory*1024*1024);
1130     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1131
1132     my $lxcswap = int(($memory + $swap)*1024*1024);
1133     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1134
1135     if (my $cpulimit = $conf->{cpulimit}) {
1136         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1137         my $value = int(100000*$cpulimit);
1138         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1139     }
1140
1141     my $shares = $conf->{cpuunits} || 1024;
1142     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1143
1144     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1145
1146     $raw .= "lxc.rootfs = $dir/rootfs\n";
1147
1148     my $netcount = 0;
1149     foreach my $k (keys %$conf) {
1150         next if $k !~ m/^net(\d+)$/;
1151         my $ind = $1;
1152         my $d = parse_lxc_network($conf->{$k});
1153         $netcount++;
1154         $raw .= "lxc.network.type = veth\n";
1155         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1156         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1157         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1158         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1159     }
1160
1161     if (my $lxcconf = $conf->{lxc}) {
1162         foreach my $entry (@$lxcconf) {
1163             my ($k, $v) = @$entry;
1164             $netcount++ if $k eq 'lxc.network.type';
1165             $raw .= "$k = $v\n";
1166         }
1167     }
1168
1169     $raw .= "lxc.network.type = empty\n" if !$netcount;
1170
1171     File::Path::mkpath("$dir/rootfs");
1172
1173     PVE::Tools::file_set_contents("$dir/config", $raw);
1174 }
1175
1176 # verify and cleanup nameserver list (replace \0 with ' ')
1177 sub verify_nameserver_list {
1178     my ($nameserver_list) = @_;
1179
1180     my @list = ();
1181     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1182         PVE::JSONSchema::pve_verify_ip($server);
1183         push @list, $server;
1184     }
1185
1186     return join(' ', @list);
1187 }
1188
1189 sub verify_searchdomain_list {
1190     my ($searchdomain_list) = @_;
1191
1192     my @list = ();
1193     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1194         # todo: should we add checks for valid dns domains?
1195         push @list, $server;
1196     }
1197
1198     return join(' ', @list);
1199 }
1200
1201 sub is_volume_in_use {
1202     my ($config, $volid, $include_snapshots) = @_;
1203     my $used = 0;
1204
1205     foreach_mountpoint($config, sub {
1206         my ($ms, $mountpoint) = @_;
1207         return if $used;
1208         if ($mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid) {
1209             $used = 1;
1210         }
1211     });
1212
1213     my $snapshots = $config->{snapshots};
1214     if ($include_snapshots && $snapshots) {
1215         foreach my $snap (keys %$snapshots) {
1216             $used ||= is_volume_in_use($snapshots->{$snap}, $volid);
1217         }
1218     }
1219
1220     return $used;
1221 }
1222
1223 sub add_unused_volume {
1224     my ($config, $volid) = @_;
1225
1226     my $key;
1227     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1228         my $test = "unused$ind";
1229         if (my $vid = $config->{$test}) {
1230             return if $vid eq $volid; # do not add duplicates
1231         } else {
1232             $key = $test;
1233         }
1234     }
1235
1236     die "Too many unused volumes - please delete them first.\n" if !$key;
1237
1238     $config->{$key} = $volid;
1239
1240     return $key;
1241 }
1242
1243 sub update_pct_config {
1244     my ($vmid, $conf, $running, $param, $delete) = @_;
1245
1246     my @nohotplug;
1247
1248     my $new_disks = 0;
1249     my @deleted_volumes;
1250
1251     my $rootdir;
1252     if ($running) {
1253         my $pid = find_lxc_pid($vmid);
1254         $rootdir = "/proc/$pid/root";
1255     }
1256
1257     my $hotplug_error = sub {
1258         if ($running) {
1259             push @nohotplug, @_;
1260             return 1;
1261         } else {
1262             return 0;
1263         }
1264     };
1265
1266     if (defined($delete)) {
1267         foreach my $opt (@$delete) {
1268             if (!exists($conf->{$opt})) {
1269                 warn "no such option: $opt\n";
1270                 next;
1271             }
1272
1273             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1274                 die "unable to delete required option '$opt'\n";
1275             } elsif ($opt eq 'swap') {
1276                 delete $conf->{$opt};
1277                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1278             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1279                 delete $conf->{$opt};
1280             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1281                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1282                 next if $hotplug_error->($opt);
1283                 delete $conf->{$opt};
1284             } elsif ($opt =~ m/^net(\d)$/) {
1285                 delete $conf->{$opt};
1286                 next if !$running;
1287                 my $netid = $1;
1288                 PVE::Network::veth_delete("veth${vmid}i$netid");
1289             } elsif ($opt eq 'protection') {
1290                 delete $conf->{$opt};
1291             } elsif ($opt =~ m/^unused(\d+)$/) {
1292                 next if $hotplug_error->($opt);
1293                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1294                 push @deleted_volumes, $conf->{$opt};
1295                 delete $conf->{$opt};
1296             } elsif ($opt =~ m/^mp(\d+)$/) {
1297                 next if $hotplug_error->($opt);
1298                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1299                 my $mp = parse_ct_mountpoint($conf->{$opt});
1300                 delete $conf->{$opt};
1301                 if ($mp->{type} eq 'volume' && !is_volume_in_use($conf, $mp->{volume})) {
1302                     add_unused_volume($conf, $mp->{volume});
1303                 }
1304             } elsif ($opt eq 'unprivileged') {
1305                 die "unable to delete read-only option: '$opt'\n";
1306             } else {
1307                 die "implement me (delete: $opt)"
1308             }
1309             write_config($vmid, $conf) if $running;
1310         }
1311     }
1312
1313     # There's no separate swap size to configure, there's memory and "total"
1314     # memory (iow. memory+swap). This means we have to change them together.
1315     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1316     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1317     if (defined($wanted_memory) || defined($wanted_swap)) {
1318
1319         my $old_memory = ($conf->{memory} || 512);
1320         my $old_swap = ($conf->{swap} || 0);
1321
1322         $wanted_memory //= $old_memory;
1323         $wanted_swap //= $old_swap;
1324
1325         my $total = $wanted_memory + $wanted_swap;
1326         if ($running) {
1327             my $old_total = $old_memory + $old_swap;
1328             if ($total > $old_total) {
1329                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1330                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1331             } else {
1332                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1333                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1334             }
1335         }
1336         $conf->{memory} = $wanted_memory;
1337         $conf->{swap} = $wanted_swap;
1338
1339         write_config($vmid, $conf) if $running;
1340     }
1341
1342     my $used_volids = {};
1343
1344     foreach my $opt (keys %$param) {
1345         my $value = $param->{$opt};
1346         if ($opt eq 'hostname') {
1347             $conf->{$opt} = $value;
1348         } elsif ($opt eq 'onboot') {
1349             $conf->{$opt} = $value ? 1 : 0;
1350         } elsif ($opt eq 'startup') {
1351             $conf->{$opt} = $value;
1352         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1353             next if $hotplug_error->($opt);
1354             $conf->{$opt} = $value;
1355         } elsif ($opt eq 'nameserver') {
1356             next if $hotplug_error->($opt);
1357             my $list = verify_nameserver_list($value);
1358             $conf->{$opt} = $list;
1359         } elsif ($opt eq 'searchdomain') {
1360             next if $hotplug_error->($opt);
1361             my $list = verify_searchdomain_list($value);
1362             $conf->{$opt} = $list;
1363         } elsif ($opt eq 'cpulimit') {
1364             next if $hotplug_error->($opt); # FIXME: hotplug
1365             $conf->{$opt} = $value;
1366         } elsif ($opt eq 'cpuunits') {
1367             $conf->{$opt} = $value;
1368             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1369         } elsif ($opt eq 'description') {
1370             $conf->{$opt} = PVE::Tools::encode_text($value);
1371         } elsif ($opt =~ m/^net(\d+)$/) {
1372             my $netid = $1;
1373             my $net = parse_lxc_network($value);
1374             if (!$running) {
1375                 $conf->{$opt} = print_lxc_network($net);
1376             } else {
1377                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1378             }
1379         } elsif ($opt eq 'protection') {
1380             $conf->{$opt} = $value ? 1 : 0;
1381         } elsif ($opt =~ m/^mp(\d+)$/) {
1382             next if $hotplug_error->($opt);
1383             check_protection($conf, "can't update CT $vmid drive '$opt'");
1384             my $old = $conf->{$opt};
1385             $conf->{$opt} = $value;
1386             if (defined($old)) {
1387                 my $mp = parse_ct_mountpoint($old);
1388                 if ($mp->{type} eq 'volume' && !is_volume_in_use($conf, $mp->{volume})) {
1389                     add_unused_volume($conf, $mp->{volume});
1390                 }
1391             }
1392             $new_disks = 1;
1393             my $mp = parse_ct_mountpoint($value);
1394             $used_volids->{$mp->{volume}} = 1;
1395         } elsif ($opt eq 'rootfs') {
1396             next if $hotplug_error->($opt);
1397             check_protection($conf, "can't update CT $vmid drive '$opt'");
1398             my $old = $conf->{$opt};
1399             $conf->{$opt} = $value;
1400             if (defined($old)) {
1401                 my $mp = parse_ct_rootfs($old);
1402                 if ($mp->{type} eq 'volume' && !is_volume_in_use($conf, $mp->{volume})) {
1403                     add_unused_volume($conf, $mp->{volume});
1404                 }
1405             }
1406             my $mp = parse_ct_rootfs($value);
1407             $used_volids->{$mp->{volume}} = 1;
1408         } elsif ($opt eq 'unprivileged') {
1409             die "unable to modify read-only option: '$opt'\n";
1410         } elsif ($opt eq 'ostype') {
1411             next if $hotplug_error->($opt);
1412             $conf->{$opt} = $value;
1413         } else {
1414             die "implement me: $opt";
1415         }
1416         write_config($vmid, $conf) if $running;
1417     }
1418
1419     # Cleanup config:
1420
1421     # Remove unused disks after re-adding
1422     foreach my $key (keys %$conf) {
1423         next if $key !~ /^unused\d+/;
1424         my $volid = $conf->{$key};
1425         if ($used_volids->{$volid}) {
1426             delete $conf->{$key};
1427         }
1428     }
1429
1430     # Apply deletions and creations of new volumes
1431     if (@deleted_volumes) {
1432         my $storage_cfg = PVE::Storage::config();
1433         foreach my $volume (@deleted_volumes) {
1434             next if $used_volids->{$volume}; # could have been re-added, too
1435             # also check for references in snapshots
1436             next if is_volume_in_use($conf, $volume, 1);
1437             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1438         }
1439     }
1440
1441     if ($new_disks) {
1442         my $storage_cfg = PVE::Storage::config();
1443         create_disks($storage_cfg, $vmid, $conf, $conf);
1444     }
1445
1446     # This should be the last thing we do here
1447     if ($running && scalar(@nohotplug)) {
1448         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1449     }
1450 }
1451
1452 sub has_dev_console {
1453     my ($conf) = @_;
1454
1455     return !(defined($conf->{console}) && !$conf->{console});
1456 }
1457
1458 sub get_tty_count {
1459     my ($conf) = @_;
1460
1461     return $conf->{tty} // $confdesc->{tty}->{default};
1462 }
1463
1464 sub get_cmode {
1465     my ($conf) = @_;
1466
1467     return $conf->{cmode} // $confdesc->{cmode}->{default};
1468 }
1469
1470 sub get_console_command {
1471     my ($vmid, $conf) = @_;
1472
1473     my $cmode = get_cmode($conf);
1474
1475     if ($cmode eq 'console') {
1476         return ['lxc-console', '-n',  $vmid, '-t', 0];
1477     } elsif ($cmode eq 'tty') {
1478         return ['lxc-console', '-n',  $vmid];
1479     } elsif ($cmode eq 'shell') {
1480         return ['lxc-attach', '--clear-env', '-n', $vmid];
1481     } else {
1482         die "internal error";
1483     }
1484 }
1485
1486 sub get_primary_ips {
1487     my ($conf) = @_;
1488
1489     # return data from net0
1490
1491     return undef if !defined($conf->{net0});
1492     my $net = parse_lxc_network($conf->{net0});
1493
1494     my $ipv4 = $net->{ip};
1495     if ($ipv4) {
1496         if ($ipv4 =~ /^(dhcp|manual)$/) {
1497             $ipv4 = undef
1498         } else {
1499             $ipv4 =~ s!/\d+$!!;
1500         }
1501     }
1502     my $ipv6 = $net->{ip6};
1503     if ($ipv6) {
1504         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1505             $ipv6 = undef;
1506         } else {
1507             $ipv6 =~ s!/\d+$!!;
1508         }
1509     }
1510
1511     return ($ipv4, $ipv6);
1512 }
1513
1514 sub delete_mountpoint_volume {
1515     my ($storage_cfg, $vmid, $volume) = @_;
1516
1517     return if classify_mountpoint($volume) ne 'volume';
1518
1519     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1520     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1521 }
1522
1523 sub destroy_lxc_container {
1524     my ($storage_cfg, $vmid, $conf) = @_;
1525
1526     foreach_mountpoint($conf, sub {
1527         my ($ms, $mountpoint) = @_;
1528         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1529     });
1530
1531     rmdir "/var/lib/lxc/$vmid/rootfs";
1532     unlink "/var/lib/lxc/$vmid/config";
1533     rmdir "/var/lib/lxc/$vmid";
1534     destroy_config($vmid);
1535
1536     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1537     #PVE::Tools::run_command($cmd);
1538 }
1539
1540 sub vm_stop_cleanup {
1541     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1542
1543     eval {
1544         if (!$keepActive) {
1545
1546             my $vollist = get_vm_volumes($conf);
1547             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1548         }
1549     };
1550     warn $@ if $@; # avoid errors - just warn
1551 }
1552
1553 my $safe_num_ne = sub {
1554     my ($a, $b) = @_;
1555
1556     return 0 if !defined($a) && !defined($b);
1557     return 1 if !defined($a);
1558     return 1 if !defined($b);
1559
1560     return $a != $b;
1561 };
1562
1563 my $safe_string_ne = sub {
1564     my ($a, $b) = @_;
1565
1566     return 0 if !defined($a) && !defined($b);
1567     return 1 if !defined($a);
1568     return 1 if !defined($b);
1569
1570     return $a ne $b;
1571 };
1572
1573 sub update_net {
1574     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1575
1576     if ($newnet->{type} ne 'veth') {
1577         # for when there are physical interfaces
1578         die "cannot update interface of type $newnet->{type}";
1579     }
1580
1581     my $veth = "veth${vmid}i${netid}";
1582     my $eth = $newnet->{name};
1583
1584     if (my $oldnetcfg = $conf->{$opt}) {
1585         my $oldnet = parse_lxc_network($oldnetcfg);
1586
1587         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1588             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1589
1590             PVE::Network::veth_delete($veth);
1591             delete $conf->{$opt};
1592             write_config($vmid, $conf);
1593
1594             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1595
1596         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1597                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1598                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1599
1600                 if ($oldnet->{bridge}) {
1601                     PVE::Network::tap_unplug($veth);
1602                     foreach (qw(bridge tag firewall)) {
1603                         delete $oldnet->{$_};
1604                     }
1605                     $conf->{$opt} = print_lxc_network($oldnet);
1606                     write_config($vmid, $conf);
1607                 }
1608
1609                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1610                 foreach (qw(bridge tag firewall)) {
1611                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1612                 }
1613                 $conf->{$opt} = print_lxc_network($oldnet);
1614                 write_config($vmid, $conf);
1615         }
1616     } else {
1617         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1618     }
1619
1620     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1621 }
1622
1623 sub hotplug_net {
1624     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1625
1626     my $veth = "veth${vmid}i${netid}";
1627     my $vethpeer = $veth . "p";
1628     my $eth = $newnet->{name};
1629
1630     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1631     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1632
1633     # attach peer in container
1634     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1635     PVE::Tools::run_command($cmd);
1636
1637     # link up peer in container
1638     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1639     PVE::Tools::run_command($cmd);
1640
1641     my $done = { type => 'veth' };
1642     foreach (qw(bridge tag firewall hwaddr name)) {
1643         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1644     }
1645     $conf->{$opt} = print_lxc_network($done);
1646
1647     write_config($vmid, $conf);
1648 }
1649
1650 sub update_ipconfig {
1651     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1652
1653     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1654
1655     my $optdata = parse_lxc_network($conf->{$opt});
1656     my $deleted = [];
1657     my $added = [];
1658     my $nscmd = sub {
1659         my $cmdargs = shift;
1660         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1661     };
1662     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1663
1664     my $change_ip_config = sub {
1665         my ($ipversion) = @_;
1666
1667         my $family_opt = "-$ipversion";
1668         my $suffix = $ipversion == 4 ? '' : $ipversion;
1669         my $gw= "gw$suffix";
1670         my $ip= "ip$suffix";
1671
1672         my $newip = $newnet->{$ip};
1673         my $newgw = $newnet->{$gw};
1674         my $oldip = $optdata->{$ip};
1675
1676         my $change_ip = &$safe_string_ne($oldip, $newip);
1677         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1678
1679         return if !$change_ip && !$change_gw;
1680
1681         # step 1: add new IP, if this fails we cancel
1682         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1683         if ($change_ip && $is_real_ip) {
1684             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1685             if (my $err = $@) {
1686                 warn $err;
1687                 return;
1688             }
1689         }
1690
1691         # step 2: replace gateway
1692         #   If this fails we delete the added IP and cancel.
1693         #   If it succeeds we save the config and delete the old IP, ignoring
1694         #   errors. The config is then saved.
1695         # Note: 'ip route replace' can add
1696         if ($change_gw) {
1697             if ($newgw) {
1698                 eval {
1699                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1700                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1701                     }
1702                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1703                 };
1704                 if (my $err = $@) {
1705                     warn $err;
1706                     # the route was not replaced, the old IP is still available
1707                     # rollback (delete new IP) and cancel
1708                     if ($change_ip) {
1709                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1710                         warn $@ if $@; # no need to die here
1711                     }
1712                     return;
1713                 }
1714             } else {
1715                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1716                 # if the route was not deleted, the guest might have deleted it manually
1717                 # warn and continue
1718                 warn $@ if $@;
1719             }
1720         }
1721
1722         # from this point on we save the configuration
1723         # step 3: delete old IP ignoring errors
1724         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1725             # We need to enable promote_secondaries, otherwise our newly added
1726             # address will be removed along with the old one.
1727             my $promote = 0;
1728             eval {
1729                 if ($ipversion == 4) {
1730                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1731                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1732                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1733                 }
1734                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1735             };
1736             warn $@ if $@; # no need to die here
1737
1738             if ($ipversion == 4) {
1739                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1740             }
1741         }
1742
1743         foreach my $property ($ip, $gw) {
1744             if ($newnet->{$property}) {
1745                 $optdata->{$property} = $newnet->{$property};
1746             } else {
1747                 delete $optdata->{$property};
1748             }
1749         }
1750         $conf->{$opt} = print_lxc_network($optdata);
1751         write_config($vmid, $conf);
1752         $lxc_setup->setup_network($conf);
1753     };
1754
1755     &$change_ip_config(4);
1756     &$change_ip_config(6);
1757
1758 }
1759
1760 # Internal snapshots
1761
1762 # NOTE: Snapshot create/delete involves several non-atomic
1763 # actions, and can take a long time.
1764 # So we try to avoid locking the file and use the 'lock' variable
1765 # inside the config file instead.
1766
1767 my $snapshot_copy_config = sub {
1768     my ($source, $dest) = @_;
1769
1770     foreach my $k (keys %$source) {
1771         next if $k eq 'snapshots';
1772         next if $k eq 'snapstate';
1773         next if $k eq 'snaptime';
1774         next if $k eq 'vmstate';
1775         next if $k eq 'lock';
1776         next if $k eq 'digest';
1777         next if $k eq 'description';
1778         next if $k =~ m/^unused\d+$/;
1779
1780         $dest->{$k} = $source->{$k};
1781     }
1782 };
1783
1784 my $snapshot_apply_config = sub {
1785     my ($conf, $snap) = @_;
1786
1787     # copy snapshot list
1788     my $newconf = {
1789         snapshots => $conf->{snapshots},
1790     };
1791
1792     # keep description and list of unused disks
1793     foreach my $k (keys %$conf) {
1794         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
1795         $newconf->{$k} = $conf->{$k};
1796     }
1797
1798     &$snapshot_copy_config($snap, $newconf);
1799
1800     return $newconf;
1801 };
1802
1803 sub snapshot_save_vmstate {
1804     die "implement me - snapshot_save_vmstate\n";
1805 }
1806
1807 sub snapshot_prepare {
1808     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1809
1810     my $snap;
1811
1812     my $updatefn =  sub {
1813
1814         my $conf = load_config($vmid);
1815
1816         die "you can't take a snapshot if it's a template\n"
1817             if is_template($conf);
1818
1819         check_lock($conf);
1820
1821         $conf->{lock} = 'snapshot';
1822
1823         die "snapshot name '$snapname' already used\n"
1824             if defined($conf->{snapshots}->{$snapname});
1825
1826         my $storecfg = PVE::Storage::config();
1827         die "snapshot feature is not available\n"
1828             if !has_feature('snapshot', $conf, $storecfg, undef, undef, $snapname eq 'vzdump');
1829
1830         $snap = $conf->{snapshots}->{$snapname} = {};
1831
1832         if ($save_vmstate && check_running($vmid)) {
1833             snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
1834         }
1835
1836         &$snapshot_copy_config($conf, $snap);
1837
1838         $snap->{snapstate} = "prepare";
1839         $snap->{snaptime} = time();
1840         $snap->{description} = $comment if $comment;
1841
1842         write_config($vmid, $conf);
1843     };
1844
1845     lock_config($vmid, $updatefn);
1846
1847     return $snap;
1848 }
1849
1850 sub snapshot_commit {
1851     my ($vmid, $snapname) = @_;
1852
1853     my $updatefn = sub {
1854
1855         my $conf = load_config($vmid);
1856
1857         die "missing snapshot lock\n"
1858             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1859
1860         my $snap = $conf->{snapshots}->{$snapname};
1861         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1862
1863         die "wrong snapshot state\n"
1864             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
1865
1866         delete $snap->{snapstate};
1867         delete $conf->{lock};
1868
1869         my $newconf = &$snapshot_apply_config($conf, $snap);
1870
1871         $newconf->{parent} = $snapname;
1872
1873         write_config($vmid, $newconf);
1874     };
1875
1876     lock_config($vmid, $updatefn);
1877 }
1878
1879 sub has_feature {
1880     my ($feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
1881
1882     my $err;
1883
1884     foreach_mountpoint($conf, sub {
1885         my ($ms, $mountpoint) = @_;
1886
1887         return if $err; # skip further test
1888         return if $backup_only && $ms ne 'rootfs' && !$mountpoint->{backup};
1889
1890         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname, $running);
1891     });
1892
1893     return $err ? 0 : 1;
1894 }
1895
1896 my $enter_namespace = sub {
1897     my ($vmid, $pid, $which, $type) = @_;
1898     sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY
1899         or die "failed to open $which namespace of container $vmid: $!\n";
1900     PVE::Tools::setns(fileno($fd), $type)
1901         or die "failed to enter $which namespace of container $vmid: $!\n";
1902     close $fd;
1903 };
1904
1905 my $do_syncfs = sub {
1906     my ($vmid, $pid, $socket) = @_;
1907
1908     &$enter_namespace($vmid, $pid, 'mnt', PVE::Tools::CLONE_NEWNS);
1909
1910     # Tell the parent process to start reading our /proc/mounts
1911     print {$socket} "go\n";
1912     $socket->flush();
1913
1914     # Receive /proc/self/mounts
1915     my $mountdata = do { local $/ = undef; <$socket> };
1916     close $socket;
1917
1918     # Now sync all mountpoints...
1919     my $mounts = PVE::ProcFSTools::parse_mounts($mountdata);
1920     foreach my $mp (@$mounts) {
1921         my ($what, $dir, $fs) = @$mp;
1922         next if $fs eq 'fuse.lxcfs';
1923         eval { PVE::Tools::sync_mountpoint($dir); };
1924         warn $@ if $@;
1925     }
1926 };
1927
1928 sub sync_container_namespace {
1929     my ($vmid) = @_;
1930     my $pid = find_lxc_pid($vmid);
1931
1932     # SOCK_DGRAM is nicer for barriers but cannot be slurped
1933     socketpair my $pfd, my $cfd, AF_UNIX, SOCK_STREAM, PF_UNSPEC
1934         or die "failed to create socketpair: $!\n";
1935
1936     my $child = fork();
1937     die "fork failed: $!\n" if !defined($child);
1938
1939     if (!$child) {
1940         eval {
1941             close $pfd;
1942             &$do_syncfs($vmid, $pid, $cfd);
1943         };
1944         if (my $err = $@) {
1945             warn $err;
1946             POSIX::_exit(1);
1947         }
1948         POSIX::_exit(0);
1949     }
1950     close $cfd;
1951     my $go = <$pfd>;
1952     die "failed to enter container namespace\n" if $go ne "go\n";
1953
1954     open my $mounts, '<', "/proc/$child/mounts"
1955         or die "failed to open container's /proc/mounts: $!\n";
1956     my $mountdata = do { local $/ = undef; <$mounts> };
1957     close $mounts;
1958     print {$pfd} $mountdata;
1959     close $pfd;
1960
1961     while (waitpid($child, 0) != $child) {}
1962     die "failed to sync container namespace\n" if $? != 0;
1963 }
1964
1965 sub check_freeze_needed {
1966     my ($vmid, $config, $save_vmstate) = @_;
1967
1968     my $ret = check_running($vmid);
1969     return ($ret, $ret);
1970 }
1971
1972 sub snapshot_create {
1973     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1974
1975     my $snap = snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
1976
1977     $save_vmstate = 0 if !$snap->{vmstate};
1978
1979     my $conf = load_config($vmid);
1980
1981     my ($running, $freezefs) = check_freeze_needed($vmid, $conf, $snap->{vmstate});
1982
1983     my $drivehash = {};
1984
1985     eval {
1986         if ($freezefs) {
1987             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1988             sync_container_namespace($vmid);
1989         }
1990
1991         my $storecfg = PVE::Storage::config();
1992         foreach_mountpoint($conf, sub {
1993             my ($ms, $mountpoint) = @_;
1994
1995             return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
1996             PVE::Storage::volume_snapshot($storecfg, $mountpoint->{volume}, $snapname);
1997             $drivehash->{$ms} = 1;
1998         });
1999     };
2000     my $err = $@;
2001
2002     if ($running) {
2003         if ($freezefs) {
2004             eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
2005             warn $@ if $@;
2006         }
2007     }
2008
2009     if ($err) {
2010         warn "snapshot create failed: starting cleanup\n";
2011         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
2012         warn "$@" if $@;
2013         die "$err\n";
2014     }
2015
2016     snapshot_commit($vmid, $snapname);
2017 }
2018
2019 # Note: $drivehash is only set when called from snapshot_create.
2020 sub snapshot_delete {
2021     my ($vmid, $snapname, $force, $drivehash) = @_;
2022
2023     my $prepare = 1;
2024
2025     my $snap;
2026     my $unused = [];
2027
2028     my $unlink_parent = sub {
2029         my ($confref, $new_parent) = @_;
2030
2031         if ($confref->{parent} && $confref->{parent} eq $snapname) {
2032             if ($new_parent) {
2033                 $confref->{parent} = $new_parent;
2034             } else {
2035                 delete $confref->{parent};
2036             }
2037         }
2038     };
2039
2040     my $updatefn =  sub {
2041         my ($remove_drive) = @_;
2042
2043         my $conf = load_config($vmid);
2044
2045         if (!$drivehash) {
2046             check_lock($conf);
2047             die "you can't delete a snapshot if vm is a template\n"
2048                 if is_template($conf);
2049         }
2050
2051         $snap = $conf->{snapshots}->{$snapname};
2052
2053         die "snapshot '$snapname' does not exist\n" if !defined($snap);
2054
2055         # remove parent refs
2056         if (!$prepare) {
2057             &$unlink_parent($conf, $snap->{parent});
2058             foreach my $sn (keys %{$conf->{snapshots}}) {
2059                 next if $sn eq $snapname;
2060                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
2061             }
2062         }
2063
2064         if ($remove_drive) {
2065             if ($remove_drive eq 'vmstate') {
2066                 die "implement me - saving vmstate\n";
2067             } else {
2068                 my $value = $snap->{$remove_drive};
2069                 my $mountpoint = $remove_drive eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2070                 delete $snap->{$remove_drive};
2071                 add_unused_volume($snap, $mountpoint->{volume})
2072                     if (!is_volume_in_use($snap, $mountpoint->{volume}));
2073             }
2074         }
2075
2076         if ($prepare) {
2077             $snap->{snapstate} = 'delete';
2078         } else {
2079             delete $conf->{snapshots}->{$snapname};
2080             delete $conf->{lock} if $drivehash;
2081             foreach my $volid (@$unused) {
2082                 add_unused_volume($conf, $volid)
2083                     if (!is_volume_in_use($conf, $volid));
2084             }
2085         }
2086
2087         write_config($vmid, $conf);
2088     };
2089
2090     lock_config($vmid, $updatefn);
2091
2092     # now remove vmstate file
2093     # never set for LXC!
2094     my $storecfg = PVE::Storage::config();
2095
2096     if ($snap->{vmstate}) {
2097         die "implement me - saving vmstate\n";
2098     };
2099
2100     # now remove all volume snapshots
2101     foreach_mountpoint($snap, sub {
2102         my ($ms, $mountpoint) = @_;
2103
2104         return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
2105         if (!$drivehash || $drivehash->{$ms}) {
2106             eval { PVE::Storage::volume_snapshot_delete($storecfg, $mountpoint->{volume}, $snapname); };
2107             if (my $err = $@) {
2108                 die $err if !$force;
2109                 warn $err;
2110             }
2111         }
2112
2113         # save changes (remove mp from snapshot)
2114         lock_config($vmid, $updatefn, $ms) if !$force;
2115         push @$unused, $mountpoint->{volume};
2116     });
2117
2118     # now cleanup config
2119     $prepare = 0;
2120     lock_config($vmid, $updatefn);
2121 }
2122
2123 sub snapshot_rollback {
2124     my ($vmid, $snapname) = @_;
2125
2126     my $prepare = 1;
2127
2128     my $storecfg = PVE::Storage::config();
2129
2130     my $conf = load_config($vmid);
2131
2132     my $get_snapshot_config = sub {
2133
2134         die "you can't rollback if vm is a template\n" if is_template($conf);
2135
2136         my $res = $conf->{snapshots}->{$snapname};
2137
2138         die "snapshot '$snapname' does not exist\n" if !defined($res);
2139
2140         return $res;
2141     };
2142
2143     my $snap = &$get_snapshot_config();
2144
2145     foreach_mountpoint($snap, sub {
2146         my ($ms, $mountpoint) = @_;
2147
2148         PVE::Storage::volume_rollback_is_possible($storecfg, $mountpoint->{volume}, $snapname);
2149     });
2150
2151     my $updatefn = sub {
2152
2153         $conf = load_config($vmid);
2154
2155         $snap = &$get_snapshot_config();
2156
2157         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
2158             if $snap->{snapstate};
2159
2160         if ($prepare) {
2161             check_lock($conf);
2162             PVE::Tools::run_command(['/usr/bin/lxc-stop', '-n', $vmid, '--kill'])
2163                 if check_running($vmid);
2164         }
2165
2166         die "unable to rollback vm $vmid: vm is running\n"
2167             if check_running($vmid);
2168
2169         if ($prepare) {
2170             $conf->{lock} = 'rollback';
2171         } else {
2172             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
2173             delete $conf->{lock};
2174         }
2175
2176         my $forcemachine;
2177
2178         if (!$prepare) {
2179             # copy snapshot config to current config
2180             $conf = &$snapshot_apply_config($conf, $snap);
2181             $conf->{parent} = $snapname;
2182         }
2183
2184         write_config($vmid, $conf);
2185
2186         if (!$prepare && $snap->{vmstate}) {
2187             die "implement me - save vmstate\n";
2188         }
2189     };
2190
2191     lock_config($vmid, $updatefn);
2192
2193     foreach_mountpoint($snap, sub {
2194         my ($ms, $mountpoint) = @_;
2195
2196         PVE::Storage::volume_snapshot_rollback($storecfg, $mountpoint->{volume}, $snapname);
2197     });
2198
2199     $prepare = 0;
2200     lock_config($vmid, $updatefn);
2201 }
2202
2203 sub template_create {
2204     my ($vmid, $conf) = @_;
2205
2206     my $storecfg = PVE::Storage::config();
2207
2208     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
2209     my $volid = $rootinfo->{volume};
2210
2211     die "Template feature is not available for '$volid'\n"
2212         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
2213
2214     PVE::Storage::activate_volumes($storecfg, [$volid]);
2215
2216     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
2217     $rootinfo->{volume} = $template_volid;
2218     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
2219
2220     write_config($vmid, $conf);
2221 }
2222
2223 sub is_template {
2224     my ($conf) = @_;
2225
2226     return 1 if defined $conf->{template} && $conf->{template} == 1;
2227 }
2228
2229 sub mountpoint_names {
2230     my ($reverse) = @_;
2231
2232     my @names = ('rootfs');
2233
2234     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
2235         push @names, "mp$i";
2236     }
2237
2238     return $reverse ? reverse @names : @names;
2239 }
2240
2241
2242 sub foreach_mountpoint_full {
2243     my ($conf, $reverse, $func) = @_;
2244
2245     foreach my $key (mountpoint_names($reverse)) {
2246         my $value = $conf->{$key};
2247         next if !defined($value);
2248         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2249         next if !defined($mountpoint);
2250
2251         &$func($key, $mountpoint);
2252     }
2253 }
2254
2255 sub foreach_mountpoint {
2256     my ($conf, $func) = @_;
2257
2258     foreach_mountpoint_full($conf, 0, $func);
2259 }
2260
2261 sub foreach_mountpoint_reverse {
2262     my ($conf, $func) = @_;
2263
2264     foreach_mountpoint_full($conf, 1, $func);
2265 }
2266
2267 sub check_ct_modify_config_perm {
2268     my ($rpcenv, $authuser, $vmid, $pool, $newconf, $delete) = @_;
2269
2270     return 1 if $authuser eq 'root@pam';
2271
2272     my $check = sub {
2273         my ($opt, $delete) = @_;
2274         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2275             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2276         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2277             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2278             return if $delete;
2279             my $data = $opt eq 'rootfs' ? parse_ct_rootfs($newconf->{$opt})
2280                                         : parse_ct_mountpoint($newconf->{$opt});
2281             raise_perm_exc("mountpoint type $data->{type}") if $data->{type} ne 'volume';
2282         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2283             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2284         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2285                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2286             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2287         } else {
2288             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2289         }
2290     };
2291
2292     foreach my $opt (keys %$newconf) {
2293         &$check($opt, 0);
2294     }
2295     foreach my $opt (@$delete) {
2296         &$check($opt, 1);
2297     }
2298
2299     return 1;
2300 }
2301
2302 sub umount_all {
2303     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2304
2305     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2306     my $volid_list = get_vm_volumes($conf);
2307
2308     foreach_mountpoint_reverse($conf, sub {
2309         my ($ms, $mountpoint) = @_;
2310
2311         my $volid = $mountpoint->{volume};
2312         my $mount = $mountpoint->{mp};
2313
2314         return if !$volid || !$mount;
2315
2316         my $mount_path = "$rootdir/$mount";
2317         $mount_path =~ s!/+!/!g;
2318
2319         return if !PVE::ProcFSTools::is_mounted($mount_path);
2320
2321         eval {
2322             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2323         };
2324         if (my $err = $@) {
2325             if ($noerr) {
2326                 warn $err;
2327             } else {
2328                 die $err;
2329             }
2330         }
2331     });
2332 }
2333
2334 sub mount_all {
2335     my ($vmid, $storage_cfg, $conf) = @_;
2336
2337     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2338     File::Path::make_path($rootdir);
2339
2340     my $volid_list = get_vm_volumes($conf);
2341     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2342
2343     eval {
2344         foreach_mountpoint($conf, sub {
2345             my ($ms, $mountpoint) = @_;
2346
2347             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2348         });
2349     };
2350     if (my $err = $@) {
2351         warn "mounting container failed\n";
2352         umount_all($vmid, $storage_cfg, $conf, 1);
2353         die $err;
2354     }
2355
2356     return $rootdir;
2357 }
2358
2359
2360 sub mountpoint_mount_path {
2361     my ($mountpoint, $storage_cfg, $snapname) = @_;
2362
2363     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2364 }
2365
2366 my $check_mount_path = sub {
2367     my ($path) = @_;
2368     $path = File::Spec->canonpath($path);
2369     my $real = Cwd::realpath($path);
2370     if ($real ne $path) {
2371         die "mount path modified by symlink: $path != $real";
2372     }
2373 };
2374
2375 sub query_loopdev {
2376     my ($path) = @_;
2377     my $found;
2378     my $parser = sub {
2379         my $line = shift;
2380         if ($line =~ m@^(/dev/loop\d+):@) {
2381             $found = $1;
2382         }
2383     };
2384     my $cmd = ['losetup', '--associated', $path];
2385     PVE::Tools::run_command($cmd, outfunc => $parser);
2386     return $found;
2387 }
2388
2389 # Run a function with a file attached to a loop device.
2390 # The loop device is always detached afterwards (or set to autoclear).
2391 # Returns the loop device.
2392 sub run_with_loopdev {
2393     my ($func, $file) = @_;
2394     my $device;
2395     my $parser = sub {
2396         my $line = shift;
2397         if ($line =~ m@^(/dev/loop\d+)$@) {
2398             $device = $1;
2399         }
2400     };
2401     PVE::Tools::run_command(['losetup', '--show', '-f', $file], outfunc => $parser);
2402     die "failed to setup loop device for $file\n" if !$device;
2403     eval { &$func($device); };
2404     my $err = $@;
2405     PVE::Tools::run_command(['losetup', '-d', $device]);
2406     die $err if $err;
2407     return $device;
2408 }
2409
2410 sub bindmount {
2411     my ($dir, $dest, $ro, @extra_opts) = @_;
2412     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $dir, $dest]);
2413     if ($ro) {
2414         eval { PVE::Tools::run_command(['mount', '-o', 'bind,remount,ro', $dest]); };
2415         if (my $err = $@) {
2416             warn "bindmount error\n";
2417             # don't leave writable bind-mounts behind...
2418             PVE::Tools::run_command(['umount', $dest]);
2419             die $err;
2420         }
2421     }
2422 }
2423
2424 # use $rootdir = undef to just return the corresponding mount path
2425 sub mountpoint_mount {
2426     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2427
2428     my $volid = $mountpoint->{volume};
2429     my $mount = $mountpoint->{mp};
2430     my $type = $mountpoint->{type};
2431     my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota};
2432     my $mounted_dev;
2433
2434     return if !$volid || !$mount;
2435
2436     my $mount_path;
2437
2438     if (defined($rootdir)) {
2439         $rootdir =~ s!/+$!!;
2440         $mount_path = "$rootdir/$mount";
2441         $mount_path =~ s!/+!/!g;
2442         &$check_mount_path($mount_path);
2443         File::Path::mkpath($mount_path);
2444     }
2445
2446     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2447
2448     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2449
2450     my $optstring = '';
2451     if (defined($mountpoint->{acl})) {
2452         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
2453     }
2454     my $readonly = $mountpoint->{ro};
2455
2456     my @extra_opts = ('-o', $optstring);
2457
2458     if ($storage) {
2459
2460         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2461         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2462
2463         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2464             PVE::Storage::parse_volname($storage_cfg, $volid);
2465
2466         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2467
2468         if ($format eq 'subvol') {
2469             if ($mount_path) {
2470                 if ($snapname) {
2471                     if ($scfg->{type} eq 'zfspool') {
2472                         my $path_arg = $path;
2473                         $path_arg =~ s!^/+!!;
2474                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
2475                     } else {
2476                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2477                     }
2478                 } else {
2479                     bindmount($path, $mount_path, $readonly, @extra_opts);
2480                     warn "cannot enable quota control for bind mounted subvolumes\n" if $quota;
2481                 }
2482             }
2483             return wantarray ? ($path, 0, $mounted_dev) : $path;
2484         } elsif ($format eq 'raw' || $format eq 'iso') {
2485             my $domount = sub {
2486                 my ($path) = @_;
2487                 if ($mount_path) {
2488                     if ($format eq 'iso') {
2489                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2490                     } elsif ($isBase || defined($snapname)) {
2491                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2492                     } else {
2493                         if ($quota) {
2494                             push @extra_opts, '-o', 'usrjquota=aquota.user,grpjquota=aquota.group,jqfmt=vfsv0';
2495                         }
2496                         push @extra_opts, '-o', 'ro' if $readonly;
2497                         PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2498                     }
2499                 }
2500             };
2501             my $use_loopdev = 0;
2502             if ($scfg->{path}) {
2503                 $mounted_dev = run_with_loopdev($domount, $path);
2504                 $use_loopdev = 1;
2505             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2506                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2507                 $mounted_dev = $path;
2508                 &$domount($path);
2509             } else {
2510                 die "unsupported storage type '$scfg->{type}'\n";
2511             }
2512             return wantarray ? ($path, $use_loopdev, $mounted_dev) : $path;
2513         } else {
2514             die "unsupported image format '$format'\n";
2515         }
2516     } elsif ($type eq 'device') {
2517                         push @extra_opts, '-o', 'ro' if $readonly;
2518         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
2519         return wantarray ? ($volid, 0, $volid) : $volid;
2520     } elsif ($type eq 'bind') {
2521         die "directory '$volid' does not exist\n" if ! -d $volid;
2522         &$check_mount_path($volid);
2523         bindmount($volid, $mount_path, $readonly, @extra_opts) if $mount_path;
2524         warn "cannot enable quota control for bind mounts\n" if $quota;
2525         return wantarray ? ($volid, 0, undef) : $volid;
2526     }
2527
2528     die "unsupported storage";
2529 }
2530
2531 sub get_vm_volumes {
2532     my ($conf, $excludes) = @_;
2533
2534     my $vollist = [];
2535
2536     foreach_mountpoint($conf, sub {
2537         my ($ms, $mountpoint) = @_;
2538
2539         return if $excludes && $ms eq $excludes;
2540
2541         my $volid = $mountpoint->{volume};
2542
2543         return if !$volid || $mountpoint->{type} ne 'volume';
2544
2545         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2546         return if !$sid;
2547
2548         push @$vollist, $volid;
2549     });
2550
2551     return $vollist;
2552 }
2553
2554 sub mkfs {
2555     my ($dev, $rootuid, $rootgid) = @_;
2556
2557     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2558                              '-E', "root_owner=$rootuid:$rootgid",
2559                              $dev]);
2560 }
2561
2562 sub format_disk {
2563     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2564
2565     if ($volid =~ m!^/dev/.+!) {
2566         mkfs($volid);
2567         return;
2568     }
2569
2570     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2571
2572     die "cannot format volume '$volid' with no storage\n" if !$storage;
2573
2574     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2575
2576     my $path = PVE::Storage::path($storage_cfg, $volid);
2577
2578     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2579         PVE::Storage::parse_volname($storage_cfg, $volid);
2580
2581     die "cannot format volume '$volid' (format == $format)\n"
2582         if $format ne 'raw';
2583
2584     mkfs($path, $rootuid, $rootgid);
2585 }
2586
2587 sub destroy_disks {
2588     my ($storecfg, $vollist) = @_;
2589
2590     foreach my $volid (@$vollist) {
2591         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2592         warn $@ if $@;
2593     }
2594 }
2595
2596 sub create_disks {
2597     my ($storecfg, $vmid, $settings, $conf) = @_;
2598
2599     my $vollist = [];
2600
2601     eval {
2602         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2603         my $chown_vollist = [];
2604
2605         foreach_mountpoint($settings, sub {
2606             my ($ms, $mountpoint) = @_;
2607
2608             my $volid = $mountpoint->{volume};
2609             my $mp = $mountpoint->{mp};
2610
2611             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2612
2613             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2614                 my ($storeid, $size_gb) = ($1, $2);
2615
2616                 my $size_kb = int(${size_gb}*1024) * 1024;
2617
2618                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2619                 # fixme: use better naming ct-$vmid-disk-X.raw?
2620
2621                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2622                     if ($size_kb > 0) {
2623                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2624                                                            undef, $size_kb);
2625                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2626                     } else {
2627                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2628                                                            undef, 0);
2629                         push @$chown_vollist, $volid;
2630                     }
2631                 } elsif ($scfg->{type} eq 'zfspool') {
2632
2633                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2634                                                        undef, $size_kb);
2635                     push @$chown_vollist, $volid;
2636                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2637
2638                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2639                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2640
2641                 } elsif ($scfg->{type} eq 'rbd') {
2642
2643                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2644                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2645                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2646                 } else {
2647                     die "unable to create containers on storage type '$scfg->{type}'\n";
2648                 }
2649                 push @$vollist, $volid;
2650                 $mountpoint->{volume} = $volid;
2651                 $mountpoint->{size} = $size_kb * 1024;
2652                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2653             } else {
2654                 # use specified/existing volid/dir/device
2655                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2656             }
2657         });
2658
2659         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2660         foreach my $volid (@$chown_vollist) {
2661             my $path = PVE::Storage::path($storecfg, $volid, undef);
2662             chown($rootuid, $rootgid, $path);
2663         }
2664         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2665     };
2666     # free allocated images on error
2667     if (my $err = $@) {
2668         destroy_disks($storecfg, $vollist);
2669         die $err;
2670     }
2671     return $vollist;
2672 }
2673
2674 # bash completion helper
2675
2676 sub complete_os_templates {
2677     my ($cmdname, $pname, $cvalue) = @_;
2678
2679     my $cfg = PVE::Storage::config();
2680
2681     my $storeid;
2682
2683     if ($cvalue =~ m/^([^:]+):/) {
2684         $storeid = $1;
2685     }
2686
2687     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2688     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2689
2690     my $res = [];
2691     foreach my $id (keys %$data) {
2692         foreach my $item (@{$data->{$id}}) {
2693             push @$res, $item->{volid} if defined($item->{volid});
2694         }
2695     }
2696
2697     return $res;
2698 }
2699
2700 my $complete_ctid_full = sub {
2701     my ($running) = @_;
2702
2703     my $idlist = vmstatus();
2704
2705     my $active_hash = list_active_containers();
2706
2707     my $res = [];
2708
2709     foreach my $id (keys %$idlist) {
2710         my $d = $idlist->{$id};
2711         if (defined($running)) {
2712             next if $d->{template};
2713             next if $running && !$active_hash->{$id};
2714             next if !$running && $active_hash->{$id};
2715         }
2716         push @$res, $id;
2717
2718     }
2719     return $res;
2720 };
2721
2722 sub complete_ctid {
2723     return &$complete_ctid_full();
2724 }
2725
2726 sub complete_ctid_stopped {
2727     return &$complete_ctid_full(0);
2728 }
2729
2730 sub complete_ctid_running {
2731     return &$complete_ctid_full(1);
2732 }
2733
2734 sub parse_id_maps {
2735     my ($conf) = @_;
2736
2737     my $id_map = [];
2738     my $rootuid = 0;
2739     my $rootgid = 0;
2740
2741     my $lxc = $conf->{lxc};
2742     foreach my $entry (@$lxc) {
2743         my ($key, $value) = @$entry;
2744         next if $key ne 'lxc.id_map';
2745         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2746             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2747             push @$id_map, [$type, $ct, $host, $length];
2748             if ($ct == 0) {
2749                 $rootuid = $host if $type eq 'u';
2750                 $rootgid = $host if $type eq 'g';
2751             }
2752         } else {
2753             die "failed to parse id_map: $value\n";
2754         }
2755     }
2756
2757     if (!@$id_map && $conf->{unprivileged}) {
2758         # Should we read them from /etc/subuid?
2759         $id_map = [ ['u', '0', '100000', '65536'],
2760                     ['g', '0', '100000', '65536'] ];
2761         $rootuid = $rootgid = 100000;
2762     }
2763
2764     return ($id_map, $rootuid, $rootgid);
2765 }
2766
2767 sub userns_command {
2768     my ($id_map) = @_;
2769     if (@$id_map) {
2770         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2771     }
2772     return [];
2773 }
2774
2775 sub set_lock {
2776     my ($vmid, $lock) = @_;
2777     my $conf;
2778     lock_config($vmid, sub {
2779         $conf = load_config($vmid);
2780         check_lock($conf);
2781         $conf->{lock} = $lock;
2782         write_config($vmid, $conf);
2783     });
2784     return $conf;
2785 }
2786
2787 sub remove_lock {
2788     my ($vmid, $lock) = @_;
2789     lock_config($vmid, sub {
2790         my $conf = load_config($vmid);
2791         if (!$conf->{lock}) {
2792             die "no lock found trying to remove lock '$lock'\n";
2793         } elsif (defined($lock) && $conf->{lock} ne $lock) {
2794             die "found lock '$conf->{lock}' trying to remove lock '$lock'\n";
2795         }
2796         delete $conf->{lock};
2797         write_config($vmid, $conf);
2798     });
2799 }
2800
2801 1;