src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use Socket;
   8
   9 use File::Path;
  10 use File::Spec;
  11 use Cwd qw();
  12 use Fcntl qw(O_RDONLY);
  13
  14 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  15 use PVE::Exception qw(raise_perm_exc);
  16 use PVE::Storage;
  17 use PVE::SafeSyslog;
  18 use PVE::INotify;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full);
  21 use PVE::Network;
  22 use PVE::AccessControl;
  23 use PVE::ProcFSTools;
  24 use Time::HiRes qw (gettimeofday);
  25
  26 use Data::Dumper;
  27
  28 my $nodename = PVE::INotify::nodename();
  29
  30 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  31
  32 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  33                           '--xattrs',
  34                           '--xattrs-include=user.*',
  35                           '--xattrs-include=security.capability',
  36                           '--warning=no-xattr-write' ];
  37
  38 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  39
  40 my $rootfs_desc = {
  41     volume => {
  42         type => 'string',
  43         default_key => 1,
  44         format => 'pve-lxc-mp-string',
  45         format_description => 'volume',
  46         description => 'Volume, device or directory to mount into the container.',
  47     },
  48     backup => {
  49         type => 'boolean',
  50         format_description => '[1|0]',
  51         description => 'Whether to include the mountpoint in backups.',
  52         optional => 1,
  53     },
  54     size => {
  55         type => 'string',
  56         format => 'disk-size',
  57         format_description => 'DiskSize',
  58         description => 'Volume size (read only value).',
  59         optional => 1,
  60     },
  61     acl => {
  62         type => 'boolean',
  63         format_description => 'acl',
  64         description => 'Explicitly enable or disable ACL support.',
  65         optional => 1,
  66     },
  67     ro => {
  68         type => 'boolean',
  69         format_description => 'ro',
  70         description => 'Read-only mountpoint (not supported with bind mounts)',
  71         optional => 1,
  72     },
  73     quota => {
  74         type => 'boolean',
  75         format_description => '[0|1]',
  76         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
  77         optional => 1,
  78     },
  79 };
  80
  81 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  82     type => 'string', format => $rootfs_desc,
  83     description => "Use volume as container root.",
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  88     description => "The name of the snapshot.",
  89     type => 'string', format => 'pve-configid',
  90     maxLength => 40,
  91 });
  92
  93 my $confdesc = {
  94     lock => {
  95         optional => 1,
  96         type => 'string',
  97         description => "Lock/unlock the VM.",
  98         enum => [qw(migrate backup snapshot rollback)],
  99     },
 100     onboot => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Specifies whether a VM will be started during system bootup.",
 104         default => 0,
 105     },
 106     startup => get_standard_option('pve-startup-order'),
 107     template => {
 108         optional => 1,
 109         type => 'boolean',
 110         description => "Enable/disable Template.",
 111         default => 0,
 112     },
 113     arch => {
 114         optional => 1,
 115         type => 'string',
 116         enum => ['amd64', 'i386'],
 117         description => "OS architecture type.",
 118         default => 'amd64',
 119     },
 120     ostype => {
 121         optional => 1,
 122         type => 'string',
 123         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux', 'alpine', 'unmanaged'],
 124         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 125     },
 126     console => {
 127         optional => 1,
 128         type => 'boolean',
 129         description => "Attach a console device (/dev/console) to the container.",
 130         default => 1,
 131     },
 132     tty => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "Specify the number of tty available to the container",
 136         minimum => 0,
 137         maximum => 6,
 138         default => 2,
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1024,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     swap => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of SWAP for the VM in MB.",
 167         minimum => 0,
 168         default => 512,
 169     },
 170     hostname => {
 171         optional => 1,
 172         description => "Set a host name for the container.",
 173         type => 'string', format => 'dns-name',
 174         maxLength => 255,
 175     },
 176     description => {
 177         optional => 1,
 178         type => 'string',
 179         description => "Container description. Only used on the configuration web interface.",
 180     },
 181     searchdomain => {
 182         optional => 1,
 183         type => 'string', format => 'dns-name-list',
 184         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 185     },
 186     nameserver => {
 187         optional => 1,
 188         type => 'string', format => 'address-list',
 189         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 190     },
 191     rootfs => get_standard_option('pve-ct-rootfs'),
 192     parent => {
 193         optional => 1,
 194         type => 'string', format => 'pve-configid',
 195         maxLength => 40,
 196         description => "Parent snapshot name. This is used internally, and should not be modified.",
 197     },
 198     snaptime => {
 199         optional => 1,
 200         description => "Timestamp for snapshots.",
 201         type => 'integer',
 202         minimum => 0,
 203     },
 204     cmode => {
 205         optional => 1,
 206         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 207         type => 'string',
 208         enum => ['shell', 'console', 'tty'],
 209         default => 'tty',
 210     },
 211     protection => {
 212         optional => 1,
 213         type => 'boolean',
 214         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 215         default => 0,
 216     },
 217     unprivileged => {
 218         optional => 1,
 219         type => 'boolean',
 220         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 221         default => 0,
 222     },
 223 };
 224
 225 my $valid_lxc_conf_keys = {
 226     'lxc.include' => 1,
 227     'lxc.arch' => 1,
 228     'lxc.utsname' => 1,
 229     'lxc.haltsignal' => 1,
 230     'lxc.rebootsignal' => 1,
 231     'lxc.stopsignal' => 1,
 232     'lxc.init_cmd' => 1,
 233     'lxc.network.type' => 1,
 234     'lxc.network.flags' => 1,
 235     'lxc.network.link' => 1,
 236     'lxc.network.mtu' => 1,
 237     'lxc.network.name' => 1,
 238     'lxc.network.hwaddr' => 1,
 239     'lxc.network.ipv4' => 1,
 240     'lxc.network.ipv4.gateway' => 1,
 241     'lxc.network.ipv6' => 1,
 242     'lxc.network.ipv6.gateway' => 1,
 243     'lxc.network.script.up' => 1,
 244     'lxc.network.script.down' => 1,
 245     'lxc.pts' => 1,
 246     'lxc.console.logfile' => 1,
 247     'lxc.console' => 1,
 248     'lxc.tty' => 1,
 249     'lxc.devttydir' => 1,
 250     'lxc.hook.autodev' => 1,
 251     'lxc.autodev' => 1,
 252     'lxc.kmsg' => 1,
 253     'lxc.mount' => 1,
 254     'lxc.mount.entry' => 1,
 255     'lxc.mount.auto' => 1,
 256     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 257     'lxc.rootfs.mount' => 1,
 258     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 259                             ', please use mountpoint options in the "rootfs" key',
 260     # lxc.cgroup.*
 261     'lxc.cap.drop' => 1,
 262     'lxc.cap.keep' => 1,
 263     'lxc.aa_profile' => 1,
 264     'lxc.aa_allow_incomplete' => 1,
 265     'lxc.se_context' => 1,
 266     'lxc.seccomp' => 1,
 267     'lxc.id_map' => 1,
 268     'lxc.hook.pre-start' => 1,
 269     'lxc.hook.pre-mount' => 1,
 270     'lxc.hook.mount' => 1,
 271     'lxc.hook.start' => 1,
 272     'lxc.hook.stop' => 1,
 273     'lxc.hook.post-stop' => 1,
 274     'lxc.hook.clone' => 1,
 275     'lxc.hook.destroy' => 1,
 276     'lxc.loglevel' => 1,
 277     'lxc.logfile' => 1,
 278     'lxc.start.auto' => 1,
 279     'lxc.start.delay' => 1,
 280     'lxc.start.order' => 1,
 281     'lxc.group' => 1,
 282     'lxc.environment' => 1,
 283 };
 284
 285 my $netconf_desc = {
 286     type => {
 287         type => 'string',
 288         optional => 1,
 289         description => "Network interface type.",
 290         enum => [qw(veth)],
 291     },
 292     name => {
 293         type => 'string',
 294         format_description => 'String',
 295         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 296         pattern => '[-_.\w\d]+',
 297     },
 298     bridge => {
 299         type => 'string',
 300         format_description => 'vmbr<Number>',
 301         description => 'Bridge to attach the network device to.',
 302         pattern => '[-_.\w\d]+',
 303         optional => 1,
 304     },
 305     hwaddr => {
 306         type => 'string',
 307         format_description => 'MAC',
 308         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 309         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 310         optional => 1,
 311     },
 312     mtu => {
 313         type => 'integer',
 314         format_description => 'Number',
 315         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 316         minimum => 64, # minimum ethernet frame is 64 bytes
 317         optional => 1,
 318     },
 319     ip => {
 320         type => 'string',
 321         format => 'pve-ipv4-config',
 322         format_description => 'IPv4Format/CIDR',
 323         description => 'IPv4 address in CIDR format.',
 324         optional => 1,
 325     },
 326     gw => {
 327         type => 'string',
 328         format => 'ipv4',
 329         format_description => 'GatewayIPv4',
 330         description => 'Default gateway for IPv4 traffic.',
 331         optional => 1,
 332     },
 333     ip6 => {
 334         type => 'string',
 335         format => 'pve-ipv6-config',
 336         format_description => 'IPv6Format/CIDR',
 337         description => 'IPv6 address in CIDR format.',
 338         optional => 1,
 339     },
 340     gw6 => {
 341         type => 'string',
 342         format => 'ipv6',
 343         format_description => 'GatewayIPv6',
 344         description => 'Default gateway for IPv6 traffic.',
 345         optional => 1,
 346     },
 347     firewall => {
 348         type => 'boolean',
 349         format_description => '[1|0]',
 350         description => "Controls whether this interface's firewall rules should be used.",
 351         optional => 1,
 352     },
 353     tag => {
 354         type => 'integer',
 355         format_description => 'VlanNo',
 356         minimum => '2',
 357         maximum => '4094',
 358         description => "VLAN tag for this interface.",
 359         optional => 1,
 360     },
 361     trunks => {
 362         type => 'string',
 363         pattern => qr/\d+(?:;\d+)*/,
 364         format_description => 'vlanid[;vlanid...]',
 365         description => "VLAN ids to pass through the interface",
 366         optional => 1,
 367     },
 368 };
 369 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 370
 371 my $MAX_LXC_NETWORKS = 10;
 372 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 373     $confdesc->{"net$i"} = {
 374         optional => 1,
 375         type => 'string', format => $netconf_desc,
 376         description => "Specifies network interfaces for the container.",
 377     };
 378 }
 379
 380 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 381 sub verify_lxc_mp_string{
 382     my ($mp, $noerr) = @_;
 383
 384     # do not allow:
 385     # /./ or /../
 386     # /. or /.. at the end
 387     # ../ at the beginning
 388
 389     if($mp =~ m@/\.\.?/@ ||
 390        $mp =~ m@/\.\.?$@ ||
 391        $mp =~ m@^\.\./@){
 392         return undef if $noerr;
 393         die "$mp contains illegal character sequences\n";
 394     }
 395     return $mp;
 396 }
 397
 398 my $mp_desc = {
 399     %$rootfs_desc,
 400     mp => {
 401         type => 'string',
 402         format => 'pve-lxc-mp-string',
 403         format_description => 'Path',
 404         description => 'Path to the mountpoint as seen from inside the container.',
 405     },
 406 };
 407 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 408
 409 my $unuseddesc = {
 410     optional => 1,
 411     type => 'string', format => 'pve-volume-id',
 412     description => "Reference to unused volumes.",
 413 };
 414
 415 my $MAX_MOUNT_POINTS = 10;
 416 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 417     $confdesc->{"mp$i"} = {
 418         optional => 1,
 419         type => 'string', format => $mp_desc,
 420         description => "Use volume as container mount point (experimental feature).",
 421         optional => 1,
 422     };
 423 }
 424
 425 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 426 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 427     $confdesc->{"unused$i"} = $unuseddesc;
 428 }
 429
 430 sub write_pct_config {
 431     my ($filename, $conf) = @_;
 432
 433     delete $conf->{snapstate}; # just to be sure
 434
 435     my $generate_raw_config = sub {
 436         my ($conf) = @_;
 437
 438         my $raw = '';
 439
 440         # add description as comment to top of file
 441         my $descr = $conf->{description} || '';
 442         foreach my $cl (split(/\n/, $descr)) {
 443             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 444         }
 445
 446         foreach my $key (sort keys %$conf) {
 447             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 448                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 449             my $value = $conf->{$key};
 450             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 451             $raw .= "$key: $value\n";
 452         }
 453
 454         if (my $lxcconf = $conf->{lxc}) {
 455             foreach my $entry (@$lxcconf) {
 456                 my ($k, $v) = @$entry;
 457                 $raw .= "$k: $v\n";
 458             }
 459         }
 460
 461         return $raw;
 462     };
 463
 464     my $raw = &$generate_raw_config($conf);
 465
 466     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 467         $raw .= "\n[$snapname]\n";
 468         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 469     }
 470
 471     return $raw;
 472 }
 473
 474 sub check_type {
 475     my ($key, $value) = @_;
 476
 477     die "unknown setting '$key'\n" if !$confdesc->{$key};
 478
 479     my $type = $confdesc->{$key}->{type};
 480
 481     if (!defined($value)) {
 482         die "got undefined value\n";
 483     }
 484
 485     if ($value =~ m/[\n\r]/) {
 486         die "property contains a line feed\n";
 487     }
 488
 489     if ($type eq 'boolean') {
 490         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 491         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 492         die "type check ('boolean') failed - got '$value'\n";
 493     } elsif ($type eq 'integer') {
 494         return int($1) if $value =~ m/^(\d+)$/;
 495         die "type check ('integer') failed - got '$value'\n";
 496     } elsif ($type eq 'number') {
 497         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 498         die "type check ('number') failed - got '$value'\n";
 499     } elsif ($type eq 'string') {
 500         if (my $fmt = $confdesc->{$key}->{format}) {
 501             PVE::JSONSchema::check_format($fmt, $value);
 502             return $value;
 503         }
 504         return $value;
 505     } else {
 506         die "internal error"
 507     }
 508 }
 509
 510 sub parse_pct_config {
 511     my ($filename, $raw) = @_;
 512
 513     return undef if !defined($raw);
 514
 515     my $res = {
 516         digest => Digest::SHA::sha1_hex($raw),
 517         snapshots => {},
 518     };
 519
 520     $filename =~ m|/lxc/(\d+).conf$|
 521         || die "got strange filename '$filename'";
 522
 523     my $vmid = $1;
 524
 525     my $conf = $res;
 526     my $descr = '';
 527     my $section = '';
 528
 529     my @lines = split(/\n/, $raw);
 530     foreach my $line (@lines) {
 531         next if $line =~ m/^\s*$/;
 532
 533         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 534             $section = $1;
 535             $conf->{description} = $descr if $descr;
 536             $descr = '';
 537             $conf = $res->{snapshots}->{$section} = {};
 538             next;
 539         }
 540
 541         if ($line =~ m/^\#(.*)\s*$/) {
 542             $descr .= PVE::Tools::decode_text($1) . "\n";
 543             next;
 544         }
 545
 546         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 547             my $key = $1;
 548             my $value = $3;
 549             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 550             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 551                 push @{$conf->{lxc}}, [$key, $value];
 552             } elsif (my $errmsg = $validity) {
 553                 warn "vm $vmid - $key: $errmsg\n";
 554             } else {
 555                 warn "vm $vmid - unable to parse config: $line\n";
 556             }
 557         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 558             $descr .= PVE::Tools::decode_text($2);
 559         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 560             $conf->{snapstate} = $1;
 561         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 562             my $key = $1;
 563             my $value = $2;
 564             eval { $value = check_type($key, $value); };
 565             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 566             $conf->{$key} = $value;
 567         } else {
 568             warn "vm $vmid - unable to parse config: $line\n";
 569         }
 570     }
 571
 572     $conf->{description} = $descr if $descr;
 573
 574     delete $res->{snapstate}; # just to be sure
 575
 576     return $res;
 577 }
 578
 579 sub config_list {
 580     my $vmlist = PVE::Cluster::get_vmlist();
 581     my $res = {};
 582     return $res if !$vmlist || !$vmlist->{ids};
 583     my $ids = $vmlist->{ids};
 584
 585     foreach my $vmid (keys %$ids) {
 586         next if !$vmid; # skip CT0
 587         my $d = $ids->{$vmid};
 588         next if !$d->{node} || $d->{node} ne $nodename;
 589         next if !$d->{type} || $d->{type} ne 'lxc';
 590         $res->{$vmid}->{type} = 'lxc';
 591     }
 592     return $res;
 593 }
 594
 595 sub cfs_config_path {
 596     my ($vmid, $node) = @_;
 597
 598     $node = $nodename if !$node;
 599     return "nodes/$node/lxc/$vmid.conf";
 600 }
 601
 602 sub config_file {
 603     my ($vmid, $node) = @_;
 604
 605     my $cfspath = cfs_config_path($vmid, $node);
 606     return "/etc/pve/$cfspath";
 607 }
 608
 609 sub load_config {
 610     my ($vmid, $node) = @_;
 611
 612     $node = $nodename if !$node;
 613     my $cfspath = cfs_config_path($vmid, $node);
 614
 615     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 616     die "container $vmid does not exist\n" if !defined($conf);
 617
 618     return $conf;
 619 }
 620
 621 sub create_config {
 622     my ($vmid, $conf) = @_;
 623
 624     my $dir = "/etc/pve/nodes/$nodename/lxc";
 625     mkdir $dir;
 626
 627     write_config($vmid, $conf);
 628 }
 629
 630 sub destroy_config {
 631     my ($vmid) = @_;
 632
 633     unlink config_file($vmid, $nodename);
 634 }
 635
 636 sub write_config {
 637     my ($vmid, $conf) = @_;
 638
 639     my $cfspath = cfs_config_path($vmid);
 640
 641     PVE::Cluster::cfs_write_file($cfspath, $conf);
 642 }
 643
 644 # flock: we use one file handle per process, so lock file
 645 # can be called multiple times and will succeed for the same process.
 646
 647 my $lock_handles =  {};
 648 my $lockdir = "/run/lock/lxc";
 649
 650 sub config_file_lock {
 651     my ($vmid) = @_;
 652
 653     return "$lockdir/pve-config-${vmid}.lock";
 654 }
 655
 656 sub lock_config_full {
 657     my ($vmid, $timeout, $code, @param) = @_;
 658
 659     my $filename = config_file_lock($vmid);
 660
 661     mkdir $lockdir if !-d $lockdir;
 662
 663     my $res = lock_file($filename, $timeout, $code, @param);
 664
 665     die $@ if $@;
 666
 667     return $res;
 668 }
 669
 670 sub lock_config_mode {
 671     my ($vmid, $timeout, $shared, $code, @param) = @_;
 672
 673     my $filename = config_file_lock($vmid);
 674
 675     mkdir $lockdir if !-d $lockdir;
 676
 677     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
 678
 679     die $@ if $@;
 680
 681     return $res;
 682 }
 683
 684 sub lock_config {
 685     my ($vmid, $code, @param) = @_;
 686
 687     return lock_config_full($vmid, 10, $code, @param);
 688 }
 689
 690 sub option_exists {
 691     my ($name) = @_;
 692
 693     return defined($confdesc->{$name});
 694 }
 695
 696 # add JSON properties for create and set function
 697 sub json_config_properties {
 698     my $prop = shift;
 699
 700     foreach my $opt (keys %$confdesc) {
 701         next if $opt eq 'parent' || $opt eq 'snaptime';
 702         next if $prop->{$opt};
 703         $prop->{$opt} = $confdesc->{$opt};
 704     }
 705
 706     return $prop;
 707 }
 708
 709 # container status helpers
 710
 711 sub list_active_containers {
 712
 713     my $filename = "/proc/net/unix";
 714
 715     # similar test is used by lcxcontainers.c: list_active_containers
 716     my $res = {};
 717
 718     my $fh = IO::File->new ($filename, "r");
 719     return $res if !$fh;
 720
 721     while (defined(my $line = <$fh>)) {
 722         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 723             my $path = $1;
 724             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 725                 $res->{$1} = 1;
 726             }
 727         }
 728     }
 729
 730     close($fh);
 731
 732     return $res;
 733 }
 734
 735 # warning: this is slow
 736 sub check_running {
 737     my ($vmid) = @_;
 738
 739     my $active_hash = list_active_containers();
 740
 741     return 1 if defined($active_hash->{$vmid});
 742
 743     return undef;
 744 }
 745
 746 sub get_container_disk_usage {
 747     my ($vmid, $pid) = @_;
 748
 749     return PVE::Tools::df("/proc/$pid/root/", 1);
 750 }
 751
 752 my $last_proc_vmid_stat;
 753
 754 my $parse_cpuacct_stat = sub {
 755     my ($vmid) = @_;
 756
 757     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 758
 759     my $stat = {};
 760
 761     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 762
 763         $stat->{utime} = $1;
 764         $stat->{stime} = $2;
 765
 766     }
 767
 768     return $stat;
 769 };
 770
 771 sub vmstatus {
 772     my ($opt_vmid) = @_;
 773
 774     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 775
 776     my $active_hash = list_active_containers();
 777
 778     my $cpucount = $cpuinfo->{cpus} || 1;
 779
 780     my $cdtime = gettimeofday;
 781
 782     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 783
 784     foreach my $vmid (keys %$list) {
 785         my $d = $list->{$vmid};
 786
 787         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 788         warn $@ if $@; # ignore errors (consider them stopped)
 789
 790         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 791
 792         my $cfspath = cfs_config_path($vmid);
 793         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 794
 795         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 796         $d->{name} =~ s/[\s]//g;
 797
 798         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 799
 800         if ($d->{pid}) {
 801             my $res = get_container_disk_usage($vmid, $d->{pid});
 802             $d->{disk} = $res->{used};
 803             $d->{maxdisk} = $res->{total};
 804         } else {
 805             $d->{disk} = 0;
 806             # use 4GB by default ??
 807             if (my $rootfs = $conf->{rootfs}) {
 808                 my $rootinfo = parse_ct_rootfs($rootfs);
 809                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 810             } else {
 811                 $d->{maxdisk} = 4*1024*1024*1024;
 812             }
 813         }
 814
 815         $d->{mem} = 0;
 816         $d->{swap} = 0;
 817         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 818         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 819
 820         $d->{uptime} = 0;
 821         $d->{cpu} = 0;
 822
 823         $d->{netout} = 0;
 824         $d->{netin} = 0;
 825
 826         $d->{diskread} = 0;
 827         $d->{diskwrite} = 0;
 828
 829         $d->{template} = is_template($conf);
 830     }
 831
 832     foreach my $vmid (keys %$list) {
 833         my $d = $list->{$vmid};
 834         my $pid = $d->{pid};
 835
 836         next if !$pid; # skip stopped CTs
 837
 838         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 839         $d->{uptime} = time - $ctime; # the method lxcfs uses
 840
 841         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 842         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 843
 844         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 845         my @bytes = split(/\n/, $blkio_bytes);
 846         foreach my $byte (@bytes) {
 847             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 848                 $d->{diskread} = $2 if $key eq 'Read';
 849                 $d->{diskwrite} = $2 if $key eq 'Write';
 850             }
 851         }
 852
 853         my $pstat = &$parse_cpuacct_stat($vmid);
 854
 855         my $used = $pstat->{utime} + $pstat->{stime};
 856
 857         my $old = $last_proc_vmid_stat->{$vmid};
 858         if (!$old) {
 859             $last_proc_vmid_stat->{$vmid} = {
 860                 time => $cdtime,
 861                 used => $used,
 862                 cpu => 0,
 863             };
 864             next;
 865         }
 866
 867         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 868
 869         if ($dtime > 1000) {
 870             my $dutime = $used -  $old->{used};
 871
 872             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 873             $last_proc_vmid_stat->{$vmid} = {
 874                 time => $cdtime,
 875                 used => $used,
 876                 cpu => $d->{cpu},
 877             };
 878         } else {
 879             $d->{cpu} = $old->{cpu};
 880         }
 881     }
 882
 883     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 884
 885     foreach my $dev (keys %$netdev) {
 886         next if $dev !~ m/^veth([1-9]\d*)i/;
 887         my $vmid = $1;
 888         my $d = $list->{$vmid};
 889
 890         next if !$d;
 891
 892         $d->{netout} += $netdev->{$dev}->{receive};
 893         $d->{netin} += $netdev->{$dev}->{transmit};
 894
 895     }
 896
 897     return $list;
 898 }
 899
 900 sub classify_mountpoint {
 901     my ($vol) = @_;
 902     if ($vol =~ m!^/!) {
 903         return 'device' if $vol =~ m!^/dev/!;
 904         return 'bind';
 905     }
 906     return 'volume';
 907 }
 908
 909 my $parse_ct_mountpoint_full = sub {
 910     my ($desc, $data, $noerr) = @_;
 911
 912     $data //= '';
 913
 914     my $res;
 915     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 916     if ($@) {
 917         return undef if $noerr;
 918         die $@;
 919     }
 920
 921     if (defined(my $size = $res->{size})) {
 922         $size = PVE::JSONSchema::parse_size($size);
 923         if (!defined($size)) {
 924             return undef if $noerr;
 925             die "invalid size: $size\n";
 926         }
 927         $res->{size} = $size;
 928     }
 929
 930     $res->{type} = classify_mountpoint($res->{volume});
 931
 932     return $res;
 933 };
 934
 935 sub parse_ct_rootfs {
 936     my ($data, $noerr) = @_;
 937
 938     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 939
 940     $res->{mp} = '/' if defined($res);
 941
 942     return $res;
 943 }
 944
 945 sub parse_ct_mountpoint {
 946     my ($data, $noerr) = @_;
 947
 948     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 949 }
 950
 951 sub print_ct_mountpoint {
 952     my ($info, $nomp) = @_;
 953     my $skip = [ 'type' ];
 954     push @$skip, 'mp' if $nomp;
 955     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 956 }
 957
 958 sub print_lxc_network {
 959     my $net = shift;
 960     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 961 }
 962
 963 sub parse_lxc_network {
 964     my ($data) = @_;
 965
 966     my $res = {};
 967
 968     return $res if !$data;
 969
 970     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 971
 972     $res->{type} = 'veth';
 973     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 974
 975     return $res;
 976 }
 977
 978 sub read_cgroup_value {
 979     my ($group, $vmid, $name, $full) = @_;
 980
 981     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 982
 983     return PVE::Tools::file_get_contents($path) if $full;
 984
 985     return PVE::Tools::file_read_firstline($path);
 986 }
 987
 988 sub write_cgroup_value {
 989    my ($group, $vmid, $name, $value) = @_;
 990
 991    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 992    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 993
 994 }
 995
 996 sub find_lxc_console_pids {
 997
 998     my $res = {};
 999
1000     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
1001         my ($pid) = @_;
1002
1003         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
1004         return if !$cmdline;
1005
1006         my @args = split(/\0/, $cmdline);
1007
1008         # search for lxc-console -n <vmid>
1009         return if scalar(@args) != 3;
1010         return if $args[1] ne '-n';
1011         return if $args[2] !~ m/^\d+$/;
1012         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1013
1014         my $vmid = $args[2];
1015
1016         push @{$res->{$vmid}}, $pid;
1017     });
1018
1019     return $res;
1020 }
1021
1022 sub find_lxc_pid {
1023     my ($vmid) = @_;
1024
1025     my $pid = undef;
1026     my $parser = sub {
1027         my $line = shift;
1028         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1029     };
1030     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1031
1032     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1033
1034     return $pid;
1035 }
1036
1037 # Note: we cannot use Net:IP, because that only allows strict
1038 # CIDR networks
1039 sub parse_ipv4_cidr {
1040     my ($cidr, $noerr) = @_;
1041
1042     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1043         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1044     }
1045
1046     return undef if $noerr;
1047
1048     die "unable to parse ipv4 address/mask\n";
1049 }
1050
1051 sub check_lock {
1052     my ($conf) = @_;
1053
1054     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1055 }
1056
1057 sub has_lock {
1058     my ($conf, $lock) = @_;
1059     return $conf->{lock} && (!defined($lock) || $lock eq $conf->{lock});
1060 }
1061
1062 sub check_protection {
1063     my ($vm_conf, $err_msg) = @_;
1064
1065     if ($vm_conf->{protection}) {
1066         die "$err_msg - protection mode enabled\n";
1067     }
1068 }
1069
1070 sub update_lxc_config {
1071     my ($storage_cfg, $vmid, $conf) = @_;
1072
1073     my $dir = "/var/lib/lxc/$vmid";
1074
1075     if ($conf->{template}) {
1076
1077         unlink "$dir/config";
1078
1079         return;
1080     }
1081
1082     my $raw = '';
1083
1084     die "missing 'arch' - internal error" if !$conf->{arch};
1085     $raw .= "lxc.arch = $conf->{arch}\n";
1086
1087     my $unprivileged = $conf->{unprivileged};
1088     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1089
1090     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1091     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux | alpine | unmanaged)$/x) {
1092         my $inc ="/usr/share/lxc/config/$ostype.common.conf";
1093         $inc ="/usr/share/lxc/config/common.conf" if !-f $inc;
1094         $raw .= "lxc.include = $inc\n";
1095         if ($unprivileged || $custom_idmap) {
1096             $inc = "/usr/share/lxc/config/$ostype.userns.conf";
1097             $inc = "/usr/share/lxc/config/userns.conf" if !-f $inc;
1098             $raw .= "lxc.include = $inc\n"
1099         }
1100     } else {
1101         die "implement me (ostype $ostype)";
1102     }
1103
1104     # WARNING: DO NOT REMOVE this without making sure that loop device nodes
1105     # cannot be exposed to the container with r/w access (cgroup perms).
1106     # When this is enabled mounts will still remain in the monitor's namespace
1107     # after the container unmounted them and thus will not detach from their
1108     # files while the container is running!
1109     $raw .= "lxc.monitor.unshare = 1\n";
1110
1111     # Should we read them from /etc/subuid?
1112     if ($unprivileged && !$custom_idmap) {
1113         $raw .= "lxc.id_map = u 0 100000 65536\n";
1114         $raw .= "lxc.id_map = g 0 100000 65536\n";
1115     }
1116
1117     if (!has_dev_console($conf)) {
1118         $raw .= "lxc.console = none\n";
1119         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1120     }
1121
1122     my $ttycount = get_tty_count($conf);
1123     $raw .= "lxc.tty = $ttycount\n";
1124
1125     # some init scripts expect a linux terminal (turnkey).
1126     $raw .= "lxc.environment = TERM=linux\n";
1127
1128     my $utsname = $conf->{hostname} || "CT$vmid";
1129     $raw .= "lxc.utsname = $utsname\n";
1130
1131     my $memory = $conf->{memory} || 512;
1132     my $swap = $conf->{swap} // 0;
1133
1134     my $lxcmem = int($memory*1024*1024);
1135     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1136
1137     my $lxcswap = int(($memory + $swap)*1024*1024);
1138     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1139
1140     if (my $cpulimit = $conf->{cpulimit}) {
1141         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1142         my $value = int(100000*$cpulimit);
1143         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1144     }
1145
1146     my $shares = $conf->{cpuunits} || 1024;
1147     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1148
1149     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1150
1151     $raw .= "lxc.rootfs = $dir/rootfs\n";
1152
1153     my $netcount = 0;
1154     foreach my $k (keys %$conf) {
1155         next if $k !~ m/^net(\d+)$/;
1156         my $ind = $1;
1157         my $d = parse_lxc_network($conf->{$k});
1158         $netcount++;
1159         $raw .= "lxc.network.type = veth\n";
1160         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1161         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1162         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1163         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1164     }
1165
1166     if (my $lxcconf = $conf->{lxc}) {
1167         foreach my $entry (@$lxcconf) {
1168             my ($k, $v) = @$entry;
1169             $netcount++ if $k eq 'lxc.network.type';
1170             $raw .= "$k = $v\n";
1171         }
1172     }
1173
1174     $raw .= "lxc.network.type = empty\n" if !$netcount;
1175
1176     File::Path::mkpath("$dir/rootfs");
1177
1178     PVE::Tools::file_set_contents("$dir/config", $raw);
1179 }
1180
1181 # verify and cleanup nameserver list (replace \0 with ' ')
1182 sub verify_nameserver_list {
1183     my ($nameserver_list) = @_;
1184
1185     my @list = ();
1186     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1187         PVE::JSONSchema::pve_verify_ip($server);
1188         push @list, $server;
1189     }
1190
1191     return join(' ', @list);
1192 }
1193
1194 sub verify_searchdomain_list {
1195     my ($searchdomain_list) = @_;
1196
1197     my @list = ();
1198     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1199         # todo: should we add checks for valid dns domains?
1200         push @list, $server;
1201     }
1202
1203     return join(' ', @list);
1204 }
1205
1206 sub is_volume_in_use {
1207     my ($config, $volid, $include_snapshots) = @_;
1208     my $used = 0;
1209
1210     foreach_mountpoint($config, sub {
1211         my ($ms, $mountpoint) = @_;
1212         return if $used;
1213         if ($mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid) {
1214             $used = 1;
1215         }
1216     });
1217
1218     my $snapshots = $config->{snapshots};
1219     if ($include_snapshots && $snapshots) {
1220         foreach my $snap (keys %$snapshots) {
1221             $used ||= is_volume_in_use($snapshots->{$snap}, $volid);
1222         }
1223     }
1224
1225     return $used;
1226 }
1227
1228 sub add_unused_volume {
1229     my ($config, $volid) = @_;
1230
1231     my $key;
1232     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1233         my $test = "unused$ind";
1234         if (my $vid = $config->{$test}) {
1235             return if $vid eq $volid; # do not add duplicates
1236         } else {
1237             $key = $test;
1238         }
1239     }
1240
1241     die "Too many unused volumes - please delete them first.\n" if !$key;
1242
1243     $config->{$key} = $volid;
1244
1245     return $key;
1246 }
1247
1248 sub update_pct_config {
1249     my ($vmid, $conf, $running, $param, $delete) = @_;
1250
1251     my @nohotplug;
1252
1253     my $new_disks = 0;
1254     my @deleted_volumes;
1255
1256     my $rootdir;
1257     if ($running) {
1258         my $pid = find_lxc_pid($vmid);
1259         $rootdir = "/proc/$pid/root";
1260     }
1261
1262     my $hotplug_error = sub {
1263         if ($running) {
1264             push @nohotplug, @_;
1265             return 1;
1266         } else {
1267             return 0;
1268         }
1269     };
1270
1271     if (defined($delete)) {
1272         foreach my $opt (@$delete) {
1273             if (!exists($conf->{$opt})) {
1274                 warn "no such option: $opt\n";
1275                 next;
1276             }
1277
1278             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1279                 die "unable to delete required option '$opt'\n";
1280             } elsif ($opt eq 'swap') {
1281                 delete $conf->{$opt};
1282                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1283             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1284                 delete $conf->{$opt};
1285             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1286                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1287                 next if $hotplug_error->($opt);
1288                 delete $conf->{$opt};
1289             } elsif ($opt =~ m/^net(\d)$/) {
1290                 delete $conf->{$opt};
1291                 next if !$running;
1292                 my $netid = $1;
1293                 PVE::Network::veth_delete("veth${vmid}i$netid");
1294             } elsif ($opt eq 'protection') {
1295                 delete $conf->{$opt};
1296             } elsif ($opt =~ m/^unused(\d+)$/) {
1297                 next if $hotplug_error->($opt);
1298                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1299                 push @deleted_volumes, $conf->{$opt};
1300                 delete $conf->{$opt};
1301             } elsif ($opt =~ m/^mp(\d+)$/) {
1302                 next if $hotplug_error->($opt);
1303                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1304                 my $mp = parse_ct_mountpoint($conf->{$opt});
1305                 delete $conf->{$opt};
1306                 if ($mp->{type} eq 'volume' && !is_volume_in_use($conf, $mp->{volume})) {
1307                     add_unused_volume($conf, $mp->{volume});
1308                 }
1309             } elsif ($opt eq 'unprivileged') {
1310                 die "unable to delete read-only option: '$opt'\n";
1311             } else {
1312                 die "implement me (delete: $opt)"
1313             }
1314             write_config($vmid, $conf) if $running;
1315         }
1316     }
1317
1318     # There's no separate swap size to configure, there's memory and "total"
1319     # memory (iow. memory+swap). This means we have to change them together.
1320     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1321     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1322     if (defined($wanted_memory) || defined($wanted_swap)) {
1323
1324         my $old_memory = ($conf->{memory} || 512);
1325         my $old_swap = ($conf->{swap} || 0);
1326
1327         $wanted_memory //= $old_memory;
1328         $wanted_swap //= $old_swap;
1329
1330         my $total = $wanted_memory + $wanted_swap;
1331         if ($running) {
1332             my $old_total = $old_memory + $old_swap;
1333             if ($total > $old_total) {
1334                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1335                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1336             } else {
1337                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1338                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1339             }
1340         }
1341         $conf->{memory} = $wanted_memory;
1342         $conf->{swap} = $wanted_swap;
1343
1344         write_config($vmid, $conf) if $running;
1345     }
1346
1347     my $used_volids = {};
1348
1349     foreach my $opt (keys %$param) {
1350         my $value = $param->{$opt};
1351         if ($opt eq 'hostname') {
1352             $conf->{$opt} = $value;
1353         } elsif ($opt eq 'onboot') {
1354             $conf->{$opt} = $value ? 1 : 0;
1355         } elsif ($opt eq 'startup') {
1356             $conf->{$opt} = $value;
1357         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1358             next if $hotplug_error->($opt);
1359             $conf->{$opt} = $value;
1360         } elsif ($opt eq 'nameserver') {
1361             next if $hotplug_error->($opt);
1362             my $list = verify_nameserver_list($value);
1363             $conf->{$opt} = $list;
1364         } elsif ($opt eq 'searchdomain') {
1365             next if $hotplug_error->($opt);
1366             my $list = verify_searchdomain_list($value);
1367             $conf->{$opt} = $list;
1368         } elsif ($opt eq 'cpulimit') {
1369             next if $hotplug_error->($opt); # FIXME: hotplug
1370             $conf->{$opt} = $value;
1371         } elsif ($opt eq 'cpuunits') {
1372             $conf->{$opt} = $value;
1373             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1374         } elsif ($opt eq 'description') {
1375             $conf->{$opt} = PVE::Tools::encode_text($value);
1376         } elsif ($opt =~ m/^net(\d+)$/) {
1377             my $netid = $1;
1378             my $net = parse_lxc_network($value);
1379             if (!$running) {
1380                 $conf->{$opt} = print_lxc_network($net);
1381             } else {
1382                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1383             }
1384         } elsif ($opt eq 'protection') {
1385             $conf->{$opt} = $value ? 1 : 0;
1386         } elsif ($opt =~ m/^mp(\d+)$/) {
1387             next if $hotplug_error->($opt);
1388             check_protection($conf, "can't update CT $vmid drive '$opt'");
1389             my $old = $conf->{$opt};
1390             $conf->{$opt} = $value;
1391             if (defined($old)) {
1392                 my $mp = parse_ct_mountpoint($old);
1393                 if ($mp->{type} eq 'volume' && !is_volume_in_use($conf, $mp->{volume})) {
1394                     add_unused_volume($conf, $mp->{volume});
1395                 }
1396             }
1397             $new_disks = 1;
1398             my $mp = parse_ct_mountpoint($value);
1399             $used_volids->{$mp->{volume}} = 1;
1400         } elsif ($opt eq 'rootfs') {
1401             next if $hotplug_error->($opt);
1402             check_protection($conf, "can't update CT $vmid drive '$opt'");
1403             my $old = $conf->{$opt};
1404             $conf->{$opt} = $value;
1405             if (defined($old)) {
1406                 my $mp = parse_ct_rootfs($old);
1407                 if ($mp->{type} eq 'volume' && !is_volume_in_use($conf, $mp->{volume})) {
1408                     add_unused_volume($conf, $mp->{volume});
1409                 }
1410             }
1411             my $mp = parse_ct_rootfs($value);
1412             $used_volids->{$mp->{volume}} = 1;
1413         } elsif ($opt eq 'unprivileged') {
1414             die "unable to modify read-only option: '$opt'\n";
1415         } elsif ($opt eq 'ostype') {
1416             next if $hotplug_error->($opt);
1417             $conf->{$opt} = $value;
1418         } else {
1419             die "implement me: $opt";
1420         }
1421         write_config($vmid, $conf) if $running;
1422     }
1423
1424     # Cleanup config:
1425
1426     # Remove unused disks after re-adding
1427     foreach my $key (keys %$conf) {
1428         next if $key !~ /^unused\d+/;
1429         my $volid = $conf->{$key};
1430         if ($used_volids->{$volid}) {
1431             delete $conf->{$key};
1432         }
1433     }
1434
1435     # Apply deletions and creations of new volumes
1436     if (@deleted_volumes) {
1437         my $storage_cfg = PVE::Storage::config();
1438         foreach my $volume (@deleted_volumes) {
1439             next if $used_volids->{$volume}; # could have been re-added, too
1440             # also check for references in snapshots
1441             next if is_volume_in_use($conf, $volume, 1);
1442             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1443         }
1444     }
1445
1446     if ($new_disks) {
1447         my $storage_cfg = PVE::Storage::config();
1448         create_disks($storage_cfg, $vmid, $conf, $conf);
1449     }
1450
1451     # This should be the last thing we do here
1452     if ($running && scalar(@nohotplug)) {
1453         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1454     }
1455 }
1456
1457 sub has_dev_console {
1458     my ($conf) = @_;
1459
1460     return !(defined($conf->{console}) && !$conf->{console});
1461 }
1462
1463 sub get_tty_count {
1464     my ($conf) = @_;
1465
1466     return $conf->{tty} // $confdesc->{tty}->{default};
1467 }
1468
1469 sub get_cmode {
1470     my ($conf) = @_;
1471
1472     return $conf->{cmode} // $confdesc->{cmode}->{default};
1473 }
1474
1475 sub get_console_command {
1476     my ($vmid, $conf) = @_;
1477
1478     my $cmode = get_cmode($conf);
1479
1480     if ($cmode eq 'console') {
1481         return ['lxc-console', '-n',  $vmid, '-t', 0];
1482     } elsif ($cmode eq 'tty') {
1483         return ['lxc-console', '-n',  $vmid];
1484     } elsif ($cmode eq 'shell') {
1485         return ['lxc-attach', '--clear-env', '-n', $vmid];
1486     } else {
1487         die "internal error";
1488     }
1489 }
1490
1491 sub get_primary_ips {
1492     my ($conf) = @_;
1493
1494     # return data from net0
1495
1496     return undef if !defined($conf->{net0});
1497     my $net = parse_lxc_network($conf->{net0});
1498
1499     my $ipv4 = $net->{ip};
1500     if ($ipv4) {
1501         if ($ipv4 =~ /^(dhcp|manual)$/) {
1502             $ipv4 = undef
1503         } else {
1504             $ipv4 =~ s!/\d+$!!;
1505         }
1506     }
1507     my $ipv6 = $net->{ip6};
1508     if ($ipv6) {
1509         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1510             $ipv6 = undef;
1511         } else {
1512             $ipv6 =~ s!/\d+$!!;
1513         }
1514     }
1515
1516     return ($ipv4, $ipv6);
1517 }
1518
1519 sub delete_mountpoint_volume {
1520     my ($storage_cfg, $vmid, $volume) = @_;
1521
1522     return if classify_mountpoint($volume) ne 'volume';
1523
1524     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1525     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1526 }
1527
1528 sub destroy_lxc_container {
1529     my ($storage_cfg, $vmid, $conf) = @_;
1530
1531     foreach_mountpoint($conf, sub {
1532         my ($ms, $mountpoint) = @_;
1533         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1534     });
1535
1536     rmdir "/var/lib/lxc/$vmid/rootfs";
1537     unlink "/var/lib/lxc/$vmid/config";
1538     rmdir "/var/lib/lxc/$vmid";
1539     destroy_config($vmid);
1540
1541     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1542     #PVE::Tools::run_command($cmd);
1543 }
1544
1545 sub vm_stop_cleanup {
1546     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1547
1548     eval {
1549         if (!$keepActive) {
1550
1551             my $vollist = get_vm_volumes($conf);
1552             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1553         }
1554     };
1555     warn $@ if $@; # avoid errors - just warn
1556 }
1557
1558 my $safe_num_ne = sub {
1559     my ($a, $b) = @_;
1560
1561     return 0 if !defined($a) && !defined($b);
1562     return 1 if !defined($a);
1563     return 1 if !defined($b);
1564
1565     return $a != $b;
1566 };
1567
1568 my $safe_string_ne = sub {
1569     my ($a, $b) = @_;
1570
1571     return 0 if !defined($a) && !defined($b);
1572     return 1 if !defined($a);
1573     return 1 if !defined($b);
1574
1575     return $a ne $b;
1576 };
1577
1578 sub update_net {
1579     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1580
1581     if ($newnet->{type} ne 'veth') {
1582         # for when there are physical interfaces
1583         die "cannot update interface of type $newnet->{type}";
1584     }
1585
1586     my $veth = "veth${vmid}i${netid}";
1587     my $eth = $newnet->{name};
1588
1589     if (my $oldnetcfg = $conf->{$opt}) {
1590         my $oldnet = parse_lxc_network($oldnetcfg);
1591
1592         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1593             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1594
1595             PVE::Network::veth_delete($veth);
1596             delete $conf->{$opt};
1597             write_config($vmid, $conf);
1598
1599             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1600
1601         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1602                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1603                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1604
1605                 if ($oldnet->{bridge}) {
1606                     PVE::Network::tap_unplug($veth);
1607                     foreach (qw(bridge tag firewall)) {
1608                         delete $oldnet->{$_};
1609                     }
1610                     $conf->{$opt} = print_lxc_network($oldnet);
1611                     write_config($vmid, $conf);
1612                 }
1613
1614                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1615                 foreach (qw(bridge tag firewall)) {
1616                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1617                 }
1618                 $conf->{$opt} = print_lxc_network($oldnet);
1619                 write_config($vmid, $conf);
1620         }
1621     } else {
1622         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1623     }
1624
1625     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1626 }
1627
1628 sub hotplug_net {
1629     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1630
1631     my $veth = "veth${vmid}i${netid}";
1632     my $vethpeer = $veth . "p";
1633     my $eth = $newnet->{name};
1634
1635     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1636     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1637
1638     # attach peer in container
1639     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1640     PVE::Tools::run_command($cmd);
1641
1642     # link up peer in container
1643     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1644     PVE::Tools::run_command($cmd);
1645
1646     my $done = { type => 'veth' };
1647     foreach (qw(bridge tag firewall hwaddr name)) {
1648         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1649     }
1650     $conf->{$opt} = print_lxc_network($done);
1651
1652     write_config($vmid, $conf);
1653 }
1654
1655 sub update_ipconfig {
1656     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1657
1658     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1659
1660     my $optdata = parse_lxc_network($conf->{$opt});
1661     my $deleted = [];
1662     my $added = [];
1663     my $nscmd = sub {
1664         my $cmdargs = shift;
1665         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1666     };
1667     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1668
1669     my $change_ip_config = sub {
1670         my ($ipversion) = @_;
1671
1672         my $family_opt = "-$ipversion";
1673         my $suffix = $ipversion == 4 ? '' : $ipversion;
1674         my $gw= "gw$suffix";
1675         my $ip= "ip$suffix";
1676
1677         my $newip = $newnet->{$ip};
1678         my $newgw = $newnet->{$gw};
1679         my $oldip = $optdata->{$ip};
1680
1681         my $change_ip = &$safe_string_ne($oldip, $newip);
1682         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1683
1684         return if !$change_ip && !$change_gw;
1685
1686         # step 1: add new IP, if this fails we cancel
1687         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1688         if ($change_ip && $is_real_ip) {
1689             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1690             if (my $err = $@) {
1691                 warn $err;
1692                 return;
1693             }
1694         }
1695
1696         # step 2: replace gateway
1697         #   If this fails we delete the added IP and cancel.
1698         #   If it succeeds we save the config and delete the old IP, ignoring
1699         #   errors. The config is then saved.
1700         # Note: 'ip route replace' can add
1701         if ($change_gw) {
1702             if ($newgw) {
1703                 eval {
1704                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1705                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1706                     }
1707                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1708                 };
1709                 if (my $err = $@) {
1710                     warn $err;
1711                     # the route was not replaced, the old IP is still available
1712                     # rollback (delete new IP) and cancel
1713                     if ($change_ip) {
1714                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1715                         warn $@ if $@; # no need to die here
1716                     }
1717                     return;
1718                 }
1719             } else {
1720                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1721                 # if the route was not deleted, the guest might have deleted it manually
1722                 # warn and continue
1723                 warn $@ if $@;
1724             }
1725         }
1726
1727         # from this point on we save the configuration
1728         # step 3: delete old IP ignoring errors
1729         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1730             # We need to enable promote_secondaries, otherwise our newly added
1731             # address will be removed along with the old one.
1732             my $promote = 0;
1733             eval {
1734                 if ($ipversion == 4) {
1735                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1736                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1737                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1738                 }
1739                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1740             };
1741             warn $@ if $@; # no need to die here
1742
1743             if ($ipversion == 4) {
1744                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1745             }
1746         }
1747
1748         foreach my $property ($ip, $gw) {
1749             if ($newnet->{$property}) {
1750                 $optdata->{$property} = $newnet->{$property};
1751             } else {
1752                 delete $optdata->{$property};
1753             }
1754         }
1755         $conf->{$opt} = print_lxc_network($optdata);
1756         write_config($vmid, $conf);
1757         $lxc_setup->setup_network($conf);
1758     };
1759
1760     &$change_ip_config(4);
1761     &$change_ip_config(6);
1762
1763 }
1764
1765 # Internal snapshots
1766
1767 # NOTE: Snapshot create/delete involves several non-atomic
1768 # actions, and can take a long time.
1769 # So we try to avoid locking the file and use the 'lock' variable
1770 # inside the config file instead.
1771
1772 my $snapshot_copy_config = sub {
1773     my ($source, $dest) = @_;
1774
1775     foreach my $k (keys %$source) {
1776         next if $k eq 'snapshots';
1777         next if $k eq 'snapstate';
1778         next if $k eq 'snaptime';
1779         next if $k eq 'vmstate';
1780         next if $k eq 'lock';
1781         next if $k eq 'digest';
1782         next if $k eq 'description';
1783         next if $k =~ m/^unused\d+$/;
1784
1785         $dest->{$k} = $source->{$k};
1786     }
1787 };
1788
1789 my $snapshot_apply_config = sub {
1790     my ($conf, $snap) = @_;
1791
1792     # copy snapshot list
1793     my $newconf = {
1794         snapshots => $conf->{snapshots},
1795     };
1796
1797     # keep description and list of unused disks
1798     foreach my $k (keys %$conf) {
1799         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
1800         $newconf->{$k} = $conf->{$k};
1801     }
1802
1803     &$snapshot_copy_config($snap, $newconf);
1804
1805     return $newconf;
1806 };
1807
1808 sub snapshot_save_vmstate {
1809     die "implement me - snapshot_save_vmstate\n";
1810 }
1811
1812 sub snapshot_prepare {
1813     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1814
1815     my $snap;
1816
1817     my $updatefn =  sub {
1818
1819         my $conf = load_config($vmid);
1820
1821         die "you can't take a snapshot if it's a template\n"
1822             if is_template($conf);
1823
1824         check_lock($conf);
1825
1826         $conf->{lock} = 'snapshot';
1827
1828         die "snapshot name '$snapname' already used\n"
1829             if defined($conf->{snapshots}->{$snapname});
1830
1831         my $storecfg = PVE::Storage::config();
1832         die "snapshot feature is not available\n"
1833             if !has_feature('snapshot', $conf, $storecfg, undef, undef, $snapname eq 'vzdump');
1834
1835         $snap = $conf->{snapshots}->{$snapname} = {};
1836
1837         if ($save_vmstate && check_running($vmid)) {
1838             snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
1839         }
1840
1841         &$snapshot_copy_config($conf, $snap);
1842
1843         $snap->{snapstate} = "prepare";
1844         $snap->{snaptime} = time();
1845         $snap->{description} = $comment if $comment;
1846
1847         write_config($vmid, $conf);
1848     };
1849
1850     lock_config($vmid, $updatefn);
1851
1852     return $snap;
1853 }
1854
1855 sub snapshot_commit {
1856     my ($vmid, $snapname) = @_;
1857
1858     my $updatefn = sub {
1859
1860         my $conf = load_config($vmid);
1861
1862         die "missing snapshot lock\n"
1863             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1864
1865         my $snap = $conf->{snapshots}->{$snapname};
1866         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1867
1868         die "wrong snapshot state\n"
1869             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
1870
1871         delete $snap->{snapstate};
1872         delete $conf->{lock};
1873
1874         my $newconf = &$snapshot_apply_config($conf, $snap);
1875
1876         $newconf->{parent} = $snapname;
1877
1878         write_config($vmid, $newconf);
1879     };
1880
1881     lock_config($vmid, $updatefn);
1882 }
1883
1884 sub has_feature {
1885     my ($feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
1886
1887     my $err;
1888
1889     foreach_mountpoint($conf, sub {
1890         my ($ms, $mountpoint) = @_;
1891
1892         return if $err; # skip further test
1893         return if $backup_only && $ms ne 'rootfs' && !$mountpoint->{backup};
1894
1895         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname, $running);
1896     });
1897
1898     return $err ? 0 : 1;
1899 }
1900
1901 my $enter_namespace = sub {
1902     my ($vmid, $pid, $which, $type) = @_;
1903     sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY
1904         or die "failed to open $which namespace of container $vmid: $!\n";
1905     PVE::Tools::setns(fileno($fd), $type)
1906         or die "failed to enter $which namespace of container $vmid: $!\n";
1907     close $fd;
1908 };
1909
1910 my $do_syncfs = sub {
1911     my ($vmid, $pid, $socket) = @_;
1912
1913     &$enter_namespace($vmid, $pid, 'mnt', PVE::Tools::CLONE_NEWNS);
1914
1915     # Tell the parent process to start reading our /proc/mounts
1916     print {$socket} "go\n";
1917     $socket->flush();
1918
1919     # Receive /proc/self/mounts
1920     my $mountdata = do { local $/ = undef; <$socket> };
1921     close $socket;
1922
1923     # Now sync all mountpoints...
1924     my $mounts = PVE::ProcFSTools::parse_mounts($mountdata);
1925     foreach my $mp (@$mounts) {
1926         my ($what, $dir, $fs) = @$mp;
1927         next if $fs eq 'fuse.lxcfs';
1928         eval { PVE::Tools::sync_mountpoint($dir); };
1929         warn $@ if $@;
1930     }
1931 };
1932
1933 sub sync_container_namespace {
1934     my ($vmid) = @_;
1935     my $pid = find_lxc_pid($vmid);
1936
1937     # SOCK_DGRAM is nicer for barriers but cannot be slurped
1938     socketpair my $pfd, my $cfd, AF_UNIX, SOCK_STREAM, PF_UNSPEC
1939         or die "failed to create socketpair: $!\n";
1940
1941     my $child = fork();
1942     die "fork failed: $!\n" if !defined($child);
1943
1944     if (!$child) {
1945         eval {
1946             close $pfd;
1947             &$do_syncfs($vmid, $pid, $cfd);
1948         };
1949         if (my $err = $@) {
1950             warn $err;
1951             POSIX::_exit(1);
1952         }
1953         POSIX::_exit(0);
1954     }
1955     close $cfd;
1956     my $go = <$pfd>;
1957     die "failed to enter container namespace\n" if $go ne "go\n";
1958
1959     open my $mounts, '<', "/proc/$child/mounts"
1960         or die "failed to open container's /proc/mounts: $!\n";
1961     my $mountdata = do { local $/ = undef; <$mounts> };
1962     close $mounts;
1963     print {$pfd} $mountdata;
1964     close $pfd;
1965
1966     while (waitpid($child, 0) != $child) {}
1967     die "failed to sync container namespace\n" if $? != 0;
1968 }
1969
1970 sub check_freeze_needed {
1971     my ($vmid, $config, $save_vmstate) = @_;
1972
1973     my $ret = check_running($vmid);
1974     return ($ret, $ret);
1975 }
1976
1977 sub snapshot_create {
1978     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1979
1980     my $snap = snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
1981
1982     $save_vmstate = 0 if !$snap->{vmstate};
1983
1984     my $conf = load_config($vmid);
1985
1986     my ($running, $freezefs) = check_freeze_needed($vmid, $conf, $snap->{vmstate});
1987
1988     my $drivehash = {};
1989
1990     eval {
1991         if ($freezefs) {
1992             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1993             sync_container_namespace($vmid);
1994         }
1995
1996         my $storecfg = PVE::Storage::config();
1997         foreach_mountpoint($conf, sub {
1998             my ($ms, $mountpoint) = @_;
1999
2000             return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
2001             PVE::Storage::volume_snapshot($storecfg, $mountpoint->{volume}, $snapname);
2002             $drivehash->{$ms} = 1;
2003         });
2004     };
2005     my $err = $@;
2006
2007     if ($running) {
2008         if ($freezefs) {
2009             eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
2010             warn $@ if $@;
2011         }
2012     }
2013
2014     if ($err) {
2015         warn "snapshot create failed: starting cleanup\n";
2016         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
2017         warn "$@" if $@;
2018         die "$err\n";
2019     }
2020
2021     snapshot_commit($vmid, $snapname);
2022 }
2023
2024 # Note: $drivehash is only set when called from snapshot_create.
2025 sub snapshot_delete {
2026     my ($vmid, $snapname, $force, $drivehash) = @_;
2027
2028     my $prepare = 1;
2029
2030     my $snap;
2031     my $unused = [];
2032
2033     my $unlink_parent = sub {
2034         my ($confref, $new_parent) = @_;
2035
2036         if ($confref->{parent} && $confref->{parent} eq $snapname) {
2037             if ($new_parent) {
2038                 $confref->{parent} = $new_parent;
2039             } else {
2040                 delete $confref->{parent};
2041             }
2042         }
2043     };
2044
2045     my $updatefn =  sub {
2046         my ($remove_drive) = @_;
2047
2048         my $conf = load_config($vmid);
2049
2050         if (!$drivehash) {
2051             check_lock($conf);
2052             die "you can't delete a snapshot if vm is a template\n"
2053                 if is_template($conf);
2054         }
2055
2056         $snap = $conf->{snapshots}->{$snapname};
2057
2058         die "snapshot '$snapname' does not exist\n" if !defined($snap);
2059
2060         # remove parent refs
2061         if (!$prepare) {
2062             &$unlink_parent($conf, $snap->{parent});
2063             foreach my $sn (keys %{$conf->{snapshots}}) {
2064                 next if $sn eq $snapname;
2065                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
2066             }
2067         }
2068
2069         if ($remove_drive) {
2070             if ($remove_drive eq 'vmstate') {
2071                 die "implement me - saving vmstate\n";
2072             } else {
2073                 my $value = $snap->{$remove_drive};
2074                 my $mountpoint = $remove_drive eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2075                 delete $snap->{$remove_drive};
2076                 add_unused_volume($snap, $mountpoint->{volume})
2077                     if (!is_volume_in_use($snap, $mountpoint->{volume}));
2078             }
2079         }
2080
2081         if ($prepare) {
2082             $snap->{snapstate} = 'delete';
2083         } else {
2084             delete $conf->{snapshots}->{$snapname};
2085             delete $conf->{lock} if $drivehash;
2086             foreach my $volid (@$unused) {
2087                 add_unused_volume($conf, $volid)
2088                     if (!is_volume_in_use($conf, $volid));
2089             }
2090         }
2091
2092         write_config($vmid, $conf);
2093     };
2094
2095     lock_config($vmid, $updatefn);
2096
2097     # now remove vmstate file
2098     # never set for LXC!
2099     my $storecfg = PVE::Storage::config();
2100
2101     if ($snap->{vmstate}) {
2102         die "implement me - saving vmstate\n";
2103     };
2104
2105     # now remove all volume snapshots
2106     foreach_mountpoint($snap, sub {
2107         my ($ms, $mountpoint) = @_;
2108
2109         return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
2110         if (!$drivehash || $drivehash->{$ms}) {
2111             eval { PVE::Storage::volume_snapshot_delete($storecfg, $mountpoint->{volume}, $snapname); };
2112             if (my $err = $@) {
2113                 die $err if !$force;
2114                 warn $err;
2115             }
2116         }
2117
2118         # save changes (remove mp from snapshot)
2119         lock_config($vmid, $updatefn, $ms) if !$force;
2120         push @$unused, $mountpoint->{volume};
2121     });
2122
2123     # now cleanup config
2124     $prepare = 0;
2125     lock_config($vmid, $updatefn);
2126 }
2127
2128 sub snapshot_rollback {
2129     my ($vmid, $snapname) = @_;
2130
2131     my $prepare = 1;
2132
2133     my $storecfg = PVE::Storage::config();
2134
2135     my $conf = load_config($vmid);
2136
2137     my $get_snapshot_config = sub {
2138
2139         die "you can't rollback if vm is a template\n" if is_template($conf);
2140
2141         my $res = $conf->{snapshots}->{$snapname};
2142
2143         die "snapshot '$snapname' does not exist\n" if !defined($res);
2144
2145         return $res;
2146     };
2147
2148     my $snap = &$get_snapshot_config();
2149
2150     foreach_mountpoint($snap, sub {
2151         my ($ms, $mountpoint) = @_;
2152
2153         PVE::Storage::volume_rollback_is_possible($storecfg, $mountpoint->{volume}, $snapname);
2154     });
2155
2156     my $updatefn = sub {
2157
2158         $conf = load_config($vmid);
2159
2160         $snap = &$get_snapshot_config();
2161
2162         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
2163             if $snap->{snapstate};
2164
2165         if ($prepare) {
2166             check_lock($conf);
2167             PVE::Tools::run_command(['/usr/bin/lxc-stop', '-n', $vmid, '--kill'])
2168                 if check_running($vmid);
2169         }
2170
2171         die "unable to rollback vm $vmid: vm is running\n"
2172             if check_running($vmid);
2173
2174         if ($prepare) {
2175             $conf->{lock} = 'rollback';
2176         } else {
2177             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
2178             delete $conf->{lock};
2179         }
2180
2181         my $forcemachine;
2182
2183         if (!$prepare) {
2184             # copy snapshot config to current config
2185             $conf = &$snapshot_apply_config($conf, $snap);
2186             $conf->{parent} = $snapname;
2187         }
2188
2189         write_config($vmid, $conf);
2190
2191         if (!$prepare && $snap->{vmstate}) {
2192             die "implement me - save vmstate\n";
2193         }
2194     };
2195
2196     lock_config($vmid, $updatefn);
2197
2198     foreach_mountpoint($snap, sub {
2199         my ($ms, $mountpoint) = @_;
2200
2201         PVE::Storage::volume_snapshot_rollback($storecfg, $mountpoint->{volume}, $snapname);
2202     });
2203
2204     $prepare = 0;
2205     lock_config($vmid, $updatefn);
2206 }
2207
2208 sub template_create {
2209     my ($vmid, $conf) = @_;
2210
2211     my $storecfg = PVE::Storage::config();
2212
2213     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
2214     my $volid = $rootinfo->{volume};
2215
2216     die "Template feature is not available for '$volid'\n"
2217         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
2218
2219     PVE::Storage::activate_volumes($storecfg, [$volid]);
2220
2221     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
2222     $rootinfo->{volume} = $template_volid;
2223     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
2224
2225     write_config($vmid, $conf);
2226 }
2227
2228 sub is_template {
2229     my ($conf) = @_;
2230
2231     return 1 if defined $conf->{template} && $conf->{template} == 1;
2232 }
2233
2234 sub mountpoint_names {
2235     my ($reverse) = @_;
2236
2237     my @names = ('rootfs');
2238
2239     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
2240         push @names, "mp$i";
2241     }
2242
2243     return $reverse ? reverse @names : @names;
2244 }
2245
2246
2247 sub foreach_mountpoint_full {
2248     my ($conf, $reverse, $func) = @_;
2249
2250     foreach my $key (mountpoint_names($reverse)) {
2251         my $value = $conf->{$key};
2252         next if !defined($value);
2253         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2254         next if !defined($mountpoint);
2255
2256         &$func($key, $mountpoint);
2257     }
2258 }
2259
2260 sub foreach_mountpoint {
2261     my ($conf, $func) = @_;
2262
2263     foreach_mountpoint_full($conf, 0, $func);
2264 }
2265
2266 sub foreach_mountpoint_reverse {
2267     my ($conf, $func) = @_;
2268
2269     foreach_mountpoint_full($conf, 1, $func);
2270 }
2271
2272 sub check_ct_modify_config_perm {
2273     my ($rpcenv, $authuser, $vmid, $pool, $newconf, $delete) = @_;
2274
2275     return 1 if $authuser eq 'root@pam';
2276
2277     my $check = sub {
2278         my ($opt, $delete) = @_;
2279         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2280             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2281         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2282             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2283             return if $delete;
2284             my $data = $opt eq 'rootfs' ? parse_ct_rootfs($newconf->{$opt})
2285                                         : parse_ct_mountpoint($newconf->{$opt});
2286             raise_perm_exc("mountpoint type $data->{type}") if $data->{type} ne 'volume';
2287         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2288             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2289         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2290                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2291             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2292         } else {
2293             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2294         }
2295     };
2296
2297     foreach my $opt (keys %$newconf) {
2298         &$check($opt, 0);
2299     }
2300     foreach my $opt (@$delete) {
2301         &$check($opt, 1);
2302     }
2303
2304     return 1;
2305 }
2306
2307 sub umount_all {
2308     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2309
2310     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2311     my $volid_list = get_vm_volumes($conf);
2312
2313     foreach_mountpoint_reverse($conf, sub {
2314         my ($ms, $mountpoint) = @_;
2315
2316         my $volid = $mountpoint->{volume};
2317         my $mount = $mountpoint->{mp};
2318
2319         return if !$volid || !$mount;
2320
2321         my $mount_path = "$rootdir/$mount";
2322         $mount_path =~ s!/+!/!g;
2323
2324         return if !PVE::ProcFSTools::is_mounted($mount_path);
2325
2326         eval {
2327             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2328         };
2329         if (my $err = $@) {
2330             if ($noerr) {
2331                 warn $err;
2332             } else {
2333                 die $err;
2334             }
2335         }
2336     });
2337 }
2338
2339 sub mount_all {
2340     my ($vmid, $storage_cfg, $conf) = @_;
2341
2342     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2343     File::Path::make_path($rootdir);
2344
2345     my $volid_list = get_vm_volumes($conf);
2346     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2347
2348     eval {
2349         foreach_mountpoint($conf, sub {
2350             my ($ms, $mountpoint) = @_;
2351
2352             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2353         });
2354     };
2355     if (my $err = $@) {
2356         warn "mounting container failed\n";
2357         umount_all($vmid, $storage_cfg, $conf, 1);
2358         die $err;
2359     }
2360
2361     return $rootdir;
2362 }
2363
2364
2365 sub mountpoint_mount_path {
2366     my ($mountpoint, $storage_cfg, $snapname) = @_;
2367
2368     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2369 }
2370
2371 my $check_mount_path = sub {
2372     my ($path) = @_;
2373     $path = File::Spec->canonpath($path);
2374     my $real = Cwd::realpath($path);
2375     if ($real ne $path) {
2376         die "mount path modified by symlink: $path != $real";
2377     }
2378 };
2379
2380 sub query_loopdev {
2381     my ($path) = @_;
2382     my $found;
2383     my $parser = sub {
2384         my $line = shift;
2385         if ($line =~ m@^(/dev/loop\d+):@) {
2386             $found = $1;
2387         }
2388     };
2389     my $cmd = ['losetup', '--associated', $path];
2390     PVE::Tools::run_command($cmd, outfunc => $parser);
2391     return $found;
2392 }
2393
2394 # Run a function with a file attached to a loop device.
2395 # The loop device is always detached afterwards (or set to autoclear).
2396 # Returns the loop device.
2397 sub run_with_loopdev {
2398     my ($func, $file) = @_;
2399     my $device = query_loopdev($file);
2400     # Try to reuse an existing device
2401     if ($device) {
2402         # We assume that whoever setup the loop device is responsible for
2403         # detaching it.
2404         &$func($device);
2405         return $device;
2406     }
2407
2408     my $parser = sub {
2409         my $line = shift;
2410         if ($line =~ m@^(/dev/loop\d+)$@) {
2411             $device = $1;
2412         }
2413     };
2414     PVE::Tools::run_command(['losetup', '--show', '-f', $file], outfunc => $parser);
2415     die "failed to setup loop device for $file\n" if !$device;
2416     eval { &$func($device); };
2417     my $err = $@;
2418     PVE::Tools::run_command(['losetup', '-d', $device]);
2419     die $err if $err;
2420     return $device;
2421 }
2422
2423 sub bindmount {
2424     my ($dir, $dest, $ro, @extra_opts) = @_;
2425     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $dir, $dest]);
2426     if ($ro) {
2427         eval { PVE::Tools::run_command(['mount', '-o', 'bind,remount,ro', $dest]); };
2428         if (my $err = $@) {
2429             warn "bindmount error\n";
2430             # don't leave writable bind-mounts behind...
2431             PVE::Tools::run_command(['umount', $dest]);
2432             die $err;
2433         }
2434     }
2435 }
2436
2437 # use $rootdir = undef to just return the corresponding mount path
2438 sub mountpoint_mount {
2439     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2440
2441     my $volid = $mountpoint->{volume};
2442     my $mount = $mountpoint->{mp};
2443     my $type = $mountpoint->{type};
2444     my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota};
2445     my $mounted_dev;
2446
2447     return if !$volid || !$mount;
2448
2449     my $mount_path;
2450
2451     if (defined($rootdir)) {
2452         $rootdir =~ s!/+$!!;
2453         $mount_path = "$rootdir/$mount";
2454         $mount_path =~ s!/+!/!g;
2455         &$check_mount_path($mount_path);
2456         File::Path::mkpath($mount_path);
2457     }
2458
2459     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2460
2461     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2462
2463     my $optstring = '';
2464     if (defined($mountpoint->{acl})) {
2465         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
2466     }
2467     my $readonly = $mountpoint->{ro};
2468
2469     my @extra_opts = ('-o', $optstring);
2470
2471     if ($storage) {
2472
2473         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2474         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2475
2476         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2477             PVE::Storage::parse_volname($storage_cfg, $volid);
2478
2479         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2480
2481         if ($format eq 'subvol') {
2482             if ($mount_path) {
2483                 if ($snapname) {
2484                     if ($scfg->{type} eq 'zfspool') {
2485                         my $path_arg = $path;
2486                         $path_arg =~ s!^/+!!;
2487                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
2488                     } else {
2489                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2490                     }
2491                 } else {
2492                     bindmount($path, $mount_path, $readonly, @extra_opts);
2493                     warn "cannot enable quota control for bind mounted subvolumes\n" if $quota;
2494                 }
2495             }
2496             return wantarray ? ($path, 0, $mounted_dev) : $path;
2497         } elsif ($format eq 'raw' || $format eq 'iso') {
2498             my $domount = sub {
2499                 my ($path) = @_;
2500                 if ($mount_path) {
2501                     if ($format eq 'iso') {
2502                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2503                     } elsif ($isBase || defined($snapname)) {
2504                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2505                     } else {
2506                         if ($quota) {
2507                             push @extra_opts, '-o', 'usrjquota=aquota.user,grpjquota=aquota.group,jqfmt=vfsv0';
2508                         }
2509                         push @extra_opts, '-o', 'ro' if $readonly;
2510                         PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2511                     }
2512                 }
2513             };
2514             my $use_loopdev = 0;
2515             if ($scfg->{path}) {
2516                 $mounted_dev = run_with_loopdev($domount, $path);
2517                 $use_loopdev = 1;
2518             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2519                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2520                 $mounted_dev = $path;
2521                 &$domount($path);
2522             } else {
2523                 die "unsupported storage type '$scfg->{type}'\n";
2524             }
2525             return wantarray ? ($path, $use_loopdev, $mounted_dev) : $path;
2526         } else {
2527             die "unsupported image format '$format'\n";
2528         }
2529     } elsif ($type eq 'device') {
2530                         push @extra_opts, '-o', 'ro' if $readonly;
2531         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
2532         return wantarray ? ($volid, 0, $volid) : $volid;
2533     } elsif ($type eq 'bind') {
2534         die "directory '$volid' does not exist\n" if ! -d $volid;
2535         &$check_mount_path($volid);
2536         bindmount($volid, $mount_path, $readonly, @extra_opts) if $mount_path;
2537         warn "cannot enable quota control for bind mounts\n" if $quota;
2538         return wantarray ? ($volid, 0, undef) : $volid;
2539     }
2540
2541     die "unsupported storage";
2542 }
2543
2544 sub get_vm_volumes {
2545     my ($conf, $excludes) = @_;
2546
2547     my $vollist = [];
2548
2549     foreach_mountpoint($conf, sub {
2550         my ($ms, $mountpoint) = @_;
2551
2552         return if $excludes && $ms eq $excludes;
2553
2554         my $volid = $mountpoint->{volume};
2555
2556         return if !$volid || $mountpoint->{type} ne 'volume';
2557
2558         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2559         return if !$sid;
2560
2561         push @$vollist, $volid;
2562     });
2563
2564     return $vollist;
2565 }
2566
2567 sub mkfs {
2568     my ($dev, $rootuid, $rootgid) = @_;
2569
2570     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2571                              '-E', "root_owner=$rootuid:$rootgid",
2572                              $dev]);
2573 }
2574
2575 sub format_disk {
2576     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2577
2578     if ($volid =~ m!^/dev/.+!) {
2579         mkfs($volid);
2580         return;
2581     }
2582
2583     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2584
2585     die "cannot format volume '$volid' with no storage\n" if !$storage;
2586
2587     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2588
2589     my $path = PVE::Storage::path($storage_cfg, $volid);
2590
2591     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2592         PVE::Storage::parse_volname($storage_cfg, $volid);
2593
2594     die "cannot format volume '$volid' (format == $format)\n"
2595         if $format ne 'raw';
2596
2597     mkfs($path, $rootuid, $rootgid);
2598 }
2599
2600 sub destroy_disks {
2601     my ($storecfg, $vollist) = @_;
2602
2603     foreach my $volid (@$vollist) {
2604         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2605         warn $@ if $@;
2606     }
2607 }
2608
2609 sub create_disks {
2610     my ($storecfg, $vmid, $settings, $conf) = @_;
2611
2612     my $vollist = [];
2613
2614     eval {
2615         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2616         my $chown_vollist = [];
2617
2618         foreach_mountpoint($settings, sub {
2619             my ($ms, $mountpoint) = @_;
2620
2621             my $volid = $mountpoint->{volume};
2622             my $mp = $mountpoint->{mp};
2623
2624             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2625
2626             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2627                 my ($storeid, $size_gb) = ($1, $2);
2628
2629                 my $size_kb = int(${size_gb}*1024) * 1024;
2630
2631                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2632                 # fixme: use better naming ct-$vmid-disk-X.raw?
2633
2634                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2635                     if ($size_kb > 0) {
2636                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2637                                                            undef, $size_kb);
2638                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2639                     } else {
2640                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2641                                                            undef, 0);
2642                         push @$chown_vollist, $volid;
2643                     }
2644                 } elsif ($scfg->{type} eq 'zfspool') {
2645
2646                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2647                                                        undef, $size_kb);
2648                     push @$chown_vollist, $volid;
2649                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2650
2651                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2652                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2653
2654                 } elsif ($scfg->{type} eq 'rbd') {
2655
2656                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2657                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2658                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2659                 } else {
2660                     die "unable to create containers on storage type '$scfg->{type}'\n";
2661                 }
2662                 push @$vollist, $volid;
2663                 $mountpoint->{volume} = $volid;
2664                 $mountpoint->{size} = $size_kb * 1024;
2665                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2666             } else {
2667                 # use specified/existing volid/dir/device
2668                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2669             }
2670         });
2671
2672         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2673         foreach my $volid (@$chown_vollist) {
2674             my $path = PVE::Storage::path($storecfg, $volid, undef);
2675             chown($rootuid, $rootgid, $path);
2676         }
2677         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2678     };
2679     # free allocated images on error
2680     if (my $err = $@) {
2681         destroy_disks($storecfg, $vollist);
2682         die $err;
2683     }
2684     return $vollist;
2685 }
2686
2687 # bash completion helper
2688
2689 sub complete_os_templates {
2690     my ($cmdname, $pname, $cvalue) = @_;
2691
2692     my $cfg = PVE::Storage::config();
2693
2694     my $storeid;
2695
2696     if ($cvalue =~ m/^([^:]+):/) {
2697         $storeid = $1;
2698     }
2699
2700     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2701     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2702
2703     my $res = [];
2704     foreach my $id (keys %$data) {
2705         foreach my $item (@{$data->{$id}}) {
2706             push @$res, $item->{volid} if defined($item->{volid});
2707         }
2708     }
2709
2710     return $res;
2711 }
2712
2713 my $complete_ctid_full = sub {
2714     my ($running) = @_;
2715
2716     my $idlist = vmstatus();
2717
2718     my $active_hash = list_active_containers();
2719
2720     my $res = [];
2721
2722     foreach my $id (keys %$idlist) {
2723         my $d = $idlist->{$id};
2724         if (defined($running)) {
2725             next if $d->{template};
2726             next if $running && !$active_hash->{$id};
2727             next if !$running && $active_hash->{$id};
2728         }
2729         push @$res, $id;
2730
2731     }
2732     return $res;
2733 };
2734
2735 sub complete_ctid {
2736     return &$complete_ctid_full();
2737 }
2738
2739 sub complete_ctid_stopped {
2740     return &$complete_ctid_full(0);
2741 }
2742
2743 sub complete_ctid_running {
2744     return &$complete_ctid_full(1);
2745 }
2746
2747 sub parse_id_maps {
2748     my ($conf) = @_;
2749
2750     my $id_map = [];
2751     my $rootuid = 0;
2752     my $rootgid = 0;
2753
2754     my $lxc = $conf->{lxc};
2755     foreach my $entry (@$lxc) {
2756         my ($key, $value) = @$entry;
2757         next if $key ne 'lxc.id_map';
2758         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2759             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2760             push @$id_map, [$type, $ct, $host, $length];
2761             if ($ct == 0) {
2762                 $rootuid = $host if $type eq 'u';
2763                 $rootgid = $host if $type eq 'g';
2764             }
2765         } else {
2766             die "failed to parse id_map: $value\n";
2767         }
2768     }
2769
2770     if (!@$id_map && $conf->{unprivileged}) {
2771         # Should we read them from /etc/subuid?
2772         $id_map = [ ['u', '0', '100000', '65536'],
2773                     ['g', '0', '100000', '65536'] ];
2774         $rootuid = $rootgid = 100000;
2775     }
2776
2777     return ($id_map, $rootuid, $rootgid);
2778 }
2779
2780 sub userns_command {
2781     my ($id_map) = @_;
2782     if (@$id_map) {
2783         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2784     }
2785     return [];
2786 }
2787
2788 sub set_lock {
2789     my ($vmid, $lock) = @_;
2790     my $conf;
2791     lock_config($vmid, sub {
2792         $conf = load_config($vmid);
2793         check_lock($conf);
2794         $conf->{lock} = $lock;
2795         write_config($vmid, $conf);
2796     });
2797     return $conf;
2798 }
2799
2800 sub remove_lock {
2801     my ($vmid, $lock) = @_;
2802     lock_config($vmid, sub {
2803         my $conf = load_config($vmid);
2804         if (!$conf->{lock}) {
2805             die "no lock found trying to remove lock '$lock'\n";
2806         } elsif (defined($lock) && $conf->{lock} ne $lock) {
2807             die "found lock '$conf->{lock}' trying to remove lock '$lock'\n";
2808         }
2809         delete $conf->{lock};
2810         write_config($vmid, $conf);
2811     });
2812 }
2813
2814 1;