src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use Socket;
   8
   9 use File::Path;
  10 use File::Spec;
  11 use Cwd qw();
  12 use Fcntl qw(O_RDONLY);
  13
  14 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  15 use PVE::Exception qw(raise_perm_exc);
  16 use PVE::Storage;
  17 use PVE::SafeSyslog;
  18 use PVE::INotify;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full);
  21 use PVE::Network;
  22 use PVE::AccessControl;
  23 use PVE::ProcFSTools;
  24 use Time::HiRes qw (gettimeofday);
  25
  26 use Data::Dumper;
  27
  28 my $nodename = PVE::INotify::nodename();
  29
  30 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  31
  32 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  33                           '--xattrs',
  34                           '--xattrs-include=user.*',
  35                           '--xattrs-include=security.capability',
  36                           '--warning=no-xattr-write' ];
  37
  38 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  39
  40 my $rootfs_desc = {
  41     volume => {
  42         type => 'string',
  43         default_key => 1,
  44         format => 'pve-lxc-mp-string',
  45         format_description => 'volume',
  46         description => 'Volume, device or directory to mount into the container.',
  47     },
  48     backup => {
  49         type => 'boolean',
  50         format_description => '[1|0]',
  51         description => 'Whether to include the mountpoint in backups.',
  52         optional => 1,
  53     },
  54     size => {
  55         type => 'string',
  56         format => 'disk-size',
  57         format_description => 'DiskSize',
  58         description => 'Volume size (read only value).',
  59         optional => 1,
  60     },
  61     acl => {
  62         type => 'boolean',
  63         format_description => 'acl',
  64         description => 'Explicitly enable or disable ACL support.',
  65         optional => 1,
  66     },
  67     ro => {
  68         type => 'boolean',
  69         format_description => 'ro',
  70         description => 'Read-only mountpoint (not supported with bind mounts)',
  71         optional => 1,
  72     },
  73     quota => {
  74         type => 'boolean',
  75         format_description => '[0|1]',
  76         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
  77         optional => 1,
  78     },
  79 };
  80
  81 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  82     type => 'string', format => $rootfs_desc,
  83     description => "Use volume as container root.",
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  88     description => "The name of the snapshot.",
  89     type => 'string', format => 'pve-configid',
  90     maxLength => 40,
  91 });
  92
  93 my $confdesc = {
  94     lock => {
  95         optional => 1,
  96         type => 'string',
  97         description => "Lock/unlock the VM.",
  98         enum => [qw(migrate backup snapshot rollback)],
  99     },
 100     onboot => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Specifies whether a VM will be started during system bootup.",
 104         default => 0,
 105     },
 106     startup => get_standard_option('pve-startup-order'),
 107     template => {
 108         optional => 1,
 109         type => 'boolean',
 110         description => "Enable/disable Template.",
 111         default => 0,
 112     },
 113     arch => {
 114         optional => 1,
 115         type => 'string',
 116         enum => ['amd64', 'i386'],
 117         description => "OS architecture type.",
 118         default => 'amd64',
 119     },
 120     ostype => {
 121         optional => 1,
 122         type => 'string',
 123         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux', 'alpine', 'unmanaged'],
 124         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 125     },
 126     console => {
 127         optional => 1,
 128         type => 'boolean',
 129         description => "Attach a console device (/dev/console) to the container.",
 130         default => 1,
 131     },
 132     tty => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "Specify the number of tty available to the container",
 136         minimum => 0,
 137         maximum => 6,
 138         default => 2,
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1024,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     swap => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of SWAP for the VM in MB.",
 167         minimum => 0,
 168         default => 512,
 169     },
 170     hostname => {
 171         optional => 1,
 172         description => "Set a host name for the container.",
 173         type => 'string', format => 'dns-name',
 174         maxLength => 255,
 175     },
 176     description => {
 177         optional => 1,
 178         type => 'string',
 179         description => "Container description. Only used on the configuration web interface.",
 180     },
 181     searchdomain => {
 182         optional => 1,
 183         type => 'string', format => 'dns-name-list',
 184         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 185     },
 186     nameserver => {
 187         optional => 1,
 188         type => 'string', format => 'address-list',
 189         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 190     },
 191     rootfs => get_standard_option('pve-ct-rootfs'),
 192     parent => {
 193         optional => 1,
 194         type => 'string', format => 'pve-configid',
 195         maxLength => 40,
 196         description => "Parent snapshot name. This is used internally, and should not be modified.",
 197     },
 198     snaptime => {
 199         optional => 1,
 200         description => "Timestamp for snapshots.",
 201         type => 'integer',
 202         minimum => 0,
 203     },
 204     cmode => {
 205         optional => 1,
 206         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 207         type => 'string',
 208         enum => ['shell', 'console', 'tty'],
 209         default => 'tty',
 210     },
 211     protection => {
 212         optional => 1,
 213         type => 'boolean',
 214         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 215         default => 0,
 216     },
 217     unprivileged => {
 218         optional => 1,
 219         type => 'boolean',
 220         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 221         default => 0,
 222     },
 223 };
 224
 225 my $valid_lxc_conf_keys = {
 226     'lxc.include' => 1,
 227     'lxc.arch' => 1,
 228     'lxc.utsname' => 1,
 229     'lxc.haltsignal' => 1,
 230     'lxc.rebootsignal' => 1,
 231     'lxc.stopsignal' => 1,
 232     'lxc.init_cmd' => 1,
 233     'lxc.network.type' => 1,
 234     'lxc.network.flags' => 1,
 235     'lxc.network.link' => 1,
 236     'lxc.network.mtu' => 1,
 237     'lxc.network.name' => 1,
 238     'lxc.network.hwaddr' => 1,
 239     'lxc.network.ipv4' => 1,
 240     'lxc.network.ipv4.gateway' => 1,
 241     'lxc.network.ipv6' => 1,
 242     'lxc.network.ipv6.gateway' => 1,
 243     'lxc.network.script.up' => 1,
 244     'lxc.network.script.down' => 1,
 245     'lxc.pts' => 1,
 246     'lxc.console.logfile' => 1,
 247     'lxc.console' => 1,
 248     'lxc.tty' => 1,
 249     'lxc.devttydir' => 1,
 250     'lxc.hook.autodev' => 1,
 251     'lxc.autodev' => 1,
 252     'lxc.kmsg' => 1,
 253     'lxc.mount' => 1,
 254     'lxc.mount.entry' => 1,
 255     'lxc.mount.auto' => 1,
 256     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 257     'lxc.rootfs.mount' => 1,
 258     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 259                             ', please use mountpoint options in the "rootfs" key',
 260     # lxc.cgroup.*
 261     'lxc.cap.drop' => 1,
 262     'lxc.cap.keep' => 1,
 263     'lxc.aa_profile' => 1,
 264     'lxc.aa_allow_incomplete' => 1,
 265     'lxc.se_context' => 1,
 266     'lxc.seccomp' => 1,
 267     'lxc.id_map' => 1,
 268     'lxc.hook.pre-start' => 1,
 269     'lxc.hook.pre-mount' => 1,
 270     'lxc.hook.mount' => 1,
 271     'lxc.hook.start' => 1,
 272     'lxc.hook.stop' => 1,
 273     'lxc.hook.post-stop' => 1,
 274     'lxc.hook.clone' => 1,
 275     'lxc.hook.destroy' => 1,
 276     'lxc.loglevel' => 1,
 277     'lxc.logfile' => 1,
 278     'lxc.start.auto' => 1,
 279     'lxc.start.delay' => 1,
 280     'lxc.start.order' => 1,
 281     'lxc.group' => 1,
 282     'lxc.environment' => 1,
 283 };
 284
 285 my $netconf_desc = {
 286     type => {
 287         type => 'string',
 288         optional => 1,
 289         description => "Network interface type.",
 290         enum => [qw(veth)],
 291     },
 292     name => {
 293         type => 'string',
 294         format_description => 'String',
 295         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 296         pattern => '[-_.\w\d]+',
 297     },
 298     bridge => {
 299         type => 'string',
 300         format_description => 'vmbr<Number>',
 301         description => 'Bridge to attach the network device to.',
 302         pattern => '[-_.\w\d]+',
 303         optional => 1,
 304     },
 305     hwaddr => {
 306         type => 'string',
 307         format_description => 'MAC',
 308         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 309         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 310         optional => 1,
 311     },
 312     mtu => {
 313         type => 'integer',
 314         format_description => 'Number',
 315         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 316         minimum => 64, # minimum ethernet frame is 64 bytes
 317         optional => 1,
 318     },
 319     ip => {
 320         type => 'string',
 321         format => 'pve-ipv4-config',
 322         format_description => 'IPv4Format/CIDR',
 323         description => 'IPv4 address in CIDR format.',
 324         optional => 1,
 325     },
 326     gw => {
 327         type => 'string',
 328         format => 'ipv4',
 329         format_description => 'GatewayIPv4',
 330         description => 'Default gateway for IPv4 traffic.',
 331         optional => 1,
 332     },
 333     ip6 => {
 334         type => 'string',
 335         format => 'pve-ipv6-config',
 336         format_description => 'IPv6Format/CIDR',
 337         description => 'IPv6 address in CIDR format.',
 338         optional => 1,
 339     },
 340     gw6 => {
 341         type => 'string',
 342         format => 'ipv6',
 343         format_description => 'GatewayIPv6',
 344         description => 'Default gateway for IPv6 traffic.',
 345         optional => 1,
 346     },
 347     firewall => {
 348         type => 'boolean',
 349         format_description => '[1|0]',
 350         description => "Controls whether this interface's firewall rules should be used.",
 351         optional => 1,
 352     },
 353     tag => {
 354         type => 'integer',
 355         format_description => 'VlanNo',
 356         minimum => '2',
 357         maximum => '4094',
 358         description => "VLAN tag for this interface.",
 359         optional => 1,
 360     },
 361     trunks => {
 362         type => 'string',
 363         pattern => qr/\d+(?:;\d+)*/,
 364         format_description => 'vlanid[;vlanid...]',
 365         description => "VLAN ids to pass through the interface",
 366         optional => 1,
 367     },
 368 };
 369 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 370
 371 my $MAX_LXC_NETWORKS = 10;
 372 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 373     $confdesc->{"net$i"} = {
 374         optional => 1,
 375         type => 'string', format => $netconf_desc,
 376         description => "Specifies network interfaces for the container.",
 377     };
 378 }
 379
 380 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 381 sub verify_lxc_mp_string{
 382     my ($mp, $noerr) = @_;
 383
 384     # do not allow:
 385     # /./ or /../
 386     # /. or /.. at the end
 387     # ../ at the beginning
 388
 389     if($mp =~ m@/\.\.?/@ ||
 390        $mp =~ m@/\.\.?$@ ||
 391        $mp =~ m@^\.\./@){
 392         return undef if $noerr;
 393         die "$mp contains illegal character sequences\n";
 394     }
 395     return $mp;
 396 }
 397
 398 my $mp_desc = {
 399     %$rootfs_desc,
 400     mp => {
 401         type => 'string',
 402         format => 'pve-lxc-mp-string',
 403         format_description => 'Path',
 404         description => 'Path to the mountpoint as seen from inside the container.',
 405     },
 406 };
 407 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 408
 409 my $unuseddesc = {
 410     optional => 1,
 411     type => 'string', format => 'pve-volume-id',
 412     description => "Reference to unused volumes.",
 413 };
 414
 415 my $MAX_MOUNT_POINTS = 10;
 416 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 417     $confdesc->{"mp$i"} = {
 418         optional => 1,
 419         type => 'string', format => $mp_desc,
 420         description => "Use volume as container mount point (experimental feature).",
 421         optional => 1,
 422     };
 423 }
 424
 425 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 426 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 427     $confdesc->{"unused$i"} = $unuseddesc;
 428 }
 429
 430 sub write_pct_config {
 431     my ($filename, $conf) = @_;
 432
 433     delete $conf->{snapstate}; # just to be sure
 434     my $volidlist = get_vm_volumes($conf);
 435     my $used_volids = {};
 436     foreach my $vid (@$volidlist) {
 437         $used_volids->{$vid} = 1;
 438     }
 439
 440     # remove 'unusedX' settings if the volume is still used
 441     foreach my $key (keys %$conf) {
 442         my $value = $conf->{$key};
 443         if ($key =~ m/^unused/ && $used_volids->{$value}) {
 444             delete $conf->{$key};
 445         }
 446     }
 447
 448     my $generate_raw_config = sub {
 449         my ($conf) = @_;
 450
 451         my $raw = '';
 452
 453         # add description as comment to top of file
 454         my $descr = $conf->{description} || '';
 455         foreach my $cl (split(/\n/, $descr)) {
 456             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 457         }
 458
 459         foreach my $key (sort keys %$conf) {
 460             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 461                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 462             my $value = $conf->{$key};
 463             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 464             $raw .= "$key: $value\n";
 465         }
 466
 467         if (my $lxcconf = $conf->{lxc}) {
 468             foreach my $entry (@$lxcconf) {
 469                 my ($k, $v) = @$entry;
 470                 $raw .= "$k: $v\n";
 471             }
 472         }
 473
 474         return $raw;
 475     };
 476
 477     my $raw = &$generate_raw_config($conf);
 478
 479     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 480         $raw .= "\n[$snapname]\n";
 481         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 482     }
 483
 484     return $raw;
 485 }
 486
 487 sub check_type {
 488     my ($key, $value) = @_;
 489
 490     die "unknown setting '$key'\n" if !$confdesc->{$key};
 491
 492     my $type = $confdesc->{$key}->{type};
 493
 494     if (!defined($value)) {
 495         die "got undefined value\n";
 496     }
 497
 498     if ($value =~ m/[\n\r]/) {
 499         die "property contains a line feed\n";
 500     }
 501
 502     if ($type eq 'boolean') {
 503         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 504         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 505         die "type check ('boolean') failed - got '$value'\n";
 506     } elsif ($type eq 'integer') {
 507         return int($1) if $value =~ m/^(\d+)$/;
 508         die "type check ('integer') failed - got '$value'\n";
 509     } elsif ($type eq 'number') {
 510         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 511         die "type check ('number') failed - got '$value'\n";
 512     } elsif ($type eq 'string') {
 513         if (my $fmt = $confdesc->{$key}->{format}) {
 514             PVE::JSONSchema::check_format($fmt, $value);
 515             return $value;
 516         }
 517         return $value;
 518     } else {
 519         die "internal error"
 520     }
 521 }
 522
 523 sub parse_pct_config {
 524     my ($filename, $raw) = @_;
 525
 526     return undef if !defined($raw);
 527
 528     my $res = {
 529         digest => Digest::SHA::sha1_hex($raw),
 530         snapshots => {},
 531     };
 532
 533     $filename =~ m|/lxc/(\d+).conf$|
 534         || die "got strange filename '$filename'";
 535
 536     my $vmid = $1;
 537
 538     my $conf = $res;
 539     my $descr = '';
 540     my $section = '';
 541
 542     my @lines = split(/\n/, $raw);
 543     foreach my $line (@lines) {
 544         next if $line =~ m/^\s*$/;
 545
 546         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 547             $section = $1;
 548             $conf->{description} = $descr if $descr;
 549             $descr = '';
 550             $conf = $res->{snapshots}->{$section} = {};
 551             next;
 552         }
 553
 554         if ($line =~ m/^\#(.*)\s*$/) {
 555             $descr .= PVE::Tools::decode_text($1) . "\n";
 556             next;
 557         }
 558
 559         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 560             my $key = $1;
 561             my $value = $3;
 562             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 563             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 564                 push @{$conf->{lxc}}, [$key, $value];
 565             } elsif (my $errmsg = $validity) {
 566                 warn "vm $vmid - $key: $errmsg\n";
 567             } else {
 568                 warn "vm $vmid - unable to parse config: $line\n";
 569             }
 570         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 571             $descr .= PVE::Tools::decode_text($2);
 572         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 573             $conf->{snapstate} = $1;
 574         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 575             my $key = $1;
 576             my $value = $2;
 577             eval { $value = check_type($key, $value); };
 578             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 579             $conf->{$key} = $value;
 580         } else {
 581             warn "vm $vmid - unable to parse config: $line\n";
 582         }
 583     }
 584
 585     $conf->{description} = $descr if $descr;
 586
 587     delete $res->{snapstate}; # just to be sure
 588
 589     return $res;
 590 }
 591
 592 sub config_list {
 593     my $vmlist = PVE::Cluster::get_vmlist();
 594     my $res = {};
 595     return $res if !$vmlist || !$vmlist->{ids};
 596     my $ids = $vmlist->{ids};
 597
 598     foreach my $vmid (keys %$ids) {
 599         next if !$vmid; # skip CT0
 600         my $d = $ids->{$vmid};
 601         next if !$d->{node} || $d->{node} ne $nodename;
 602         next if !$d->{type} || $d->{type} ne 'lxc';
 603         $res->{$vmid}->{type} = 'lxc';
 604     }
 605     return $res;
 606 }
 607
 608 sub cfs_config_path {
 609     my ($vmid, $node) = @_;
 610
 611     $node = $nodename if !$node;
 612     return "nodes/$node/lxc/$vmid.conf";
 613 }
 614
 615 sub config_file {
 616     my ($vmid, $node) = @_;
 617
 618     my $cfspath = cfs_config_path($vmid, $node);
 619     return "/etc/pve/$cfspath";
 620 }
 621
 622 sub load_config {
 623     my ($vmid, $node) = @_;
 624
 625     $node = $nodename if !$node;
 626     my $cfspath = cfs_config_path($vmid, $node);
 627
 628     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 629     die "container $vmid does not exist\n" if !defined($conf);
 630
 631     return $conf;
 632 }
 633
 634 sub create_config {
 635     my ($vmid, $conf) = @_;
 636
 637     my $dir = "/etc/pve/nodes/$nodename/lxc";
 638     mkdir $dir;
 639
 640     write_config($vmid, $conf);
 641 }
 642
 643 sub destroy_config {
 644     my ($vmid) = @_;
 645
 646     unlink config_file($vmid, $nodename);
 647 }
 648
 649 sub write_config {
 650     my ($vmid, $conf) = @_;
 651
 652     my $cfspath = cfs_config_path($vmid);
 653
 654     PVE::Cluster::cfs_write_file($cfspath, $conf);
 655 }
 656
 657 # flock: we use one file handle per process, so lock file
 658 # can be called multiple times and will succeed for the same process.
 659
 660 my $lock_handles =  {};
 661 my $lockdir = "/run/lock/lxc";
 662
 663 sub config_file_lock {
 664     my ($vmid) = @_;
 665
 666     return "$lockdir/pve-config-${vmid}.lock";
 667 }
 668
 669 sub lock_config_full {
 670     my ($vmid, $timeout, $code, @param) = @_;
 671
 672     my $filename = config_file_lock($vmid);
 673
 674     mkdir $lockdir if !-d $lockdir;
 675
 676     my $res = lock_file($filename, $timeout, $code, @param);
 677
 678     die $@ if $@;
 679
 680     return $res;
 681 }
 682
 683 sub lock_config_mode {
 684     my ($vmid, $timeout, $shared, $code, @param) = @_;
 685
 686     my $filename = config_file_lock($vmid);
 687
 688     mkdir $lockdir if !-d $lockdir;
 689
 690     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
 691
 692     die $@ if $@;
 693
 694     return $res;
 695 }
 696
 697 sub lock_config {
 698     my ($vmid, $code, @param) = @_;
 699
 700     return lock_config_full($vmid, 10, $code, @param);
 701 }
 702
 703 sub option_exists {
 704     my ($name) = @_;
 705
 706     return defined($confdesc->{$name});
 707 }
 708
 709 # add JSON properties for create and set function
 710 sub json_config_properties {
 711     my $prop = shift;
 712
 713     foreach my $opt (keys %$confdesc) {
 714         next if $opt eq 'parent' || $opt eq 'snaptime';
 715         next if $prop->{$opt};
 716         $prop->{$opt} = $confdesc->{$opt};
 717     }
 718
 719     return $prop;
 720 }
 721
 722 # container status helpers
 723
 724 sub list_active_containers {
 725
 726     my $filename = "/proc/net/unix";
 727
 728     # similar test is used by lcxcontainers.c: list_active_containers
 729     my $res = {};
 730
 731     my $fh = IO::File->new ($filename, "r");
 732     return $res if !$fh;
 733
 734     while (defined(my $line = <$fh>)) {
 735         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 736             my $path = $1;
 737             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 738                 $res->{$1} = 1;
 739             }
 740         }
 741     }
 742
 743     close($fh);
 744
 745     return $res;
 746 }
 747
 748 # warning: this is slow
 749 sub check_running {
 750     my ($vmid) = @_;
 751
 752     my $active_hash = list_active_containers();
 753
 754     return 1 if defined($active_hash->{$vmid});
 755
 756     return undef;
 757 }
 758
 759 sub get_container_disk_usage {
 760     my ($vmid, $pid) = @_;
 761
 762     return PVE::Tools::df("/proc/$pid/root/", 1);
 763 }
 764
 765 my $last_proc_vmid_stat;
 766
 767 my $parse_cpuacct_stat = sub {
 768     my ($vmid) = @_;
 769
 770     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 771
 772     my $stat = {};
 773
 774     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 775
 776         $stat->{utime} = $1;
 777         $stat->{stime} = $2;
 778
 779     }
 780
 781     return $stat;
 782 };
 783
 784 sub vmstatus {
 785     my ($opt_vmid) = @_;
 786
 787     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 788
 789     my $active_hash = list_active_containers();
 790
 791     my $cpucount = $cpuinfo->{cpus} || 1;
 792
 793     my $cdtime = gettimeofday;
 794
 795     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 796
 797     foreach my $vmid (keys %$list) {
 798         my $d = $list->{$vmid};
 799
 800         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 801         warn $@ if $@; # ignore errors (consider them stopped)
 802
 803         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 804
 805         my $cfspath = cfs_config_path($vmid);
 806         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 807
 808         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 809         $d->{name} =~ s/[\s]//g;
 810
 811         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 812
 813         $d->{lock} = $conf->{lock} || '';
 814
 815         if ($d->{pid}) {
 816             my $res = get_container_disk_usage($vmid, $d->{pid});
 817             $d->{disk} = $res->{used};
 818             $d->{maxdisk} = $res->{total};
 819         } else {
 820             $d->{disk} = 0;
 821             # use 4GB by default ??
 822             if (my $rootfs = $conf->{rootfs}) {
 823                 my $rootinfo = parse_ct_rootfs($rootfs);
 824                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 825             } else {
 826                 $d->{maxdisk} = 4*1024*1024*1024;
 827             }
 828         }
 829
 830         $d->{mem} = 0;
 831         $d->{swap} = 0;
 832         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 833         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 834
 835         $d->{uptime} = 0;
 836         $d->{cpu} = 0;
 837
 838         $d->{netout} = 0;
 839         $d->{netin} = 0;
 840
 841         $d->{diskread} = 0;
 842         $d->{diskwrite} = 0;
 843
 844         $d->{template} = is_template($conf);
 845     }
 846
 847     foreach my $vmid (keys %$list) {
 848         my $d = $list->{$vmid};
 849         my $pid = $d->{pid};
 850
 851         next if !$pid; # skip stopped CTs
 852
 853         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 854         $d->{uptime} = time - $ctime; # the method lxcfs uses
 855
 856         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 857         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 858
 859         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 860         my @bytes = split(/\n/, $blkio_bytes);
 861         foreach my $byte (@bytes) {
 862             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 863                 $d->{diskread} = $2 if $key eq 'Read';
 864                 $d->{diskwrite} = $2 if $key eq 'Write';
 865             }
 866         }
 867
 868         my $pstat = &$parse_cpuacct_stat($vmid);
 869
 870         my $used = $pstat->{utime} + $pstat->{stime};
 871
 872         my $old = $last_proc_vmid_stat->{$vmid};
 873         if (!$old) {
 874             $last_proc_vmid_stat->{$vmid} = {
 875                 time => $cdtime,
 876                 used => $used,
 877                 cpu => 0,
 878             };
 879             next;
 880         }
 881
 882         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 883
 884         if ($dtime > 1000) {
 885             my $dutime = $used -  $old->{used};
 886
 887             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 888             $last_proc_vmid_stat->{$vmid} = {
 889                 time => $cdtime,
 890                 used => $used,
 891                 cpu => $d->{cpu},
 892             };
 893         } else {
 894             $d->{cpu} = $old->{cpu};
 895         }
 896     }
 897
 898     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 899
 900     foreach my $dev (keys %$netdev) {
 901         next if $dev !~ m/^veth([1-9]\d*)i/;
 902         my $vmid = $1;
 903         my $d = $list->{$vmid};
 904
 905         next if !$d;
 906
 907         $d->{netout} += $netdev->{$dev}->{receive};
 908         $d->{netin} += $netdev->{$dev}->{transmit};
 909
 910     }
 911
 912     return $list;
 913 }
 914
 915 sub classify_mountpoint {
 916     my ($vol) = @_;
 917     if ($vol =~ m!^/!) {
 918         return 'device' if $vol =~ m!^/dev/!;
 919         return 'bind';
 920     }
 921     return 'volume';
 922 }
 923
 924 my $parse_ct_mountpoint_full = sub {
 925     my ($desc, $data, $noerr) = @_;
 926
 927     $data //= '';
 928
 929     my $res;
 930     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 931     if ($@) {
 932         return undef if $noerr;
 933         die $@;
 934     }
 935
 936     if (defined(my $size = $res->{size})) {
 937         $size = PVE::JSONSchema::parse_size($size);
 938         if (!defined($size)) {
 939             return undef if $noerr;
 940             die "invalid size: $size\n";
 941         }
 942         $res->{size} = $size;
 943     }
 944
 945     $res->{type} = classify_mountpoint($res->{volume});
 946
 947     return $res;
 948 };
 949
 950 sub parse_ct_rootfs {
 951     my ($data, $noerr) = @_;
 952
 953     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 954
 955     $res->{mp} = '/' if defined($res);
 956
 957     return $res;
 958 }
 959
 960 sub parse_ct_mountpoint {
 961     my ($data, $noerr) = @_;
 962
 963     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 964 }
 965
 966 sub print_ct_mountpoint {
 967     my ($info, $nomp) = @_;
 968     my $skip = [ 'type' ];
 969     push @$skip, 'mp' if $nomp;
 970     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 971 }
 972
 973 sub print_lxc_network {
 974     my $net = shift;
 975     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 976 }
 977
 978 sub parse_lxc_network {
 979     my ($data) = @_;
 980
 981     my $res = {};
 982
 983     return $res if !$data;
 984
 985     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 986
 987     $res->{type} = 'veth';
 988     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 989
 990     return $res;
 991 }
 992
 993 sub read_cgroup_value {
 994     my ($group, $vmid, $name, $full) = @_;
 995
 996     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 997
 998     return PVE::Tools::file_get_contents($path) if $full;
 999
1000     return PVE::Tools::file_read_firstline($path);
1001 }
1002
1003 sub write_cgroup_value {
1004    my ($group, $vmid, $name, $value) = @_;
1005
1006    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
1007    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
1008
1009 }
1010
1011 sub find_lxc_console_pids {
1012
1013     my $res = {};
1014
1015     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
1016         my ($pid) = @_;
1017
1018         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
1019         return if !$cmdline;
1020
1021         my @args = split(/\0/, $cmdline);
1022
1023         # search for lxc-console -n <vmid>
1024         return if scalar(@args) != 3;
1025         return if $args[1] ne '-n';
1026         return if $args[2] !~ m/^\d+$/;
1027         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1028
1029         my $vmid = $args[2];
1030
1031         push @{$res->{$vmid}}, $pid;
1032     });
1033
1034     return $res;
1035 }
1036
1037 sub find_lxc_pid {
1038     my ($vmid) = @_;
1039
1040     my $pid = undef;
1041     my $parser = sub {
1042         my $line = shift;
1043         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1044     };
1045     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1046
1047     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1048
1049     return $pid;
1050 }
1051
1052 # Note: we cannot use Net:IP, because that only allows strict
1053 # CIDR networks
1054 sub parse_ipv4_cidr {
1055     my ($cidr, $noerr) = @_;
1056
1057     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1058         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1059     }
1060
1061     return undef if $noerr;
1062
1063     die "unable to parse ipv4 address/mask\n";
1064 }
1065
1066 sub check_lock {
1067     my ($conf) = @_;
1068
1069     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1070 }
1071
1072 sub has_lock {
1073     my ($conf, $lock) = @_;
1074     return $conf->{lock} && (!defined($lock) || $lock eq $conf->{lock});
1075 }
1076
1077 sub check_protection {
1078     my ($vm_conf, $err_msg) = @_;
1079
1080     if ($vm_conf->{protection}) {
1081         die "$err_msg - protection mode enabled\n";
1082     }
1083 }
1084
1085 sub update_lxc_config {
1086     my ($storage_cfg, $vmid, $conf) = @_;
1087
1088     my $dir = "/var/lib/lxc/$vmid";
1089
1090     if ($conf->{template}) {
1091
1092         unlink "$dir/config";
1093
1094         return;
1095     }
1096
1097     my $raw = '';
1098
1099     die "missing 'arch' - internal error" if !$conf->{arch};
1100     $raw .= "lxc.arch = $conf->{arch}\n";
1101
1102     my $unprivileged = $conf->{unprivileged};
1103     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1104
1105     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1106     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux | alpine | unmanaged)$/x) {
1107         my $inc ="/usr/share/lxc/config/$ostype.common.conf";
1108         $inc ="/usr/share/lxc/config/common.conf" if !-f $inc;
1109         $raw .= "lxc.include = $inc\n";
1110         if ($unprivileged || $custom_idmap) {
1111             $inc = "/usr/share/lxc/config/$ostype.userns.conf";
1112             $inc = "/usr/share/lxc/config/userns.conf" if !-f $inc;
1113             $raw .= "lxc.include = $inc\n"
1114         }
1115     } else {
1116         die "implement me (ostype $ostype)";
1117     }
1118
1119     # WARNING: DO NOT REMOVE this without making sure that loop device nodes
1120     # cannot be exposed to the container with r/w access (cgroup perms).
1121     # When this is enabled mounts will still remain in the monitor's namespace
1122     # after the container unmounted them and thus will not detach from their
1123     # files while the container is running!
1124     $raw .= "lxc.monitor.unshare = 1\n";
1125
1126     # Should we read them from /etc/subuid?
1127     if ($unprivileged && !$custom_idmap) {
1128         $raw .= "lxc.id_map = u 0 100000 65536\n";
1129         $raw .= "lxc.id_map = g 0 100000 65536\n";
1130     }
1131
1132     if (!has_dev_console($conf)) {
1133         $raw .= "lxc.console = none\n";
1134         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1135     }
1136
1137     my $ttycount = get_tty_count($conf);
1138     $raw .= "lxc.tty = $ttycount\n";
1139
1140     # some init scripts expect a linux terminal (turnkey).
1141     $raw .= "lxc.environment = TERM=linux\n";
1142
1143     my $utsname = $conf->{hostname} || "CT$vmid";
1144     $raw .= "lxc.utsname = $utsname\n";
1145
1146     my $memory = $conf->{memory} || 512;
1147     my $swap = $conf->{swap} // 0;
1148
1149     my $lxcmem = int($memory*1024*1024);
1150     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1151
1152     my $lxcswap = int(($memory + $swap)*1024*1024);
1153     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1154
1155     if (my $cpulimit = $conf->{cpulimit}) {
1156         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1157         my $value = int(100000*$cpulimit);
1158         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1159     }
1160
1161     my $shares = $conf->{cpuunits} || 1024;
1162     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1163
1164     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1165
1166     $raw .= "lxc.rootfs = $dir/rootfs\n";
1167
1168     my $netcount = 0;
1169     foreach my $k (keys %$conf) {
1170         next if $k !~ m/^net(\d+)$/;
1171         my $ind = $1;
1172         my $d = parse_lxc_network($conf->{$k});
1173         $netcount++;
1174         $raw .= "lxc.network.type = veth\n";
1175         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1176         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1177         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1178         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1179     }
1180
1181     if (my $lxcconf = $conf->{lxc}) {
1182         foreach my $entry (@$lxcconf) {
1183             my ($k, $v) = @$entry;
1184             $netcount++ if $k eq 'lxc.network.type';
1185             $raw .= "$k = $v\n";
1186         }
1187     }
1188
1189     $raw .= "lxc.network.type = empty\n" if !$netcount;
1190
1191     File::Path::mkpath("$dir/rootfs");
1192
1193     PVE::Tools::file_set_contents("$dir/config", $raw);
1194 }
1195
1196 # verify and cleanup nameserver list (replace \0 with ' ')
1197 sub verify_nameserver_list {
1198     my ($nameserver_list) = @_;
1199
1200     my @list = ();
1201     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1202         PVE::JSONSchema::pve_verify_ip($server);
1203         push @list, $server;
1204     }
1205
1206     return join(' ', @list);
1207 }
1208
1209 sub verify_searchdomain_list {
1210     my ($searchdomain_list) = @_;
1211
1212     my @list = ();
1213     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1214         # todo: should we add checks for valid dns domains?
1215         push @list, $server;
1216     }
1217
1218     return join(' ', @list);
1219 }
1220
1221 sub is_volume_in_use {
1222     my ($config, $volid, $include_snapshots) = @_;
1223     my $used = 0;
1224
1225     foreach_mountpoint($config, sub {
1226         my ($ms, $mountpoint) = @_;
1227         return if $used;
1228         if ($mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid) {
1229             $used = 1;
1230         }
1231     });
1232
1233     my $snapshots = $config->{snapshots};
1234     if ($include_snapshots && $snapshots) {
1235         foreach my $snap (keys %$snapshots) {
1236             $used ||= is_volume_in_use($snapshots->{$snap}, $volid);
1237         }
1238     }
1239
1240     return $used;
1241 }
1242
1243 sub add_unused_volume {
1244     my ($config, $volid) = @_;
1245
1246     my $key;
1247     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1248         my $test = "unused$ind";
1249         if (my $vid = $config->{$test}) {
1250             return if $vid eq $volid; # do not add duplicates
1251         } else {
1252             $key = $test;
1253         }
1254     }
1255
1256     die "Too many unused volumes - please delete them first.\n" if !$key;
1257
1258     $config->{$key} = $volid;
1259
1260     return $key;
1261 }
1262
1263 sub update_pct_config {
1264     my ($vmid, $conf, $running, $param, $delete) = @_;
1265
1266     my @nohotplug;
1267
1268     my $new_disks = 0;
1269     my @deleted_volumes;
1270
1271     my $rootdir;
1272     if ($running) {
1273         my $pid = find_lxc_pid($vmid);
1274         $rootdir = "/proc/$pid/root";
1275     }
1276
1277     my $hotplug_error = sub {
1278         if ($running) {
1279             push @nohotplug, @_;
1280             return 1;
1281         } else {
1282             return 0;
1283         }
1284     };
1285
1286     if (defined($delete)) {
1287         foreach my $opt (@$delete) {
1288             if (!exists($conf->{$opt})) {
1289                 warn "no such option: $opt\n";
1290                 next;
1291             }
1292
1293             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1294                 die "unable to delete required option '$opt'\n";
1295             } elsif ($opt eq 'swap') {
1296                 delete $conf->{$opt};
1297                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1298             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1299                 delete $conf->{$opt};
1300             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1301                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1302                 next if $hotplug_error->($opt);
1303                 delete $conf->{$opt};
1304             } elsif ($opt =~ m/^net(\d)$/) {
1305                 delete $conf->{$opt};
1306                 next if !$running;
1307                 my $netid = $1;
1308                 PVE::Network::veth_delete("veth${vmid}i$netid");
1309             } elsif ($opt eq 'protection') {
1310                 delete $conf->{$opt};
1311             } elsif ($opt =~ m/^unused(\d+)$/) {
1312                 next if $hotplug_error->($opt);
1313                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1314                 push @deleted_volumes, $conf->{$opt};
1315                 delete $conf->{$opt};
1316             } elsif ($opt =~ m/^mp(\d+)$/) {
1317                 next if $hotplug_error->($opt);
1318                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1319                 my $mp = parse_ct_mountpoint($conf->{$opt});
1320                 delete $conf->{$opt};
1321                 if ($mp->{type} eq 'volume') {
1322                     add_unused_volume($conf, $mp->{volume});
1323                 }
1324             } elsif ($opt eq 'unprivileged') {
1325                 die "unable to delete read-only option: '$opt'\n";
1326             } else {
1327                 die "implement me (delete: $opt)"
1328             }
1329             write_config($vmid, $conf) if $running;
1330         }
1331     }
1332
1333     # There's no separate swap size to configure, there's memory and "total"
1334     # memory (iow. memory+swap). This means we have to change them together.
1335     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1336     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1337     if (defined($wanted_memory) || defined($wanted_swap)) {
1338
1339         my $old_memory = ($conf->{memory} || 512);
1340         my $old_swap = ($conf->{swap} || 0);
1341
1342         $wanted_memory //= $old_memory;
1343         $wanted_swap //= $old_swap;
1344
1345         my $total = $wanted_memory + $wanted_swap;
1346         if ($running) {
1347             my $old_total = $old_memory + $old_swap;
1348             if ($total > $old_total) {
1349                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1350                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1351             } else {
1352                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1353                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1354             }
1355         }
1356         $conf->{memory} = $wanted_memory;
1357         $conf->{swap} = $wanted_swap;
1358
1359         write_config($vmid, $conf) if $running;
1360     }
1361
1362     my $used_volids = {};
1363
1364     foreach my $opt (keys %$param) {
1365         my $value = $param->{$opt};
1366         if ($opt eq 'hostname') {
1367             $conf->{$opt} = $value;
1368         } elsif ($opt eq 'onboot') {
1369             $conf->{$opt} = $value ? 1 : 0;
1370         } elsif ($opt eq 'startup') {
1371             $conf->{$opt} = $value;
1372         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1373             next if $hotplug_error->($opt);
1374             $conf->{$opt} = $value;
1375         } elsif ($opt eq 'nameserver') {
1376             next if $hotplug_error->($opt);
1377             my $list = verify_nameserver_list($value);
1378             $conf->{$opt} = $list;
1379         } elsif ($opt eq 'searchdomain') {
1380             next if $hotplug_error->($opt);
1381             my $list = verify_searchdomain_list($value);
1382             $conf->{$opt} = $list;
1383         } elsif ($opt eq 'cpulimit') {
1384             next if $hotplug_error->($opt); # FIXME: hotplug
1385             $conf->{$opt} = $value;
1386         } elsif ($opt eq 'cpuunits') {
1387             $conf->{$opt} = $value;
1388             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1389         } elsif ($opt eq 'description') {
1390             $conf->{$opt} = PVE::Tools::encode_text($value);
1391         } elsif ($opt =~ m/^net(\d+)$/) {
1392             my $netid = $1;
1393             my $net = parse_lxc_network($value);
1394             if (!$running) {
1395                 $conf->{$opt} = print_lxc_network($net);
1396             } else {
1397                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1398             }
1399         } elsif ($opt eq 'protection') {
1400             $conf->{$opt} = $value ? 1 : 0;
1401         } elsif ($opt =~ m/^mp(\d+)$/) {
1402             next if $hotplug_error->($opt);
1403             check_protection($conf, "can't update CT $vmid drive '$opt'");
1404             my $old = $conf->{$opt};
1405             $conf->{$opt} = $value;
1406             if (defined($old)) {
1407                 my $mp = parse_ct_mountpoint($old);
1408                 if ($mp->{type} eq 'volume') {
1409                     add_unused_volume($conf, $mp->{volume});
1410                 }
1411             }
1412             $new_disks = 1;
1413             my $mp = parse_ct_mountpoint($value);
1414             $used_volids->{$mp->{volume}} = 1;
1415         } elsif ($opt eq 'rootfs') {
1416             next if $hotplug_error->($opt);
1417             check_protection($conf, "can't update CT $vmid drive '$opt'");
1418             my $old = $conf->{$opt};
1419             $conf->{$opt} = $value;
1420             if (defined($old)) {
1421                 my $mp = parse_ct_rootfs($old);
1422                 if ($mp->{type} eq 'volume') {
1423                     add_unused_volume($conf, $mp->{volume});
1424                 }
1425             }
1426             my $mp = parse_ct_rootfs($value);
1427             $used_volids->{$mp->{volume}} = 1;
1428         } elsif ($opt eq 'unprivileged') {
1429             die "unable to modify read-only option: '$opt'\n";
1430         } elsif ($opt eq 'ostype') {
1431             next if $hotplug_error->($opt);
1432             $conf->{$opt} = $value;
1433         } else {
1434             die "implement me: $opt";
1435         }
1436         write_config($vmid, $conf) if $running;
1437     }
1438
1439     # Apply deletions and creations of new volumes
1440     if (@deleted_volumes) {
1441         my $storage_cfg = PVE::Storage::config();
1442         foreach my $volume (@deleted_volumes) {
1443             next if $used_volids->{$volume}; # could have been re-added, too
1444             # also check for references in snapshots
1445             next if is_volume_in_use($conf, $volume, 1);
1446             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1447         }
1448     }
1449
1450     if ($new_disks) {
1451         my $storage_cfg = PVE::Storage::config();
1452         create_disks($storage_cfg, $vmid, $conf, $conf);
1453     }
1454
1455     # This should be the last thing we do here
1456     if ($running && scalar(@nohotplug)) {
1457         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1458     }
1459 }
1460
1461 sub has_dev_console {
1462     my ($conf) = @_;
1463
1464     return !(defined($conf->{console}) && !$conf->{console});
1465 }
1466
1467 sub get_tty_count {
1468     my ($conf) = @_;
1469
1470     return $conf->{tty} // $confdesc->{tty}->{default};
1471 }
1472
1473 sub get_cmode {
1474     my ($conf) = @_;
1475
1476     return $conf->{cmode} // $confdesc->{cmode}->{default};
1477 }
1478
1479 sub get_console_command {
1480     my ($vmid, $conf) = @_;
1481
1482     my $cmode = get_cmode($conf);
1483
1484     if ($cmode eq 'console') {
1485         return ['lxc-console', '-n',  $vmid, '-t', 0];
1486     } elsif ($cmode eq 'tty') {
1487         return ['lxc-console', '-n',  $vmid];
1488     } elsif ($cmode eq 'shell') {
1489         return ['lxc-attach', '--clear-env', '-n', $vmid];
1490     } else {
1491         die "internal error";
1492     }
1493 }
1494
1495 sub get_primary_ips {
1496     my ($conf) = @_;
1497
1498     # return data from net0
1499
1500     return undef if !defined($conf->{net0});
1501     my $net = parse_lxc_network($conf->{net0});
1502
1503     my $ipv4 = $net->{ip};
1504     if ($ipv4) {
1505         if ($ipv4 =~ /^(dhcp|manual)$/) {
1506             $ipv4 = undef
1507         } else {
1508             $ipv4 =~ s!/\d+$!!;
1509         }
1510     }
1511     my $ipv6 = $net->{ip6};
1512     if ($ipv6) {
1513         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1514             $ipv6 = undef;
1515         } else {
1516             $ipv6 =~ s!/\d+$!!;
1517         }
1518     }
1519
1520     return ($ipv4, $ipv6);
1521 }
1522
1523 sub delete_mountpoint_volume {
1524     my ($storage_cfg, $vmid, $volume) = @_;
1525
1526     return if classify_mountpoint($volume) ne 'volume';
1527
1528     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1529     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1530 }
1531
1532 sub destroy_lxc_container {
1533     my ($storage_cfg, $vmid, $conf) = @_;
1534
1535     foreach_mountpoint($conf, sub {
1536         my ($ms, $mountpoint) = @_;
1537         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1538     });
1539
1540     rmdir "/var/lib/lxc/$vmid/rootfs";
1541     unlink "/var/lib/lxc/$vmid/config";
1542     rmdir "/var/lib/lxc/$vmid";
1543     destroy_config($vmid);
1544
1545     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1546     #PVE::Tools::run_command($cmd);
1547 }
1548
1549 sub vm_stop_cleanup {
1550     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1551
1552     eval {
1553         if (!$keepActive) {
1554
1555             my $vollist = get_vm_volumes($conf);
1556             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1557         }
1558     };
1559     warn $@ if $@; # avoid errors - just warn
1560 }
1561
1562 my $safe_num_ne = sub {
1563     my ($a, $b) = @_;
1564
1565     return 0 if !defined($a) && !defined($b);
1566     return 1 if !defined($a);
1567     return 1 if !defined($b);
1568
1569     return $a != $b;
1570 };
1571
1572 my $safe_string_ne = sub {
1573     my ($a, $b) = @_;
1574
1575     return 0 if !defined($a) && !defined($b);
1576     return 1 if !defined($a);
1577     return 1 if !defined($b);
1578
1579     return $a ne $b;
1580 };
1581
1582 sub update_net {
1583     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1584
1585     if ($newnet->{type} ne 'veth') {
1586         # for when there are physical interfaces
1587         die "cannot update interface of type $newnet->{type}";
1588     }
1589
1590     my $veth = "veth${vmid}i${netid}";
1591     my $eth = $newnet->{name};
1592
1593     if (my $oldnetcfg = $conf->{$opt}) {
1594         my $oldnet = parse_lxc_network($oldnetcfg);
1595
1596         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1597             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1598
1599             PVE::Network::veth_delete($veth);
1600             delete $conf->{$opt};
1601             write_config($vmid, $conf);
1602
1603             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1604
1605         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1606                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1607                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1608
1609                 if ($oldnet->{bridge}) {
1610                     PVE::Network::tap_unplug($veth);
1611                     foreach (qw(bridge tag firewall)) {
1612                         delete $oldnet->{$_};
1613                     }
1614                     $conf->{$opt} = print_lxc_network($oldnet);
1615                     write_config($vmid, $conf);
1616                 }
1617
1618                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1619                 foreach (qw(bridge tag firewall)) {
1620                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1621                 }
1622                 $conf->{$opt} = print_lxc_network($oldnet);
1623                 write_config($vmid, $conf);
1624         }
1625     } else {
1626         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1627     }
1628
1629     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1630 }
1631
1632 sub hotplug_net {
1633     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1634
1635     my $veth = "veth${vmid}i${netid}";
1636     my $vethpeer = $veth . "p";
1637     my $eth = $newnet->{name};
1638
1639     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1640     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1641
1642     # attach peer in container
1643     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1644     PVE::Tools::run_command($cmd);
1645
1646     # link up peer in container
1647     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1648     PVE::Tools::run_command($cmd);
1649
1650     my $done = { type => 'veth' };
1651     foreach (qw(bridge tag firewall hwaddr name)) {
1652         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1653     }
1654     $conf->{$opt} = print_lxc_network($done);
1655
1656     write_config($vmid, $conf);
1657 }
1658
1659 sub update_ipconfig {
1660     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1661
1662     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1663
1664     my $optdata = parse_lxc_network($conf->{$opt});
1665     my $deleted = [];
1666     my $added = [];
1667     my $nscmd = sub {
1668         my $cmdargs = shift;
1669         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1670     };
1671     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1672
1673     my $change_ip_config = sub {
1674         my ($ipversion) = @_;
1675
1676         my $family_opt = "-$ipversion";
1677         my $suffix = $ipversion == 4 ? '' : $ipversion;
1678         my $gw= "gw$suffix";
1679         my $ip= "ip$suffix";
1680
1681         my $newip = $newnet->{$ip};
1682         my $newgw = $newnet->{$gw};
1683         my $oldip = $optdata->{$ip};
1684
1685         my $change_ip = &$safe_string_ne($oldip, $newip);
1686         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1687
1688         return if !$change_ip && !$change_gw;
1689
1690         # step 1: add new IP, if this fails we cancel
1691         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1692         if ($change_ip && $is_real_ip) {
1693             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1694             if (my $err = $@) {
1695                 warn $err;
1696                 return;
1697             }
1698         }
1699
1700         # step 2: replace gateway
1701         #   If this fails we delete the added IP and cancel.
1702         #   If it succeeds we save the config and delete the old IP, ignoring
1703         #   errors. The config is then saved.
1704         # Note: 'ip route replace' can add
1705         if ($change_gw) {
1706             if ($newgw) {
1707                 eval {
1708                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1709                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1710                     }
1711                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1712                 };
1713                 if (my $err = $@) {
1714                     warn $err;
1715                     # the route was not replaced, the old IP is still available
1716                     # rollback (delete new IP) and cancel
1717                     if ($change_ip) {
1718                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1719                         warn $@ if $@; # no need to die here
1720                     }
1721                     return;
1722                 }
1723             } else {
1724                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1725                 # if the route was not deleted, the guest might have deleted it manually
1726                 # warn and continue
1727                 warn $@ if $@;
1728             }
1729         }
1730
1731         # from this point on we save the configuration
1732         # step 3: delete old IP ignoring errors
1733         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1734             # We need to enable promote_secondaries, otherwise our newly added
1735             # address will be removed along with the old one.
1736             my $promote = 0;
1737             eval {
1738                 if ($ipversion == 4) {
1739                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1740                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1741                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1742                 }
1743                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1744             };
1745             warn $@ if $@; # no need to die here
1746
1747             if ($ipversion == 4) {
1748                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1749             }
1750         }
1751
1752         foreach my $property ($ip, $gw) {
1753             if ($newnet->{$property}) {
1754                 $optdata->{$property} = $newnet->{$property};
1755             } else {
1756                 delete $optdata->{$property};
1757             }
1758         }
1759         $conf->{$opt} = print_lxc_network($optdata);
1760         write_config($vmid, $conf);
1761         $lxc_setup->setup_network($conf);
1762     };
1763
1764     &$change_ip_config(4);
1765     &$change_ip_config(6);
1766
1767 }
1768
1769 # Internal snapshots
1770
1771 # NOTE: Snapshot create/delete involves several non-atomic
1772 # actions, and can take a long time.
1773 # So we try to avoid locking the file and use the 'lock' variable
1774 # inside the config file instead.
1775
1776 my $snapshot_copy_config = sub {
1777     my ($source, $dest) = @_;
1778
1779     foreach my $k (keys %$source) {
1780         next if $k eq 'snapshots';
1781         next if $k eq 'snapstate';
1782         next if $k eq 'snaptime';
1783         next if $k eq 'vmstate';
1784         next if $k eq 'lock';
1785         next if $k eq 'digest';
1786         next if $k eq 'description';
1787         next if $k =~ m/^unused\d+$/;
1788
1789         $dest->{$k} = $source->{$k};
1790     }
1791 };
1792
1793 my $snapshot_apply_config = sub {
1794     my ($conf, $snap) = @_;
1795
1796     # copy snapshot list
1797     my $newconf = {
1798         snapshots => $conf->{snapshots},
1799     };
1800
1801     # keep description and list of unused disks
1802     foreach my $k (keys %$conf) {
1803         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
1804         $newconf->{$k} = $conf->{$k};
1805     }
1806
1807     &$snapshot_copy_config($snap, $newconf);
1808
1809     return $newconf;
1810 };
1811
1812 sub snapshot_save_vmstate {
1813     die "implement me - snapshot_save_vmstate\n";
1814 }
1815
1816 sub snapshot_prepare {
1817     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1818
1819     my $snap;
1820
1821     my $updatefn =  sub {
1822
1823         my $conf = load_config($vmid);
1824
1825         die "you can't take a snapshot if it's a template\n"
1826             if is_template($conf);
1827
1828         check_lock($conf);
1829
1830         $conf->{lock} = 'snapshot';
1831
1832         die "snapshot name '$snapname' already used\n"
1833             if defined($conf->{snapshots}->{$snapname});
1834
1835         my $storecfg = PVE::Storage::config();
1836         die "snapshot feature is not available\n"
1837             if !has_feature('snapshot', $conf, $storecfg, undef, undef, $snapname eq 'vzdump');
1838
1839         $snap = $conf->{snapshots}->{$snapname} = {};
1840
1841         if ($save_vmstate && check_running($vmid)) {
1842             snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
1843         }
1844
1845         &$snapshot_copy_config($conf, $snap);
1846
1847         $snap->{snapstate} = "prepare";
1848         $snap->{snaptime} = time();
1849         $snap->{description} = $comment if $comment;
1850
1851         write_config($vmid, $conf);
1852     };
1853
1854     lock_config($vmid, $updatefn);
1855
1856     return $snap;
1857 }
1858
1859 sub snapshot_commit {
1860     my ($vmid, $snapname) = @_;
1861
1862     my $updatefn = sub {
1863
1864         my $conf = load_config($vmid);
1865
1866         die "missing snapshot lock\n"
1867             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1868
1869         my $snap = $conf->{snapshots}->{$snapname};
1870         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1871
1872         die "wrong snapshot state\n"
1873             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
1874
1875         delete $snap->{snapstate};
1876         delete $conf->{lock};
1877
1878         $conf->{parent} = $snapname;
1879
1880         write_config($vmid, $conf);
1881     };
1882
1883     lock_config($vmid, $updatefn);
1884 }
1885
1886 sub has_feature {
1887     my ($feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
1888
1889     my $err;
1890
1891     foreach_mountpoint($conf, sub {
1892         my ($ms, $mountpoint) = @_;
1893
1894         return if $err; # skip further test
1895         return if $backup_only && $ms ne 'rootfs' && !$mountpoint->{backup};
1896
1897         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname, $running);
1898     });
1899
1900     return $err ? 0 : 1;
1901 }
1902
1903 my $enter_namespace = sub {
1904     my ($vmid, $pid, $which, $type) = @_;
1905     sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY
1906         or die "failed to open $which namespace of container $vmid: $!\n";
1907     PVE::Tools::setns(fileno($fd), $type)
1908         or die "failed to enter $which namespace of container $vmid: $!\n";
1909     close $fd;
1910 };
1911
1912 my $do_syncfs = sub {
1913     my ($vmid, $pid, $socket) = @_;
1914
1915     &$enter_namespace($vmid, $pid, 'mnt', PVE::Tools::CLONE_NEWNS);
1916
1917     # Tell the parent process to start reading our /proc/mounts
1918     print {$socket} "go\n";
1919     $socket->flush();
1920
1921     # Receive /proc/self/mounts
1922     my $mountdata = do { local $/ = undef; <$socket> };
1923     close $socket;
1924
1925     # Now sync all mountpoints...
1926     my $mounts = PVE::ProcFSTools::parse_mounts($mountdata);
1927     foreach my $mp (@$mounts) {
1928         my ($what, $dir, $fs) = @$mp;
1929         next if $fs eq 'fuse.lxcfs';
1930         eval { PVE::Tools::sync_mountpoint($dir); };
1931         warn $@ if $@;
1932     }
1933 };
1934
1935 sub sync_container_namespace {
1936     my ($vmid) = @_;
1937     my $pid = find_lxc_pid($vmid);
1938
1939     # SOCK_DGRAM is nicer for barriers but cannot be slurped
1940     socketpair my $pfd, my $cfd, AF_UNIX, SOCK_STREAM, PF_UNSPEC
1941         or die "failed to create socketpair: $!\n";
1942
1943     my $child = fork();
1944     die "fork failed: $!\n" if !defined($child);
1945
1946     if (!$child) {
1947         eval {
1948             close $pfd;
1949             &$do_syncfs($vmid, $pid, $cfd);
1950         };
1951         if (my $err = $@) {
1952             warn $err;
1953             POSIX::_exit(1);
1954         }
1955         POSIX::_exit(0);
1956     }
1957     close $cfd;
1958     my $go = <$pfd>;
1959     die "failed to enter container namespace\n" if $go ne "go\n";
1960
1961     open my $mounts, '<', "/proc/$child/mounts"
1962         or die "failed to open container's /proc/mounts: $!\n";
1963     my $mountdata = do { local $/ = undef; <$mounts> };
1964     close $mounts;
1965     print {$pfd} $mountdata;
1966     close $pfd;
1967
1968     while (waitpid($child, 0) != $child) {}
1969     die "failed to sync container namespace\n" if $? != 0;
1970 }
1971
1972 sub check_freeze_needed {
1973     my ($vmid, $config, $save_vmstate) = @_;
1974
1975     my $ret = check_running($vmid);
1976     return ($ret, $ret);
1977 }
1978
1979 sub snapshot_create {
1980     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1981
1982     my $snap = snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
1983
1984     $save_vmstate = 0 if !$snap->{vmstate};
1985
1986     my $conf = load_config($vmid);
1987
1988     my ($running, $freezefs) = check_freeze_needed($vmid, $conf, $snap->{vmstate});
1989
1990     my $drivehash = {};
1991
1992     eval {
1993         if ($freezefs) {
1994             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1995             sync_container_namespace($vmid);
1996         }
1997
1998         my $storecfg = PVE::Storage::config();
1999         foreach_mountpoint($conf, sub {
2000             my ($ms, $mountpoint) = @_;
2001
2002             return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
2003             PVE::Storage::volume_snapshot($storecfg, $mountpoint->{volume}, $snapname);
2004             $drivehash->{$ms} = 1;
2005         });
2006     };
2007     my $err = $@;
2008
2009     if ($running) {
2010         if ($freezefs) {
2011             eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
2012             warn $@ if $@;
2013         }
2014     }
2015
2016     if ($err) {
2017         warn "snapshot create failed: starting cleanup\n";
2018         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
2019         warn "$@" if $@;
2020         die "$err\n";
2021     }
2022
2023     snapshot_commit($vmid, $snapname);
2024 }
2025
2026 # Note: $drivehash is only set when called from snapshot_create.
2027 sub snapshot_delete {
2028     my ($vmid, $snapname, $force, $drivehash) = @_;
2029
2030     my $prepare = 1;
2031
2032     my $snap;
2033     my $unused = [];
2034
2035     my $unlink_parent = sub {
2036         my ($confref, $new_parent) = @_;
2037
2038         if ($confref->{parent} && $confref->{parent} eq $snapname) {
2039             if ($new_parent) {
2040                 $confref->{parent} = $new_parent;
2041             } else {
2042                 delete $confref->{parent};
2043             }
2044         }
2045     };
2046
2047     my $updatefn =  sub {
2048         my ($remove_drive) = @_;
2049
2050         my $conf = load_config($vmid);
2051
2052         if (!$drivehash) {
2053             check_lock($conf);
2054             die "you can't delete a snapshot if vm is a template\n"
2055                 if is_template($conf);
2056         }
2057
2058         $snap = $conf->{snapshots}->{$snapname};
2059
2060         die "snapshot '$snapname' does not exist\n" if !defined($snap);
2061
2062         # remove parent refs
2063         if (!$prepare) {
2064             &$unlink_parent($conf, $snap->{parent});
2065             foreach my $sn (keys %{$conf->{snapshots}}) {
2066                 next if $sn eq $snapname;
2067                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
2068             }
2069         }
2070
2071         if ($remove_drive) {
2072             if ($remove_drive eq 'vmstate') {
2073                 die "implement me - saving vmstate\n";
2074             } else {
2075                 my $value = $snap->{$remove_drive};
2076                 my $mountpoint = $remove_drive eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2077                 delete $snap->{$remove_drive};
2078                 add_unused_volume($snap, $mountpoint->{volume});
2079             }
2080         }
2081
2082         if ($prepare) {
2083             $snap->{snapstate} = 'delete';
2084         } else {
2085             delete $conf->{snapshots}->{$snapname};
2086             delete $conf->{lock} if $drivehash;
2087             foreach my $volid (@$unused) {
2088                 add_unused_volume($conf, $volid);
2089             }
2090         }
2091
2092         write_config($vmid, $conf);
2093     };
2094
2095     lock_config($vmid, $updatefn);
2096
2097     # now remove vmstate file
2098     # never set for LXC!
2099     my $storecfg = PVE::Storage::config();
2100
2101     if ($snap->{vmstate}) {
2102         die "implement me - saving vmstate\n";
2103     };
2104
2105     # now remove all volume snapshots
2106     foreach_mountpoint($snap, sub {
2107         my ($ms, $mountpoint) = @_;
2108
2109         return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
2110         if (!$drivehash || $drivehash->{$ms}) {
2111             eval { PVE::Storage::volume_snapshot_delete($storecfg, $mountpoint->{volume}, $snapname); };
2112             if (my $err = $@) {
2113                 die $err if !$force;
2114                 warn $err;
2115             }
2116         }
2117
2118         # save changes (remove mp from snapshot)
2119         lock_config($vmid, $updatefn, $ms) if !$force;
2120         push @$unused, $mountpoint->{volume};
2121     });
2122
2123     # now cleanup config
2124     $prepare = 0;
2125     lock_config($vmid, $updatefn);
2126 }
2127
2128 sub snapshot_rollback {
2129     my ($vmid, $snapname) = @_;
2130
2131     my $prepare = 1;
2132
2133     my $storecfg = PVE::Storage::config();
2134
2135     my $conf = load_config($vmid);
2136
2137     my $get_snapshot_config = sub {
2138
2139         die "you can't rollback if vm is a template\n" if is_template($conf);
2140
2141         my $res = $conf->{snapshots}->{$snapname};
2142
2143         die "snapshot '$snapname' does not exist\n" if !defined($res);
2144
2145         return $res;
2146     };
2147
2148     my $snap = &$get_snapshot_config();
2149
2150     foreach_mountpoint($snap, sub {
2151         my ($ms, $mountpoint) = @_;
2152
2153         PVE::Storage::volume_rollback_is_possible($storecfg, $mountpoint->{volume}, $snapname);
2154     });
2155
2156     my $updatefn = sub {
2157
2158         $conf = load_config($vmid);
2159
2160         $snap = &$get_snapshot_config();
2161
2162         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
2163             if $snap->{snapstate};
2164
2165         if ($prepare) {
2166             check_lock($conf);
2167             PVE::Tools::run_command(['/usr/bin/lxc-stop', '-n', $vmid, '--kill'])
2168                 if check_running($vmid);
2169         }
2170
2171         die "unable to rollback vm $vmid: vm is running\n"
2172             if check_running($vmid);
2173
2174         if ($prepare) {
2175             $conf->{lock} = 'rollback';
2176         } else {
2177             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
2178             delete $conf->{lock};
2179         }
2180
2181         my $forcemachine;
2182
2183         if (!$prepare) {
2184             # copy snapshot config to current config
2185             $conf = &$snapshot_apply_config($conf, $snap);
2186             $conf->{parent} = $snapname;
2187         }
2188
2189         write_config($vmid, $conf);
2190
2191         if (!$prepare && $snap->{vmstate}) {
2192             die "implement me - save vmstate\n";
2193         }
2194     };
2195
2196     lock_config($vmid, $updatefn);
2197
2198     foreach_mountpoint($snap, sub {
2199         my ($ms, $mountpoint) = @_;
2200
2201         PVE::Storage::volume_snapshot_rollback($storecfg, $mountpoint->{volume}, $snapname);
2202     });
2203
2204     $prepare = 0;
2205     lock_config($vmid, $updatefn);
2206 }
2207
2208 sub template_create {
2209     my ($vmid, $conf) = @_;
2210
2211     my $storecfg = PVE::Storage::config();
2212
2213     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
2214     my $volid = $rootinfo->{volume};
2215
2216     die "Template feature is not available for '$volid'\n"
2217         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
2218
2219     PVE::Storage::activate_volumes($storecfg, [$volid]);
2220
2221     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
2222     $rootinfo->{volume} = $template_volid;
2223     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
2224
2225     write_config($vmid, $conf);
2226 }
2227
2228 sub is_template {
2229     my ($conf) = @_;
2230
2231     return 1 if defined $conf->{template} && $conf->{template} == 1;
2232 }
2233
2234 sub mountpoint_names {
2235     my ($reverse) = @_;
2236
2237     my @names = ('rootfs');
2238
2239     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
2240         push @names, "mp$i";
2241     }
2242
2243     return $reverse ? reverse @names : @names;
2244 }
2245
2246
2247 sub foreach_mountpoint_full {
2248     my ($conf, $reverse, $func) = @_;
2249
2250     foreach my $key (mountpoint_names($reverse)) {
2251         my $value = $conf->{$key};
2252         next if !defined($value);
2253         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2254         next if !defined($mountpoint);
2255
2256         &$func($key, $mountpoint);
2257     }
2258 }
2259
2260 sub foreach_mountpoint {
2261     my ($conf, $func) = @_;
2262
2263     foreach_mountpoint_full($conf, 0, $func);
2264 }
2265
2266 sub foreach_mountpoint_reverse {
2267     my ($conf, $func) = @_;
2268
2269     foreach_mountpoint_full($conf, 1, $func);
2270 }
2271
2272 sub check_ct_modify_config_perm {
2273     my ($rpcenv, $authuser, $vmid, $pool, $newconf, $delete) = @_;
2274
2275     return 1 if $authuser eq 'root@pam';
2276
2277     my $check = sub {
2278         my ($opt, $delete) = @_;
2279         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2280             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2281         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2282             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2283             return if $delete;
2284             my $data = $opt eq 'rootfs' ? parse_ct_rootfs($newconf->{$opt})
2285                                         : parse_ct_mountpoint($newconf->{$opt});
2286             raise_perm_exc("mountpoint type $data->{type}") if $data->{type} ne 'volume';
2287         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2288             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2289         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2290                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2291             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2292         } else {
2293             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2294         }
2295     };
2296
2297     foreach my $opt (keys %$newconf) {
2298         &$check($opt, 0);
2299     }
2300     foreach my $opt (@$delete) {
2301         &$check($opt, 1);
2302     }
2303
2304     return 1;
2305 }
2306
2307 sub umount_all {
2308     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2309
2310     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2311     my $volid_list = get_vm_volumes($conf);
2312
2313     foreach_mountpoint_reverse($conf, sub {
2314         my ($ms, $mountpoint) = @_;
2315
2316         my $volid = $mountpoint->{volume};
2317         my $mount = $mountpoint->{mp};
2318
2319         return if !$volid || !$mount;
2320
2321         my $mount_path = "$rootdir/$mount";
2322         $mount_path =~ s!/+!/!g;
2323
2324         return if !PVE::ProcFSTools::is_mounted($mount_path);
2325
2326         eval {
2327             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2328         };
2329         if (my $err = $@) {
2330             if ($noerr) {
2331                 warn $err;
2332             } else {
2333                 die $err;
2334             }
2335         }
2336     });
2337 }
2338
2339 sub mount_all {
2340     my ($vmid, $storage_cfg, $conf) = @_;
2341
2342     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2343     File::Path::make_path($rootdir);
2344
2345     my $volid_list = get_vm_volumes($conf);
2346     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2347
2348     eval {
2349         foreach_mountpoint($conf, sub {
2350             my ($ms, $mountpoint) = @_;
2351
2352             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2353         });
2354     };
2355     if (my $err = $@) {
2356         warn "mounting container failed\n";
2357         umount_all($vmid, $storage_cfg, $conf, 1);
2358         die $err;
2359     }
2360
2361     return $rootdir;
2362 }
2363
2364
2365 sub mountpoint_mount_path {
2366     my ($mountpoint, $storage_cfg, $snapname) = @_;
2367
2368     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2369 }
2370
2371 my $check_mount_path = sub {
2372     my ($path) = @_;
2373     $path = File::Spec->canonpath($path);
2374     my $real = Cwd::realpath($path);
2375     if ($real ne $path) {
2376         die "mount path modified by symlink: $path != $real";
2377     }
2378 };
2379
2380 sub query_loopdev {
2381     my ($path) = @_;
2382     my $found;
2383     my $parser = sub {
2384         my $line = shift;
2385         if ($line =~ m@^(/dev/loop\d+):@) {
2386             $found = $1;
2387         }
2388     };
2389     my $cmd = ['losetup', '--associated', $path];
2390     PVE::Tools::run_command($cmd, outfunc => $parser);
2391     return $found;
2392 }
2393
2394 # Run a function with a file attached to a loop device.
2395 # The loop device is always detached afterwards (or set to autoclear).
2396 # Returns the loop device.
2397 sub run_with_loopdev {
2398     my ($func, $file) = @_;
2399     my $device = query_loopdev($file);
2400     # Try to reuse an existing device
2401     if ($device) {
2402         # We assume that whoever setup the loop device is responsible for
2403         # detaching it.
2404         &$func($device);
2405         return $device;
2406     }
2407
2408     my $parser = sub {
2409         my $line = shift;
2410         if ($line =~ m@^(/dev/loop\d+)$@) {
2411             $device = $1;
2412         }
2413     };
2414     PVE::Tools::run_command(['losetup', '--show', '-f', $file], outfunc => $parser);
2415     die "failed to setup loop device for $file\n" if !$device;
2416     eval { &$func($device); };
2417     my $err = $@;
2418     PVE::Tools::run_command(['losetup', '-d', $device]);
2419     die $err if $err;
2420     return $device;
2421 }
2422
2423 sub bindmount {
2424     my ($dir, $dest, $ro, @extra_opts) = @_;
2425     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $dir, $dest]);
2426     if ($ro) {
2427         eval { PVE::Tools::run_command(['mount', '-o', 'bind,remount,ro', $dest]); };
2428         if (my $err = $@) {
2429             warn "bindmount error\n";
2430             # don't leave writable bind-mounts behind...
2431             PVE::Tools::run_command(['umount', $dest]);
2432             die $err;
2433         }
2434     }
2435 }
2436
2437 # use $rootdir = undef to just return the corresponding mount path
2438 sub mountpoint_mount {
2439     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2440
2441     my $volid = $mountpoint->{volume};
2442     my $mount = $mountpoint->{mp};
2443     my $type = $mountpoint->{type};
2444     my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota};
2445     my $mounted_dev;
2446
2447     return if !$volid || !$mount;
2448
2449     my $mount_path;
2450
2451     if (defined($rootdir)) {
2452         $rootdir =~ s!/+$!!;
2453         $mount_path = "$rootdir/$mount";
2454         $mount_path =~ s!/+!/!g;
2455         &$check_mount_path($mount_path);
2456         File::Path::mkpath($mount_path);
2457     }
2458
2459     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2460
2461     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2462
2463     my $optstring = '';
2464     if (defined($mountpoint->{acl})) {
2465         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
2466     }
2467     my $readonly = $mountpoint->{ro};
2468
2469     my @extra_opts = ('-o', $optstring);
2470
2471     if ($storage) {
2472
2473         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2474         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2475
2476         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2477             PVE::Storage::parse_volname($storage_cfg, $volid);
2478
2479         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2480
2481         if ($format eq 'subvol') {
2482             if ($mount_path) {
2483                 if ($snapname) {
2484                     if ($scfg->{type} eq 'zfspool') {
2485                         my $path_arg = $path;
2486                         $path_arg =~ s!^/+!!;
2487                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
2488                     } else {
2489                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2490                     }
2491                 } else {
2492                     bindmount($path, $mount_path, $readonly, @extra_opts);
2493                     warn "cannot enable quota control for bind mounted subvolumes\n" if $quota;
2494                 }
2495             }
2496             return wantarray ? ($path, 0, $mounted_dev) : $path;
2497         } elsif ($format eq 'raw' || $format eq 'iso') {
2498             my $domount = sub {
2499                 my ($path) = @_;
2500                 if ($mount_path) {
2501                     if ($format eq 'iso') {
2502                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2503                     } elsif ($isBase || defined($snapname)) {
2504                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2505                     } else {
2506                         if ($quota) {
2507                             push @extra_opts, '-o', 'usrjquota=aquota.user,grpjquota=aquota.group,jqfmt=vfsv0';
2508                         }
2509                         push @extra_opts, '-o', 'ro' if $readonly;
2510                         PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2511                     }
2512                 }
2513             };
2514             my $use_loopdev = 0;
2515             if ($scfg->{path}) {
2516                 $mounted_dev = run_with_loopdev($domount, $path);
2517                 $use_loopdev = 1;
2518             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2519                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2520                 $mounted_dev = $path;
2521                 &$domount($path);
2522             } else {
2523                 die "unsupported storage type '$scfg->{type}'\n";
2524             }
2525             return wantarray ? ($path, $use_loopdev, $mounted_dev) : $path;
2526         } else {
2527             die "unsupported image format '$format'\n";
2528         }
2529     } elsif ($type eq 'device') {
2530                         push @extra_opts, '-o', 'ro' if $readonly;
2531         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
2532         return wantarray ? ($volid, 0, $volid) : $volid;
2533     } elsif ($type eq 'bind') {
2534         die "directory '$volid' does not exist\n" if ! -d $volid;
2535         &$check_mount_path($volid);
2536         bindmount($volid, $mount_path, $readonly, @extra_opts) if $mount_path;
2537         warn "cannot enable quota control for bind mounts\n" if $quota;
2538         return wantarray ? ($volid, 0, undef) : $volid;
2539     }
2540
2541     die "unsupported storage";
2542 }
2543
2544 sub get_vm_volumes {
2545     my ($conf, $excludes) = @_;
2546
2547     my $vollist = [];
2548
2549     foreach_mountpoint($conf, sub {
2550         my ($ms, $mountpoint) = @_;
2551
2552         return if $excludes && $ms eq $excludes;
2553
2554         my $volid = $mountpoint->{volume};
2555
2556         return if !$volid || $mountpoint->{type} ne 'volume';
2557
2558         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2559         return if !$sid;
2560
2561         push @$vollist, $volid;
2562     });
2563
2564     return $vollist;
2565 }
2566
2567 sub mkfs {
2568     my ($dev, $rootuid, $rootgid) = @_;
2569
2570     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2571                              '-E', "root_owner=$rootuid:$rootgid",
2572                              $dev]);
2573 }
2574
2575 sub format_disk {
2576     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2577
2578     if ($volid =~ m!^/dev/.+!) {
2579         mkfs($volid);
2580         return;
2581     }
2582
2583     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2584
2585     die "cannot format volume '$volid' with no storage\n" if !$storage;
2586
2587     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2588
2589     my $path = PVE::Storage::path($storage_cfg, $volid);
2590
2591     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2592         PVE::Storage::parse_volname($storage_cfg, $volid);
2593
2594     die "cannot format volume '$volid' (format == $format)\n"
2595         if $format ne 'raw';
2596
2597     mkfs($path, $rootuid, $rootgid);
2598 }
2599
2600 sub destroy_disks {
2601     my ($storecfg, $vollist) = @_;
2602
2603     foreach my $volid (@$vollist) {
2604         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2605         warn $@ if $@;
2606     }
2607 }
2608
2609 sub create_disks {
2610     my ($storecfg, $vmid, $settings, $conf) = @_;
2611
2612     my $vollist = [];
2613
2614     eval {
2615         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2616         my $chown_vollist = [];
2617
2618         foreach_mountpoint($settings, sub {
2619             my ($ms, $mountpoint) = @_;
2620
2621             my $volid = $mountpoint->{volume};
2622             my $mp = $mountpoint->{mp};
2623
2624             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2625
2626             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2627                 my ($storeid, $size_gb) = ($1, $2);
2628
2629                 my $size_kb = int(${size_gb}*1024) * 1024;
2630
2631                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2632                 # fixme: use better naming ct-$vmid-disk-X.raw?
2633
2634                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2635                     if ($size_kb > 0) {
2636                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2637                                                            undef, $size_kb);
2638                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2639                     } else {
2640                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2641                                                            undef, 0);
2642                         push @$chown_vollist, $volid;
2643                     }
2644                 } elsif ($scfg->{type} eq 'zfspool') {
2645
2646                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2647                                                        undef, $size_kb);
2648                     push @$chown_vollist, $volid;
2649                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2650
2651                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2652                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2653
2654                 } elsif ($scfg->{type} eq 'rbd') {
2655
2656                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2657                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2658                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2659                 } else {
2660                     die "unable to create containers on storage type '$scfg->{type}'\n";
2661                 }
2662                 push @$vollist, $volid;
2663                 $mountpoint->{volume} = $volid;
2664                 $mountpoint->{size} = $size_kb * 1024;
2665                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2666             } else {
2667                 # use specified/existing volid/dir/device
2668                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2669             }
2670         });
2671
2672         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2673         foreach my $volid (@$chown_vollist) {
2674             my $path = PVE::Storage::path($storecfg, $volid, undef);
2675             chown($rootuid, $rootgid, $path);
2676         }
2677         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2678     };
2679     # free allocated images on error
2680     if (my $err = $@) {
2681         destroy_disks($storecfg, $vollist);
2682         die $err;
2683     }
2684     return $vollist;
2685 }
2686
2687 # bash completion helper
2688
2689 sub complete_os_templates {
2690     my ($cmdname, $pname, $cvalue) = @_;
2691
2692     my $cfg = PVE::Storage::config();
2693
2694     my $storeid;
2695
2696     if ($cvalue =~ m/^([^:]+):/) {
2697         $storeid = $1;
2698     }
2699
2700     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2701     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2702
2703     my $res = [];
2704     foreach my $id (keys %$data) {
2705         foreach my $item (@{$data->{$id}}) {
2706             push @$res, $item->{volid} if defined($item->{volid});
2707         }
2708     }
2709
2710     return $res;
2711 }
2712
2713 my $complete_ctid_full = sub {
2714     my ($running) = @_;
2715
2716     my $idlist = vmstatus();
2717
2718     my $active_hash = list_active_containers();
2719
2720     my $res = [];
2721
2722     foreach my $id (keys %$idlist) {
2723         my $d = $idlist->{$id};
2724         if (defined($running)) {
2725             next if $d->{template};
2726             next if $running && !$active_hash->{$id};
2727             next if !$running && $active_hash->{$id};
2728         }
2729         push @$res, $id;
2730
2731     }
2732     return $res;
2733 };
2734
2735 sub complete_ctid {
2736     return &$complete_ctid_full();
2737 }
2738
2739 sub complete_ctid_stopped {
2740     return &$complete_ctid_full(0);
2741 }
2742
2743 sub complete_ctid_running {
2744     return &$complete_ctid_full(1);
2745 }
2746
2747 sub parse_id_maps {
2748     my ($conf) = @_;
2749
2750     my $id_map = [];
2751     my $rootuid = 0;
2752     my $rootgid = 0;
2753
2754     my $lxc = $conf->{lxc};
2755     foreach my $entry (@$lxc) {
2756         my ($key, $value) = @$entry;
2757         next if $key ne 'lxc.id_map';
2758         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2759             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2760             push @$id_map, [$type, $ct, $host, $length];
2761             if ($ct == 0) {
2762                 $rootuid = $host if $type eq 'u';
2763                 $rootgid = $host if $type eq 'g';
2764             }
2765         } else {
2766             die "failed to parse id_map: $value\n";
2767         }
2768     }
2769
2770     if (!@$id_map && $conf->{unprivileged}) {
2771         # Should we read them from /etc/subuid?
2772         $id_map = [ ['u', '0', '100000', '65536'],
2773                     ['g', '0', '100000', '65536'] ];
2774         $rootuid = $rootgid = 100000;
2775     }
2776
2777     return ($id_map, $rootuid, $rootgid);
2778 }
2779
2780 sub userns_command {
2781     my ($id_map) = @_;
2782     if (@$id_map) {
2783         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2784     }
2785     return [];
2786 }
2787
2788 sub set_lock {
2789     my ($vmid, $lock) = @_;
2790     my $conf;
2791     lock_config($vmid, sub {
2792         $conf = load_config($vmid);
2793         check_lock($conf);
2794         $conf->{lock} = $lock;
2795         write_config($vmid, $conf);
2796     });
2797     return $conf;
2798 }
2799
2800 sub remove_lock {
2801     my ($vmid, $lock) = @_;
2802     lock_config($vmid, sub {
2803         my $conf = load_config($vmid);
2804         if (!$conf->{lock}) {
2805             die "no lock found trying to remove lock '$lock'\n";
2806         } elsif (defined($lock) && $conf->{lock} ne $lock) {
2807             die "found lock '$conf->{lock}' trying to remove lock '$lock'\n";
2808         }
2809         delete $conf->{lock};
2810         write_config($vmid, $conf);
2811     });
2812 }
2813
2814 1;