src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use Socket;
   8
   9 use File::Path;
  10 use File::Spec;
  11 use Cwd qw();
  12 use Fcntl qw(O_RDONLY);
  13
  14 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  15 use PVE::Exception qw(raise_perm_exc);
  16 use PVE::Storage;
  17 use PVE::SafeSyslog;
  18 use PVE::INotify;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach lock_file lock_file_full);
  21 use PVE::Network;
  22 use PVE::AccessControl;
  23 use PVE::ProcFSTools;
  24 use Time::HiRes qw (gettimeofday);
  25
  26 use Data::Dumper;
  27
  28 my $nodename = PVE::INotify::nodename();
  29
  30 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  31
  32 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  33                           '--xattrs',
  34                           '--xattrs-include=user.*',
  35                           '--xattrs-include=security.capability',
  36                           '--warning=no-xattr-write' ];
  37
  38 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  39
  40 my $rootfs_desc = {
  41     volume => {
  42         type => 'string',
  43         default_key => 1,
  44         format => 'pve-lxc-mp-string',
  45         format_description => 'volume',
  46         description => 'Volume, device or directory to mount into the container.',
  47     },
  48     backup => {
  49         type => 'boolean',
  50         format_description => '[1|0]',
  51         description => 'Whether to include the mountpoint in backups.',
  52         optional => 1,
  53     },
  54     size => {
  55         type => 'string',
  56         format => 'disk-size',
  57         format_description => 'DiskSize',
  58         description => 'Volume size (read only value).',
  59         optional => 1,
  60     },
  61     acl => {
  62         type => 'boolean',
  63         format_description => 'acl',
  64         description => 'Explicitly enable or disable ACL support.',
  65         optional => 1,
  66     },
  67     ro => {
  68         type => 'boolean',
  69         format_description => 'ro',
  70         description => 'Read-only mountpoint (not supported with bind mounts)',
  71         optional => 1,
  72     },
  73     quota => {
  74         type => 'boolean',
  75         format_description => '[0|1]',
  76         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
  77         optional => 1,
  78     },
  79 };
  80
  81 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  82     type => 'string', format => $rootfs_desc,
  83     description => "Use volume as container root.",
  84     optional => 1,
  85 });
  86
  87 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  88     description => "The name of the snapshot.",
  89     type => 'string', format => 'pve-configid',
  90     maxLength => 40,
  91 });
  92
  93 my $confdesc = {
  94     lock => {
  95         optional => 1,
  96         type => 'string',
  97         description => "Lock/unlock the VM.",
  98         enum => [qw(migrate backup snapshot rollback)],
  99     },
 100     onboot => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Specifies whether a VM will be started during system bootup.",
 104         default => 0,
 105     },
 106     startup => get_standard_option('pve-startup-order'),
 107     template => {
 108         optional => 1,
 109         type => 'boolean',
 110         description => "Enable/disable Template.",
 111         default => 0,
 112     },
 113     arch => {
 114         optional => 1,
 115         type => 'string',
 116         enum => ['amd64', 'i386'],
 117         description => "OS architecture type.",
 118         default => 'amd64',
 119     },
 120     ostype => {
 121         optional => 1,
 122         type => 'string',
 123         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux', 'alpine', 'unmanaged'],
 124         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 125     },
 126     console => {
 127         optional => 1,
 128         type => 'boolean',
 129         description => "Attach a console device (/dev/console) to the container.",
 130         default => 1,
 131     },
 132     tty => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "Specify the number of tty available to the container",
 136         minimum => 0,
 137         maximum => 6,
 138         default => 2,
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1024,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     swap => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of SWAP for the VM in MB.",
 167         minimum => 0,
 168         default => 512,
 169     },
 170     hostname => {
 171         optional => 1,
 172         description => "Set a host name for the container.",
 173         type => 'string', format => 'dns-name',
 174         maxLength => 255,
 175     },
 176     description => {
 177         optional => 1,
 178         type => 'string',
 179         description => "Container description. Only used on the configuration web interface.",
 180     },
 181     searchdomain => {
 182         optional => 1,
 183         type => 'string', format => 'dns-name-list',
 184         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 185     },
 186     nameserver => {
 187         optional => 1,
 188         type => 'string', format => 'address-list',
 189         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 190     },
 191     rootfs => get_standard_option('pve-ct-rootfs'),
 192     parent => {
 193         optional => 1,
 194         type => 'string', format => 'pve-configid',
 195         maxLength => 40,
 196         description => "Parent snapshot name. This is used internally, and should not be modified.",
 197     },
 198     snaptime => {
 199         optional => 1,
 200         description => "Timestamp for snapshots.",
 201         type => 'integer',
 202         minimum => 0,
 203     },
 204     cmode => {
 205         optional => 1,
 206         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 207         type => 'string',
 208         enum => ['shell', 'console', 'tty'],
 209         default => 'tty',
 210     },
 211     protection => {
 212         optional => 1,
 213         type => 'boolean',
 214         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 215         default => 0,
 216     },
 217     unprivileged => {
 218         optional => 1,
 219         type => 'boolean',
 220         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 221         default => 0,
 222     },
 223 };
 224
 225 my $valid_lxc_conf_keys = {
 226     'lxc.include' => 1,
 227     'lxc.arch' => 1,
 228     'lxc.utsname' => 1,
 229     'lxc.haltsignal' => 1,
 230     'lxc.rebootsignal' => 1,
 231     'lxc.stopsignal' => 1,
 232     'lxc.init_cmd' => 1,
 233     'lxc.network.type' => 1,
 234     'lxc.network.flags' => 1,
 235     'lxc.network.link' => 1,
 236     'lxc.network.mtu' => 1,
 237     'lxc.network.name' => 1,
 238     'lxc.network.hwaddr' => 1,
 239     'lxc.network.ipv4' => 1,
 240     'lxc.network.ipv4.gateway' => 1,
 241     'lxc.network.ipv6' => 1,
 242     'lxc.network.ipv6.gateway' => 1,
 243     'lxc.network.script.up' => 1,
 244     'lxc.network.script.down' => 1,
 245     'lxc.pts' => 1,
 246     'lxc.console.logfile' => 1,
 247     'lxc.console' => 1,
 248     'lxc.tty' => 1,
 249     'lxc.devttydir' => 1,
 250     'lxc.hook.autodev' => 1,
 251     'lxc.autodev' => 1,
 252     'lxc.kmsg' => 1,
 253     'lxc.mount' => 1,
 254     'lxc.mount.entry' => 1,
 255     'lxc.mount.auto' => 1,
 256     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 257     'lxc.rootfs.mount' => 1,
 258     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 259                             ', please use mountpoint options in the "rootfs" key',
 260     # lxc.cgroup.*
 261     'lxc.cap.drop' => 1,
 262     'lxc.cap.keep' => 1,
 263     'lxc.aa_profile' => 1,
 264     'lxc.aa_allow_incomplete' => 1,
 265     'lxc.se_context' => 1,
 266     'lxc.seccomp' => 1,
 267     'lxc.id_map' => 1,
 268     'lxc.hook.pre-start' => 1,
 269     'lxc.hook.pre-mount' => 1,
 270     'lxc.hook.mount' => 1,
 271     'lxc.hook.start' => 1,
 272     'lxc.hook.stop' => 1,
 273     'lxc.hook.post-stop' => 1,
 274     'lxc.hook.clone' => 1,
 275     'lxc.hook.destroy' => 1,
 276     'lxc.loglevel' => 1,
 277     'lxc.logfile' => 1,
 278     'lxc.start.auto' => 1,
 279     'lxc.start.delay' => 1,
 280     'lxc.start.order' => 1,
 281     'lxc.group' => 1,
 282     'lxc.environment' => 1,
 283 };
 284
 285 my $netconf_desc = {
 286     type => {
 287         type => 'string',
 288         optional => 1,
 289         description => "Network interface type.",
 290         enum => [qw(veth)],
 291     },
 292     name => {
 293         type => 'string',
 294         format_description => 'String',
 295         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 296         pattern => '[-_.\w\d]+',
 297     },
 298     bridge => {
 299         type => 'string',
 300         format_description => 'vmbr<Number>',
 301         description => 'Bridge to attach the network device to.',
 302         pattern => '[-_.\w\d]+',
 303         optional => 1,
 304     },
 305     hwaddr => {
 306         type => 'string',
 307         format_description => 'MAC',
 308         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 309         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 310         optional => 1,
 311     },
 312     mtu => {
 313         type => 'integer',
 314         format_description => 'Number',
 315         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 316         minimum => 64, # minimum ethernet frame is 64 bytes
 317         optional => 1,
 318     },
 319     ip => {
 320         type => 'string',
 321         format => 'pve-ipv4-config',
 322         format_description => 'IPv4Format/CIDR',
 323         description => 'IPv4 address in CIDR format.',
 324         optional => 1,
 325     },
 326     gw => {
 327         type => 'string',
 328         format => 'ipv4',
 329         format_description => 'GatewayIPv4',
 330         description => 'Default gateway for IPv4 traffic.',
 331         optional => 1,
 332     },
 333     ip6 => {
 334         type => 'string',
 335         format => 'pve-ipv6-config',
 336         format_description => 'IPv6Format/CIDR',
 337         description => 'IPv6 address in CIDR format.',
 338         optional => 1,
 339     },
 340     gw6 => {
 341         type => 'string',
 342         format => 'ipv6',
 343         format_description => 'GatewayIPv6',
 344         description => 'Default gateway for IPv6 traffic.',
 345         optional => 1,
 346     },
 347     firewall => {
 348         type => 'boolean',
 349         format_description => '[1|0]',
 350         description => "Controls whether this interface's firewall rules should be used.",
 351         optional => 1,
 352     },
 353     tag => {
 354         type => 'integer',
 355         format_description => 'VlanNo',
 356         minimum => '2',
 357         maximum => '4094',
 358         description => "VLAN tag for this interface.",
 359         optional => 1,
 360     },
 361     trunks => {
 362         type => 'string',
 363         pattern => qr/\d+(?:;\d+)*/,
 364         format_description => 'vlanid[;vlanid...]',
 365         description => "VLAN ids to pass through the interface",
 366         optional => 1,
 367     },
 368 };
 369 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 370
 371 my $MAX_LXC_NETWORKS = 10;
 372 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 373     $confdesc->{"net$i"} = {
 374         optional => 1,
 375         type => 'string', format => $netconf_desc,
 376         description => "Specifies network interfaces for the container.",
 377     };
 378 }
 379
 380 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 381 sub verify_lxc_mp_string{
 382     my ($mp, $noerr) = @_;
 383
 384     # do not allow:
 385     # /./ or /../
 386     # /. or /.. at the end
 387     # ../ at the beginning
 388
 389     if($mp =~ m@/\.\.?/@ ||
 390        $mp =~ m@/\.\.?$@ ||
 391        $mp =~ m@^\.\./@){
 392         return undef if $noerr;
 393         die "$mp contains illegal character sequences\n";
 394     }
 395     return $mp;
 396 }
 397
 398 my $mp_desc = {
 399     %$rootfs_desc,
 400     mp => {
 401         type => 'string',
 402         format => 'pve-lxc-mp-string',
 403         format_description => 'Path',
 404         description => 'Path to the mountpoint as seen from inside the container.',
 405     },
 406 };
 407 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 408
 409 my $unuseddesc = {
 410     optional => 1,
 411     type => 'string', format => 'pve-volume-id',
 412     description => "Reference to unused volumes.",
 413 };
 414
 415 my $MAX_MOUNT_POINTS = 10;
 416 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 417     $confdesc->{"mp$i"} = {
 418         optional => 1,
 419         type => 'string', format => $mp_desc,
 420         description => "Use volume as container mount point (experimental feature).",
 421         optional => 1,
 422     };
 423 }
 424
 425 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 426 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 427     $confdesc->{"unused$i"} = $unuseddesc;
 428 }
 429
 430 sub write_pct_config {
 431     my ($filename, $conf) = @_;
 432
 433     delete $conf->{snapstate}; # just to be sure
 434     my $volidlist = get_vm_volumes($conf);
 435     my $used_volids = {};
 436     foreach my $vid (@$volidlist) {
 437         $used_volids->{$vid} = 1;
 438     }
 439
 440     # remove 'unusedX' settings if the volume is still used
 441     foreach my $key (keys %$conf) {
 442         my $value = $conf->{$key};
 443         if ($key =~ m/^unused/ && $used_volids->{$value}) {
 444             delete $conf->{$key};
 445         }
 446     }
 447
 448     my $generate_raw_config = sub {
 449         my ($conf) = @_;
 450
 451         my $raw = '';
 452
 453         # add description as comment to top of file
 454         my $descr = $conf->{description} || '';
 455         foreach my $cl (split(/\n/, $descr)) {
 456             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 457         }
 458
 459         foreach my $key (sort keys %$conf) {
 460             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 461                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 462             my $value = $conf->{$key};
 463             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 464             $raw .= "$key: $value\n";
 465         }
 466
 467         if (my $lxcconf = $conf->{lxc}) {
 468             foreach my $entry (@$lxcconf) {
 469                 my ($k, $v) = @$entry;
 470                 $raw .= "$k: $v\n";
 471             }
 472         }
 473
 474         return $raw;
 475     };
 476
 477     my $raw = &$generate_raw_config($conf);
 478
 479     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 480         $raw .= "\n[$snapname]\n";
 481         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 482     }
 483
 484     return $raw;
 485 }
 486
 487 sub check_type {
 488     my ($key, $value) = @_;
 489
 490     die "unknown setting '$key'\n" if !$confdesc->{$key};
 491
 492     my $type = $confdesc->{$key}->{type};
 493
 494     if (!defined($value)) {
 495         die "got undefined value\n";
 496     }
 497
 498     if ($value =~ m/[\n\r]/) {
 499         die "property contains a line feed\n";
 500     }
 501
 502     if ($type eq 'boolean') {
 503         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 504         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 505         die "type check ('boolean') failed - got '$value'\n";
 506     } elsif ($type eq 'integer') {
 507         return int($1) if $value =~ m/^(\d+)$/;
 508         die "type check ('integer') failed - got '$value'\n";
 509     } elsif ($type eq 'number') {
 510         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 511         die "type check ('number') failed - got '$value'\n";
 512     } elsif ($type eq 'string') {
 513         if (my $fmt = $confdesc->{$key}->{format}) {
 514             PVE::JSONSchema::check_format($fmt, $value);
 515             return $value;
 516         }
 517         return $value;
 518     } else {
 519         die "internal error"
 520     }
 521 }
 522
 523 sub parse_pct_config {
 524     my ($filename, $raw) = @_;
 525
 526     return undef if !defined($raw);
 527
 528     my $res = {
 529         digest => Digest::SHA::sha1_hex($raw),
 530         snapshots => {},
 531     };
 532
 533     $filename =~ m|/lxc/(\d+).conf$|
 534         || die "got strange filename '$filename'";
 535
 536     my $vmid = $1;
 537
 538     my $conf = $res;
 539     my $descr = '';
 540     my $section = '';
 541
 542     my @lines = split(/\n/, $raw);
 543     foreach my $line (@lines) {
 544         next if $line =~ m/^\s*$/;
 545
 546         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 547             $section = $1;
 548             $conf->{description} = $descr if $descr;
 549             $descr = '';
 550             $conf = $res->{snapshots}->{$section} = {};
 551             next;
 552         }
 553
 554         if ($line =~ m/^\#(.*)\s*$/) {
 555             $descr .= PVE::Tools::decode_text($1) . "\n";
 556             next;
 557         }
 558
 559         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 560             my $key = $1;
 561             my $value = $3;
 562             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 563             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 564                 push @{$conf->{lxc}}, [$key, $value];
 565             } elsif (my $errmsg = $validity) {
 566                 warn "vm $vmid - $key: $errmsg\n";
 567             } else {
 568                 warn "vm $vmid - unable to parse config: $line\n";
 569             }
 570         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 571             $descr .= PVE::Tools::decode_text($2);
 572         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 573             $conf->{snapstate} = $1;
 574         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 575             my $key = $1;
 576             my $value = $2;
 577             eval { $value = check_type($key, $value); };
 578             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 579             $conf->{$key} = $value;
 580         } else {
 581             warn "vm $vmid - unable to parse config: $line\n";
 582         }
 583     }
 584
 585     $conf->{description} = $descr if $descr;
 586
 587     delete $res->{snapstate}; # just to be sure
 588
 589     return $res;
 590 }
 591
 592 sub config_list {
 593     my $vmlist = PVE::Cluster::get_vmlist();
 594     my $res = {};
 595     return $res if !$vmlist || !$vmlist->{ids};
 596     my $ids = $vmlist->{ids};
 597
 598     foreach my $vmid (keys %$ids) {
 599         next if !$vmid; # skip CT0
 600         my $d = $ids->{$vmid};
 601         next if !$d->{node} || $d->{node} ne $nodename;
 602         next if !$d->{type} || $d->{type} ne 'lxc';
 603         $res->{$vmid}->{type} = 'lxc';
 604     }
 605     return $res;
 606 }
 607
 608 sub cfs_config_path {
 609     my ($vmid, $node) = @_;
 610
 611     $node = $nodename if !$node;
 612     return "nodes/$node/lxc/$vmid.conf";
 613 }
 614
 615 sub config_file {
 616     my ($vmid, $node) = @_;
 617
 618     my $cfspath = cfs_config_path($vmid, $node);
 619     return "/etc/pve/$cfspath";
 620 }
 621
 622 sub load_config {
 623     my ($vmid, $node) = @_;
 624
 625     $node = $nodename if !$node;
 626     my $cfspath = cfs_config_path($vmid, $node);
 627
 628     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 629     die "container $vmid does not exist\n" if !defined($conf);
 630
 631     return $conf;
 632 }
 633
 634 sub create_config {
 635     my ($vmid, $conf) = @_;
 636
 637     my $dir = "/etc/pve/nodes/$nodename/lxc";
 638     mkdir $dir;
 639
 640     write_config($vmid, $conf);
 641 }
 642
 643 sub destroy_config {
 644     my ($vmid) = @_;
 645
 646     unlink config_file($vmid, $nodename);
 647 }
 648
 649 sub write_config {
 650     my ($vmid, $conf) = @_;
 651
 652     my $cfspath = cfs_config_path($vmid);
 653
 654     PVE::Cluster::cfs_write_file($cfspath, $conf);
 655 }
 656
 657 # flock: we use one file handle per process, so lock file
 658 # can be called multiple times and will succeed for the same process.
 659
 660 my $lock_handles =  {};
 661 my $lockdir = "/run/lock/lxc";
 662
 663 sub config_file_lock {
 664     my ($vmid) = @_;
 665
 666     return "$lockdir/pve-config-${vmid}.lock";
 667 }
 668
 669 sub lock_config_full {
 670     my ($vmid, $timeout, $code, @param) = @_;
 671
 672     my $filename = config_file_lock($vmid);
 673
 674     mkdir $lockdir if !-d $lockdir;
 675
 676     my $res = lock_file($filename, $timeout, $code, @param);
 677
 678     die $@ if $@;
 679
 680     return $res;
 681 }
 682
 683 sub lock_config_mode {
 684     my ($vmid, $timeout, $shared, $code, @param) = @_;
 685
 686     my $filename = config_file_lock($vmid);
 687
 688     mkdir $lockdir if !-d $lockdir;
 689
 690     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
 691
 692     die $@ if $@;
 693
 694     return $res;
 695 }
 696
 697 sub lock_config {
 698     my ($vmid, $code, @param) = @_;
 699
 700     return lock_config_full($vmid, 10, $code, @param);
 701 }
 702
 703 sub option_exists {
 704     my ($name) = @_;
 705
 706     return defined($confdesc->{$name});
 707 }
 708
 709 # add JSON properties for create and set function
 710 sub json_config_properties {
 711     my $prop = shift;
 712
 713     foreach my $opt (keys %$confdesc) {
 714         next if $opt eq 'parent' || $opt eq 'snaptime';
 715         next if $prop->{$opt};
 716         $prop->{$opt} = $confdesc->{$opt};
 717     }
 718
 719     return $prop;
 720 }
 721
 722 # container status helpers
 723
 724 sub list_active_containers {
 725
 726     my $filename = "/proc/net/unix";
 727
 728     # similar test is used by lcxcontainers.c: list_active_containers
 729     my $res = {};
 730
 731     my $fh = IO::File->new ($filename, "r");
 732     return $res if !$fh;
 733
 734     while (defined(my $line = <$fh>)) {
 735         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 736             my $path = $1;
 737             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 738                 $res->{$1} = 1;
 739             }
 740         }
 741     }
 742
 743     close($fh);
 744
 745     return $res;
 746 }
 747
 748 # warning: this is slow
 749 sub check_running {
 750     my ($vmid) = @_;
 751
 752     my $active_hash = list_active_containers();
 753
 754     return 1 if defined($active_hash->{$vmid});
 755
 756     return undef;
 757 }
 758
 759 sub get_container_disk_usage {
 760     my ($vmid, $pid) = @_;
 761
 762     return PVE::Tools::df("/proc/$pid/root/", 1);
 763 }
 764
 765 my $last_proc_vmid_stat;
 766
 767 my $parse_cpuacct_stat = sub {
 768     my ($vmid) = @_;
 769
 770     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 771
 772     my $stat = {};
 773
 774     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 775
 776         $stat->{utime} = $1;
 777         $stat->{stime} = $2;
 778
 779     }
 780
 781     return $stat;
 782 };
 783
 784 sub vmstatus {
 785     my ($opt_vmid) = @_;
 786
 787     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 788
 789     my $active_hash = list_active_containers();
 790
 791     my $cpucount = $cpuinfo->{cpus} || 1;
 792
 793     my $cdtime = gettimeofday;
 794
 795     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 796
 797     foreach my $vmid (keys %$list) {
 798         my $d = $list->{$vmid};
 799
 800         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 801         warn $@ if $@; # ignore errors (consider them stopped)
 802
 803         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 804
 805         my $cfspath = cfs_config_path($vmid);
 806         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 807
 808         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 809         $d->{name} =~ s/[\s]//g;
 810
 811         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 812
 813         if ($d->{pid}) {
 814             my $res = get_container_disk_usage($vmid, $d->{pid});
 815             $d->{disk} = $res->{used};
 816             $d->{maxdisk} = $res->{total};
 817         } else {
 818             $d->{disk} = 0;
 819             # use 4GB by default ??
 820             if (my $rootfs = $conf->{rootfs}) {
 821                 my $rootinfo = parse_ct_rootfs($rootfs);
 822                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 823             } else {
 824                 $d->{maxdisk} = 4*1024*1024*1024;
 825             }
 826         }
 827
 828         $d->{mem} = 0;
 829         $d->{swap} = 0;
 830         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 831         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 832
 833         $d->{uptime} = 0;
 834         $d->{cpu} = 0;
 835
 836         $d->{netout} = 0;
 837         $d->{netin} = 0;
 838
 839         $d->{diskread} = 0;
 840         $d->{diskwrite} = 0;
 841
 842         $d->{template} = is_template($conf);
 843     }
 844
 845     foreach my $vmid (keys %$list) {
 846         my $d = $list->{$vmid};
 847         my $pid = $d->{pid};
 848
 849         next if !$pid; # skip stopped CTs
 850
 851         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 852         $d->{uptime} = time - $ctime; # the method lxcfs uses
 853
 854         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 855         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 856
 857         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 858         my @bytes = split(/\n/, $blkio_bytes);
 859         foreach my $byte (@bytes) {
 860             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 861                 $d->{diskread} = $2 if $key eq 'Read';
 862                 $d->{diskwrite} = $2 if $key eq 'Write';
 863             }
 864         }
 865
 866         my $pstat = &$parse_cpuacct_stat($vmid);
 867
 868         my $used = $pstat->{utime} + $pstat->{stime};
 869
 870         my $old = $last_proc_vmid_stat->{$vmid};
 871         if (!$old) {
 872             $last_proc_vmid_stat->{$vmid} = {
 873                 time => $cdtime,
 874                 used => $used,
 875                 cpu => 0,
 876             };
 877             next;
 878         }
 879
 880         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 881
 882         if ($dtime > 1000) {
 883             my $dutime = $used -  $old->{used};
 884
 885             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 886             $last_proc_vmid_stat->{$vmid} = {
 887                 time => $cdtime,
 888                 used => $used,
 889                 cpu => $d->{cpu},
 890             };
 891         } else {
 892             $d->{cpu} = $old->{cpu};
 893         }
 894     }
 895
 896     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 897
 898     foreach my $dev (keys %$netdev) {
 899         next if $dev !~ m/^veth([1-9]\d*)i/;
 900         my $vmid = $1;
 901         my $d = $list->{$vmid};
 902
 903         next if !$d;
 904
 905         $d->{netout} += $netdev->{$dev}->{receive};
 906         $d->{netin} += $netdev->{$dev}->{transmit};
 907
 908     }
 909
 910     return $list;
 911 }
 912
 913 sub classify_mountpoint {
 914     my ($vol) = @_;
 915     if ($vol =~ m!^/!) {
 916         return 'device' if $vol =~ m!^/dev/!;
 917         return 'bind';
 918     }
 919     return 'volume';
 920 }
 921
 922 my $parse_ct_mountpoint_full = sub {
 923     my ($desc, $data, $noerr) = @_;
 924
 925     $data //= '';
 926
 927     my $res;
 928     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 929     if ($@) {
 930         return undef if $noerr;
 931         die $@;
 932     }
 933
 934     if (defined(my $size = $res->{size})) {
 935         $size = PVE::JSONSchema::parse_size($size);
 936         if (!defined($size)) {
 937             return undef if $noerr;
 938             die "invalid size: $size\n";
 939         }
 940         $res->{size} = $size;
 941     }
 942
 943     $res->{type} = classify_mountpoint($res->{volume});
 944
 945     return $res;
 946 };
 947
 948 sub parse_ct_rootfs {
 949     my ($data, $noerr) = @_;
 950
 951     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 952
 953     $res->{mp} = '/' if defined($res);
 954
 955     return $res;
 956 }
 957
 958 sub parse_ct_mountpoint {
 959     my ($data, $noerr) = @_;
 960
 961     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 962 }
 963
 964 sub print_ct_mountpoint {
 965     my ($info, $nomp) = @_;
 966     my $skip = [ 'type' ];
 967     push @$skip, 'mp' if $nomp;
 968     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 969 }
 970
 971 sub print_lxc_network {
 972     my $net = shift;
 973     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 974 }
 975
 976 sub parse_lxc_network {
 977     my ($data) = @_;
 978
 979     my $res = {};
 980
 981     return $res if !$data;
 982
 983     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 984
 985     $res->{type} = 'veth';
 986     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 987
 988     return $res;
 989 }
 990
 991 sub read_cgroup_value {
 992     my ($group, $vmid, $name, $full) = @_;
 993
 994     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 995
 996     return PVE::Tools::file_get_contents($path) if $full;
 997
 998     return PVE::Tools::file_read_firstline($path);
 999 }
1000
1001 sub write_cgroup_value {
1002    my ($group, $vmid, $name, $value) = @_;
1003
1004    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
1005    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
1006
1007 }
1008
1009 sub find_lxc_console_pids {
1010
1011     my $res = {};
1012
1013     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
1014         my ($pid) = @_;
1015
1016         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
1017         return if !$cmdline;
1018
1019         my @args = split(/\0/, $cmdline);
1020
1021         # search for lxc-console -n <vmid>
1022         return if scalar(@args) != 3;
1023         return if $args[1] ne '-n';
1024         return if $args[2] !~ m/^\d+$/;
1025         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1026
1027         my $vmid = $args[2];
1028
1029         push @{$res->{$vmid}}, $pid;
1030     });
1031
1032     return $res;
1033 }
1034
1035 sub find_lxc_pid {
1036     my ($vmid) = @_;
1037
1038     my $pid = undef;
1039     my $parser = sub {
1040         my $line = shift;
1041         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1042     };
1043     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1044
1045     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1046
1047     return $pid;
1048 }
1049
1050 # Note: we cannot use Net:IP, because that only allows strict
1051 # CIDR networks
1052 sub parse_ipv4_cidr {
1053     my ($cidr, $noerr) = @_;
1054
1055     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1056         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1057     }
1058
1059     return undef if $noerr;
1060
1061     die "unable to parse ipv4 address/mask\n";
1062 }
1063
1064 sub check_lock {
1065     my ($conf) = @_;
1066
1067     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1068 }
1069
1070 sub has_lock {
1071     my ($conf, $lock) = @_;
1072     return $conf->{lock} && (!defined($lock) || $lock eq $conf->{lock});
1073 }
1074
1075 sub check_protection {
1076     my ($vm_conf, $err_msg) = @_;
1077
1078     if ($vm_conf->{protection}) {
1079         die "$err_msg - protection mode enabled\n";
1080     }
1081 }
1082
1083 sub update_lxc_config {
1084     my ($storage_cfg, $vmid, $conf) = @_;
1085
1086     my $dir = "/var/lib/lxc/$vmid";
1087
1088     if ($conf->{template}) {
1089
1090         unlink "$dir/config";
1091
1092         return;
1093     }
1094
1095     my $raw = '';
1096
1097     die "missing 'arch' - internal error" if !$conf->{arch};
1098     $raw .= "lxc.arch = $conf->{arch}\n";
1099
1100     my $unprivileged = $conf->{unprivileged};
1101     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1102
1103     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1104     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux | alpine | unmanaged)$/x) {
1105         my $inc ="/usr/share/lxc/config/$ostype.common.conf";
1106         $inc ="/usr/share/lxc/config/common.conf" if !-f $inc;
1107         $raw .= "lxc.include = $inc\n";
1108         if ($unprivileged || $custom_idmap) {
1109             $inc = "/usr/share/lxc/config/$ostype.userns.conf";
1110             $inc = "/usr/share/lxc/config/userns.conf" if !-f $inc;
1111             $raw .= "lxc.include = $inc\n"
1112         }
1113     } else {
1114         die "implement me (ostype $ostype)";
1115     }
1116
1117     # WARNING: DO NOT REMOVE this without making sure that loop device nodes
1118     # cannot be exposed to the container with r/w access (cgroup perms).
1119     # When this is enabled mounts will still remain in the monitor's namespace
1120     # after the container unmounted them and thus will not detach from their
1121     # files while the container is running!
1122     $raw .= "lxc.monitor.unshare = 1\n";
1123
1124     # Should we read them from /etc/subuid?
1125     if ($unprivileged && !$custom_idmap) {
1126         $raw .= "lxc.id_map = u 0 100000 65536\n";
1127         $raw .= "lxc.id_map = g 0 100000 65536\n";
1128     }
1129
1130     if (!has_dev_console($conf)) {
1131         $raw .= "lxc.console = none\n";
1132         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1133     }
1134
1135     my $ttycount = get_tty_count($conf);
1136     $raw .= "lxc.tty = $ttycount\n";
1137
1138     # some init scripts expect a linux terminal (turnkey).
1139     $raw .= "lxc.environment = TERM=linux\n";
1140
1141     my $utsname = $conf->{hostname} || "CT$vmid";
1142     $raw .= "lxc.utsname = $utsname\n";
1143
1144     my $memory = $conf->{memory} || 512;
1145     my $swap = $conf->{swap} // 0;
1146
1147     my $lxcmem = int($memory*1024*1024);
1148     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1149
1150     my $lxcswap = int(($memory + $swap)*1024*1024);
1151     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1152
1153     if (my $cpulimit = $conf->{cpulimit}) {
1154         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1155         my $value = int(100000*$cpulimit);
1156         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1157     }
1158
1159     my $shares = $conf->{cpuunits} || 1024;
1160     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1161
1162     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1163
1164     $raw .= "lxc.rootfs = $dir/rootfs\n";
1165
1166     my $netcount = 0;
1167     foreach my $k (keys %$conf) {
1168         next if $k !~ m/^net(\d+)$/;
1169         my $ind = $1;
1170         my $d = parse_lxc_network($conf->{$k});
1171         $netcount++;
1172         $raw .= "lxc.network.type = veth\n";
1173         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1174         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1175         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1176         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1177     }
1178
1179     if (my $lxcconf = $conf->{lxc}) {
1180         foreach my $entry (@$lxcconf) {
1181             my ($k, $v) = @$entry;
1182             $netcount++ if $k eq 'lxc.network.type';
1183             $raw .= "$k = $v\n";
1184         }
1185     }
1186
1187     $raw .= "lxc.network.type = empty\n" if !$netcount;
1188
1189     File::Path::mkpath("$dir/rootfs");
1190
1191     PVE::Tools::file_set_contents("$dir/config", $raw);
1192 }
1193
1194 # verify and cleanup nameserver list (replace \0 with ' ')
1195 sub verify_nameserver_list {
1196     my ($nameserver_list) = @_;
1197
1198     my @list = ();
1199     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1200         PVE::JSONSchema::pve_verify_ip($server);
1201         push @list, $server;
1202     }
1203
1204     return join(' ', @list);
1205 }
1206
1207 sub verify_searchdomain_list {
1208     my ($searchdomain_list) = @_;
1209
1210     my @list = ();
1211     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1212         # todo: should we add checks for valid dns domains?
1213         push @list, $server;
1214     }
1215
1216     return join(' ', @list);
1217 }
1218
1219 sub is_volume_in_use {
1220     my ($config, $volid, $include_snapshots) = @_;
1221     my $used = 0;
1222
1223     foreach_mountpoint($config, sub {
1224         my ($ms, $mountpoint) = @_;
1225         return if $used;
1226         if ($mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid) {
1227             $used = 1;
1228         }
1229     });
1230
1231     my $snapshots = $config->{snapshots};
1232     if ($include_snapshots && $snapshots) {
1233         foreach my $snap (keys %$snapshots) {
1234             $used ||= is_volume_in_use($snapshots->{$snap}, $volid);
1235         }
1236     }
1237
1238     return $used;
1239 }
1240
1241 sub add_unused_volume {
1242     my ($config, $volid) = @_;
1243
1244     my $key;
1245     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1246         my $test = "unused$ind";
1247         if (my $vid = $config->{$test}) {
1248             return if $vid eq $volid; # do not add duplicates
1249         } else {
1250             $key = $test;
1251         }
1252     }
1253
1254     die "Too many unused volumes - please delete them first.\n" if !$key;
1255
1256     $config->{$key} = $volid;
1257
1258     return $key;
1259 }
1260
1261 sub update_pct_config {
1262     my ($vmid, $conf, $running, $param, $delete) = @_;
1263
1264     my @nohotplug;
1265
1266     my $new_disks = 0;
1267     my @deleted_volumes;
1268
1269     my $rootdir;
1270     if ($running) {
1271         my $pid = find_lxc_pid($vmid);
1272         $rootdir = "/proc/$pid/root";
1273     }
1274
1275     my $hotplug_error = sub {
1276         if ($running) {
1277             push @nohotplug, @_;
1278             return 1;
1279         } else {
1280             return 0;
1281         }
1282     };
1283
1284     if (defined($delete)) {
1285         foreach my $opt (@$delete) {
1286             if (!exists($conf->{$opt})) {
1287                 warn "no such option: $opt\n";
1288                 next;
1289             }
1290
1291             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1292                 die "unable to delete required option '$opt'\n";
1293             } elsif ($opt eq 'swap') {
1294                 delete $conf->{$opt};
1295                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1296             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1297                 delete $conf->{$opt};
1298             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1299                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1300                 next if $hotplug_error->($opt);
1301                 delete $conf->{$opt};
1302             } elsif ($opt =~ m/^net(\d)$/) {
1303                 delete $conf->{$opt};
1304                 next if !$running;
1305                 my $netid = $1;
1306                 PVE::Network::veth_delete("veth${vmid}i$netid");
1307             } elsif ($opt eq 'protection') {
1308                 delete $conf->{$opt};
1309             } elsif ($opt =~ m/^unused(\d+)$/) {
1310                 next if $hotplug_error->($opt);
1311                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1312                 push @deleted_volumes, $conf->{$opt};
1313                 delete $conf->{$opt};
1314             } elsif ($opt =~ m/^mp(\d+)$/) {
1315                 next if $hotplug_error->($opt);
1316                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1317                 my $mp = parse_ct_mountpoint($conf->{$opt});
1318                 delete $conf->{$opt};
1319                 if ($mp->{type} eq 'volume') {
1320                     add_unused_volume($conf, $mp->{volume});
1321                 }
1322             } elsif ($opt eq 'unprivileged') {
1323                 die "unable to delete read-only option: '$opt'\n";
1324             } else {
1325                 die "implement me (delete: $opt)"
1326             }
1327             write_config($vmid, $conf) if $running;
1328         }
1329     }
1330
1331     # There's no separate swap size to configure, there's memory and "total"
1332     # memory (iow. memory+swap). This means we have to change them together.
1333     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1334     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1335     if (defined($wanted_memory) || defined($wanted_swap)) {
1336
1337         my $old_memory = ($conf->{memory} || 512);
1338         my $old_swap = ($conf->{swap} || 0);
1339
1340         $wanted_memory //= $old_memory;
1341         $wanted_swap //= $old_swap;
1342
1343         my $total = $wanted_memory + $wanted_swap;
1344         if ($running) {
1345             my $old_total = $old_memory + $old_swap;
1346             if ($total > $old_total) {
1347                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1348                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1349             } else {
1350                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1351                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1352             }
1353         }
1354         $conf->{memory} = $wanted_memory;
1355         $conf->{swap} = $wanted_swap;
1356
1357         write_config($vmid, $conf) if $running;
1358     }
1359
1360     my $used_volids = {};
1361
1362     foreach my $opt (keys %$param) {
1363         my $value = $param->{$opt};
1364         if ($opt eq 'hostname') {
1365             $conf->{$opt} = $value;
1366         } elsif ($opt eq 'onboot') {
1367             $conf->{$opt} = $value ? 1 : 0;
1368         } elsif ($opt eq 'startup') {
1369             $conf->{$opt} = $value;
1370         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1371             next if $hotplug_error->($opt);
1372             $conf->{$opt} = $value;
1373         } elsif ($opt eq 'nameserver') {
1374             next if $hotplug_error->($opt);
1375             my $list = verify_nameserver_list($value);
1376             $conf->{$opt} = $list;
1377         } elsif ($opt eq 'searchdomain') {
1378             next if $hotplug_error->($opt);
1379             my $list = verify_searchdomain_list($value);
1380             $conf->{$opt} = $list;
1381         } elsif ($opt eq 'cpulimit') {
1382             next if $hotplug_error->($opt); # FIXME: hotplug
1383             $conf->{$opt} = $value;
1384         } elsif ($opt eq 'cpuunits') {
1385             $conf->{$opt} = $value;
1386             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1387         } elsif ($opt eq 'description') {
1388             $conf->{$opt} = PVE::Tools::encode_text($value);
1389         } elsif ($opt =~ m/^net(\d+)$/) {
1390             my $netid = $1;
1391             my $net = parse_lxc_network($value);
1392             if (!$running) {
1393                 $conf->{$opt} = print_lxc_network($net);
1394             } else {
1395                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1396             }
1397         } elsif ($opt eq 'protection') {
1398             $conf->{$opt} = $value ? 1 : 0;
1399         } elsif ($opt =~ m/^mp(\d+)$/) {
1400             next if $hotplug_error->($opt);
1401             check_protection($conf, "can't update CT $vmid drive '$opt'");
1402             my $old = $conf->{$opt};
1403             $conf->{$opt} = $value;
1404             if (defined($old)) {
1405                 my $mp = parse_ct_mountpoint($old);
1406                 if ($mp->{type} eq 'volume') {
1407                     add_unused_volume($conf, $mp->{volume});
1408                 }
1409             }
1410             $new_disks = 1;
1411             my $mp = parse_ct_mountpoint($value);
1412             $used_volids->{$mp->{volume}} = 1;
1413         } elsif ($opt eq 'rootfs') {
1414             next if $hotplug_error->($opt);
1415             check_protection($conf, "can't update CT $vmid drive '$opt'");
1416             my $old = $conf->{$opt};
1417             $conf->{$opt} = $value;
1418             if (defined($old)) {
1419                 my $mp = parse_ct_rootfs($old);
1420                 if ($mp->{type} eq 'volume') {
1421                     add_unused_volume($conf, $mp->{volume});
1422                 }
1423             }
1424             my $mp = parse_ct_rootfs($value);
1425             $used_volids->{$mp->{volume}} = 1;
1426         } elsif ($opt eq 'unprivileged') {
1427             die "unable to modify read-only option: '$opt'\n";
1428         } elsif ($opt eq 'ostype') {
1429             next if $hotplug_error->($opt);
1430             $conf->{$opt} = $value;
1431         } else {
1432             die "implement me: $opt";
1433         }
1434         write_config($vmid, $conf) if $running;
1435     }
1436
1437     # Apply deletions and creations of new volumes
1438     if (@deleted_volumes) {
1439         my $storage_cfg = PVE::Storage::config();
1440         foreach my $volume (@deleted_volumes) {
1441             next if $used_volids->{$volume}; # could have been re-added, too
1442             # also check for references in snapshots
1443             next if is_volume_in_use($conf, $volume, 1);
1444             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1445         }
1446     }
1447
1448     if ($new_disks) {
1449         my $storage_cfg = PVE::Storage::config();
1450         create_disks($storage_cfg, $vmid, $conf, $conf);
1451     }
1452
1453     # This should be the last thing we do here
1454     if ($running && scalar(@nohotplug)) {
1455         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1456     }
1457 }
1458
1459 sub has_dev_console {
1460     my ($conf) = @_;
1461
1462     return !(defined($conf->{console}) && !$conf->{console});
1463 }
1464
1465 sub get_tty_count {
1466     my ($conf) = @_;
1467
1468     return $conf->{tty} // $confdesc->{tty}->{default};
1469 }
1470
1471 sub get_cmode {
1472     my ($conf) = @_;
1473
1474     return $conf->{cmode} // $confdesc->{cmode}->{default};
1475 }
1476
1477 sub get_console_command {
1478     my ($vmid, $conf) = @_;
1479
1480     my $cmode = get_cmode($conf);
1481
1482     if ($cmode eq 'console') {
1483         return ['lxc-console', '-n',  $vmid, '-t', 0];
1484     } elsif ($cmode eq 'tty') {
1485         return ['lxc-console', '-n',  $vmid];
1486     } elsif ($cmode eq 'shell') {
1487         return ['lxc-attach', '--clear-env', '-n', $vmid];
1488     } else {
1489         die "internal error";
1490     }
1491 }
1492
1493 sub get_primary_ips {
1494     my ($conf) = @_;
1495
1496     # return data from net0
1497
1498     return undef if !defined($conf->{net0});
1499     my $net = parse_lxc_network($conf->{net0});
1500
1501     my $ipv4 = $net->{ip};
1502     if ($ipv4) {
1503         if ($ipv4 =~ /^(dhcp|manual)$/) {
1504             $ipv4 = undef
1505         } else {
1506             $ipv4 =~ s!/\d+$!!;
1507         }
1508     }
1509     my $ipv6 = $net->{ip6};
1510     if ($ipv6) {
1511         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1512             $ipv6 = undef;
1513         } else {
1514             $ipv6 =~ s!/\d+$!!;
1515         }
1516     }
1517
1518     return ($ipv4, $ipv6);
1519 }
1520
1521 sub delete_mountpoint_volume {
1522     my ($storage_cfg, $vmid, $volume) = @_;
1523
1524     return if classify_mountpoint($volume) ne 'volume';
1525
1526     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1527     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1528 }
1529
1530 sub destroy_lxc_container {
1531     my ($storage_cfg, $vmid, $conf) = @_;
1532
1533     foreach_mountpoint($conf, sub {
1534         my ($ms, $mountpoint) = @_;
1535         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1536     });
1537
1538     rmdir "/var/lib/lxc/$vmid/rootfs";
1539     unlink "/var/lib/lxc/$vmid/config";
1540     rmdir "/var/lib/lxc/$vmid";
1541     destroy_config($vmid);
1542
1543     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1544     #PVE::Tools::run_command($cmd);
1545 }
1546
1547 sub vm_stop_cleanup {
1548     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1549
1550     eval {
1551         if (!$keepActive) {
1552
1553             my $vollist = get_vm_volumes($conf);
1554             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1555         }
1556     };
1557     warn $@ if $@; # avoid errors - just warn
1558 }
1559
1560 my $safe_num_ne = sub {
1561     my ($a, $b) = @_;
1562
1563     return 0 if !defined($a) && !defined($b);
1564     return 1 if !defined($a);
1565     return 1 if !defined($b);
1566
1567     return $a != $b;
1568 };
1569
1570 my $safe_string_ne = sub {
1571     my ($a, $b) = @_;
1572
1573     return 0 if !defined($a) && !defined($b);
1574     return 1 if !defined($a);
1575     return 1 if !defined($b);
1576
1577     return $a ne $b;
1578 };
1579
1580 sub update_net {
1581     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1582
1583     if ($newnet->{type} ne 'veth') {
1584         # for when there are physical interfaces
1585         die "cannot update interface of type $newnet->{type}";
1586     }
1587
1588     my $veth = "veth${vmid}i${netid}";
1589     my $eth = $newnet->{name};
1590
1591     if (my $oldnetcfg = $conf->{$opt}) {
1592         my $oldnet = parse_lxc_network($oldnetcfg);
1593
1594         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1595             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1596
1597             PVE::Network::veth_delete($veth);
1598             delete $conf->{$opt};
1599             write_config($vmid, $conf);
1600
1601             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1602
1603         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1604                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1605                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1606
1607                 if ($oldnet->{bridge}) {
1608                     PVE::Network::tap_unplug($veth);
1609                     foreach (qw(bridge tag firewall)) {
1610                         delete $oldnet->{$_};
1611                     }
1612                     $conf->{$opt} = print_lxc_network($oldnet);
1613                     write_config($vmid, $conf);
1614                 }
1615
1616                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1617                 foreach (qw(bridge tag firewall)) {
1618                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1619                 }
1620                 $conf->{$opt} = print_lxc_network($oldnet);
1621                 write_config($vmid, $conf);
1622         }
1623     } else {
1624         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1625     }
1626
1627     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1628 }
1629
1630 sub hotplug_net {
1631     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1632
1633     my $veth = "veth${vmid}i${netid}";
1634     my $vethpeer = $veth . "p";
1635     my $eth = $newnet->{name};
1636
1637     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1638     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1639
1640     # attach peer in container
1641     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1642     PVE::Tools::run_command($cmd);
1643
1644     # link up peer in container
1645     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1646     PVE::Tools::run_command($cmd);
1647
1648     my $done = { type => 'veth' };
1649     foreach (qw(bridge tag firewall hwaddr name)) {
1650         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1651     }
1652     $conf->{$opt} = print_lxc_network($done);
1653
1654     write_config($vmid, $conf);
1655 }
1656
1657 sub update_ipconfig {
1658     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1659
1660     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1661
1662     my $optdata = parse_lxc_network($conf->{$opt});
1663     my $deleted = [];
1664     my $added = [];
1665     my $nscmd = sub {
1666         my $cmdargs = shift;
1667         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1668     };
1669     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1670
1671     my $change_ip_config = sub {
1672         my ($ipversion) = @_;
1673
1674         my $family_opt = "-$ipversion";
1675         my $suffix = $ipversion == 4 ? '' : $ipversion;
1676         my $gw= "gw$suffix";
1677         my $ip= "ip$suffix";
1678
1679         my $newip = $newnet->{$ip};
1680         my $newgw = $newnet->{$gw};
1681         my $oldip = $optdata->{$ip};
1682
1683         my $change_ip = &$safe_string_ne($oldip, $newip);
1684         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1685
1686         return if !$change_ip && !$change_gw;
1687
1688         # step 1: add new IP, if this fails we cancel
1689         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1690         if ($change_ip && $is_real_ip) {
1691             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1692             if (my $err = $@) {
1693                 warn $err;
1694                 return;
1695             }
1696         }
1697
1698         # step 2: replace gateway
1699         #   If this fails we delete the added IP and cancel.
1700         #   If it succeeds we save the config and delete the old IP, ignoring
1701         #   errors. The config is then saved.
1702         # Note: 'ip route replace' can add
1703         if ($change_gw) {
1704             if ($newgw) {
1705                 eval {
1706                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1707                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1708                     }
1709                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1710                 };
1711                 if (my $err = $@) {
1712                     warn $err;
1713                     # the route was not replaced, the old IP is still available
1714                     # rollback (delete new IP) and cancel
1715                     if ($change_ip) {
1716                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1717                         warn $@ if $@; # no need to die here
1718                     }
1719                     return;
1720                 }
1721             } else {
1722                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1723                 # if the route was not deleted, the guest might have deleted it manually
1724                 # warn and continue
1725                 warn $@ if $@;
1726             }
1727         }
1728
1729         # from this point on we save the configuration
1730         # step 3: delete old IP ignoring errors
1731         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1732             # We need to enable promote_secondaries, otherwise our newly added
1733             # address will be removed along with the old one.
1734             my $promote = 0;
1735             eval {
1736                 if ($ipversion == 4) {
1737                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1738                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1739                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1740                 }
1741                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1742             };
1743             warn $@ if $@; # no need to die here
1744
1745             if ($ipversion == 4) {
1746                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1747             }
1748         }
1749
1750         foreach my $property ($ip, $gw) {
1751             if ($newnet->{$property}) {
1752                 $optdata->{$property} = $newnet->{$property};
1753             } else {
1754                 delete $optdata->{$property};
1755             }
1756         }
1757         $conf->{$opt} = print_lxc_network($optdata);
1758         write_config($vmid, $conf);
1759         $lxc_setup->setup_network($conf);
1760     };
1761
1762     &$change_ip_config(4);
1763     &$change_ip_config(6);
1764
1765 }
1766
1767 # Internal snapshots
1768
1769 # NOTE: Snapshot create/delete involves several non-atomic
1770 # actions, and can take a long time.
1771 # So we try to avoid locking the file and use the 'lock' variable
1772 # inside the config file instead.
1773
1774 my $snapshot_copy_config = sub {
1775     my ($source, $dest) = @_;
1776
1777     foreach my $k (keys %$source) {
1778         next if $k eq 'snapshots';
1779         next if $k eq 'snapstate';
1780         next if $k eq 'snaptime';
1781         next if $k eq 'vmstate';
1782         next if $k eq 'lock';
1783         next if $k eq 'digest';
1784         next if $k eq 'description';
1785         next if $k =~ m/^unused\d+$/;
1786
1787         $dest->{$k} = $source->{$k};
1788     }
1789 };
1790
1791 my $snapshot_apply_config = sub {
1792     my ($conf, $snap) = @_;
1793
1794     # copy snapshot list
1795     my $newconf = {
1796         snapshots => $conf->{snapshots},
1797     };
1798
1799     # keep description and list of unused disks
1800     foreach my $k (keys %$conf) {
1801         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
1802         $newconf->{$k} = $conf->{$k};
1803     }
1804
1805     &$snapshot_copy_config($snap, $newconf);
1806
1807     return $newconf;
1808 };
1809
1810 sub snapshot_save_vmstate {
1811     die "implement me - snapshot_save_vmstate\n";
1812 }
1813
1814 sub snapshot_prepare {
1815     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1816
1817     my $snap;
1818
1819     my $updatefn =  sub {
1820
1821         my $conf = load_config($vmid);
1822
1823         die "you can't take a snapshot if it's a template\n"
1824             if is_template($conf);
1825
1826         check_lock($conf);
1827
1828         $conf->{lock} = 'snapshot';
1829
1830         die "snapshot name '$snapname' already used\n"
1831             if defined($conf->{snapshots}->{$snapname});
1832
1833         my $storecfg = PVE::Storage::config();
1834         die "snapshot feature is not available\n"
1835             if !has_feature('snapshot', $conf, $storecfg, undef, undef, $snapname eq 'vzdump');
1836
1837         $snap = $conf->{snapshots}->{$snapname} = {};
1838
1839         if ($save_vmstate && check_running($vmid)) {
1840             snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
1841         }
1842
1843         &$snapshot_copy_config($conf, $snap);
1844
1845         $snap->{snapstate} = "prepare";
1846         $snap->{snaptime} = time();
1847         $snap->{description} = $comment if $comment;
1848
1849         write_config($vmid, $conf);
1850     };
1851
1852     lock_config($vmid, $updatefn);
1853
1854     return $snap;
1855 }
1856
1857 sub snapshot_commit {
1858     my ($vmid, $snapname) = @_;
1859
1860     my $updatefn = sub {
1861
1862         my $conf = load_config($vmid);
1863
1864         die "missing snapshot lock\n"
1865             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1866
1867         my $snap = $conf->{snapshots}->{$snapname};
1868         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1869
1870         die "wrong snapshot state\n"
1871             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
1872
1873         delete $snap->{snapstate};
1874         delete $conf->{lock};
1875
1876         $conf->{parent} = $snapname;
1877
1878         write_config($vmid, $conf);
1879     };
1880
1881     lock_config($vmid, $updatefn);
1882 }
1883
1884 sub has_feature {
1885     my ($feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
1886
1887     my $err;
1888
1889     foreach_mountpoint($conf, sub {
1890         my ($ms, $mountpoint) = @_;
1891
1892         return if $err; # skip further test
1893         return if $backup_only && $ms ne 'rootfs' && !$mountpoint->{backup};
1894
1895         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname, $running);
1896     });
1897
1898     return $err ? 0 : 1;
1899 }
1900
1901 my $enter_namespace = sub {
1902     my ($vmid, $pid, $which, $type) = @_;
1903     sysopen my $fd, "/proc/$pid/ns/$which", O_RDONLY
1904         or die "failed to open $which namespace of container $vmid: $!\n";
1905     PVE::Tools::setns(fileno($fd), $type)
1906         or die "failed to enter $which namespace of container $vmid: $!\n";
1907     close $fd;
1908 };
1909
1910 my $do_syncfs = sub {
1911     my ($vmid, $pid, $socket) = @_;
1912
1913     &$enter_namespace($vmid, $pid, 'mnt', PVE::Tools::CLONE_NEWNS);
1914
1915     # Tell the parent process to start reading our /proc/mounts
1916     print {$socket} "go\n";
1917     $socket->flush();
1918
1919     # Receive /proc/self/mounts
1920     my $mountdata = do { local $/ = undef; <$socket> };
1921     close $socket;
1922
1923     # Now sync all mountpoints...
1924     my $mounts = PVE::ProcFSTools::parse_mounts($mountdata);
1925     foreach my $mp (@$mounts) {
1926         my ($what, $dir, $fs) = @$mp;
1927         next if $fs eq 'fuse.lxcfs';
1928         eval { PVE::Tools::sync_mountpoint($dir); };
1929         warn $@ if $@;
1930     }
1931 };
1932
1933 sub sync_container_namespace {
1934     my ($vmid) = @_;
1935     my $pid = find_lxc_pid($vmid);
1936
1937     # SOCK_DGRAM is nicer for barriers but cannot be slurped
1938     socketpair my $pfd, my $cfd, AF_UNIX, SOCK_STREAM, PF_UNSPEC
1939         or die "failed to create socketpair: $!\n";
1940
1941     my $child = fork();
1942     die "fork failed: $!\n" if !defined($child);
1943
1944     if (!$child) {
1945         eval {
1946             close $pfd;
1947             &$do_syncfs($vmid, $pid, $cfd);
1948         };
1949         if (my $err = $@) {
1950             warn $err;
1951             POSIX::_exit(1);
1952         }
1953         POSIX::_exit(0);
1954     }
1955     close $cfd;
1956     my $go = <$pfd>;
1957     die "failed to enter container namespace\n" if $go ne "go\n";
1958
1959     open my $mounts, '<', "/proc/$child/mounts"
1960         or die "failed to open container's /proc/mounts: $!\n";
1961     my $mountdata = do { local $/ = undef; <$mounts> };
1962     close $mounts;
1963     print {$pfd} $mountdata;
1964     close $pfd;
1965
1966     while (waitpid($child, 0) != $child) {}
1967     die "failed to sync container namespace\n" if $? != 0;
1968 }
1969
1970 sub check_freeze_needed {
1971     my ($vmid, $config, $save_vmstate) = @_;
1972
1973     my $ret = check_running($vmid);
1974     return ($ret, $ret);
1975 }
1976
1977 sub snapshot_create {
1978     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
1979
1980     my $snap = snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
1981
1982     $save_vmstate = 0 if !$snap->{vmstate};
1983
1984     my $conf = load_config($vmid);
1985
1986     my ($running, $freezefs) = check_freeze_needed($vmid, $conf, $snap->{vmstate});
1987
1988     my $drivehash = {};
1989
1990     eval {
1991         if ($freezefs) {
1992             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1993             sync_container_namespace($vmid);
1994         }
1995
1996         my $storecfg = PVE::Storage::config();
1997         foreach_mountpoint($conf, sub {
1998             my ($ms, $mountpoint) = @_;
1999
2000             return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
2001             PVE::Storage::volume_snapshot($storecfg, $mountpoint->{volume}, $snapname);
2002             $drivehash->{$ms} = 1;
2003         });
2004     };
2005     my $err = $@;
2006
2007     if ($running) {
2008         if ($freezefs) {
2009             eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
2010             warn $@ if $@;
2011         }
2012     }
2013
2014     if ($err) {
2015         warn "snapshot create failed: starting cleanup\n";
2016         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
2017         warn "$@" if $@;
2018         die "$err\n";
2019     }
2020
2021     snapshot_commit($vmid, $snapname);
2022 }
2023
2024 # Note: $drivehash is only set when called from snapshot_create.
2025 sub snapshot_delete {
2026     my ($vmid, $snapname, $force, $drivehash) = @_;
2027
2028     my $prepare = 1;
2029
2030     my $snap;
2031     my $unused = [];
2032
2033     my $unlink_parent = sub {
2034         my ($confref, $new_parent) = @_;
2035
2036         if ($confref->{parent} && $confref->{parent} eq $snapname) {
2037             if ($new_parent) {
2038                 $confref->{parent} = $new_parent;
2039             } else {
2040                 delete $confref->{parent};
2041             }
2042         }
2043     };
2044
2045     my $updatefn =  sub {
2046         my ($remove_drive) = @_;
2047
2048         my $conf = load_config($vmid);
2049
2050         if (!$drivehash) {
2051             check_lock($conf);
2052             die "you can't delete a snapshot if vm is a template\n"
2053                 if is_template($conf);
2054         }
2055
2056         $snap = $conf->{snapshots}->{$snapname};
2057
2058         die "snapshot '$snapname' does not exist\n" if !defined($snap);
2059
2060         # remove parent refs
2061         if (!$prepare) {
2062             &$unlink_parent($conf, $snap->{parent});
2063             foreach my $sn (keys %{$conf->{snapshots}}) {
2064                 next if $sn eq $snapname;
2065                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
2066             }
2067         }
2068
2069         if ($remove_drive) {
2070             if ($remove_drive eq 'vmstate') {
2071                 die "implement me - saving vmstate\n";
2072             } else {
2073                 my $value = $snap->{$remove_drive};
2074                 my $mountpoint = $remove_drive eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2075                 delete $snap->{$remove_drive};
2076                 add_unused_volume($snap, $mountpoint->{volume});
2077             }
2078         }
2079
2080         if ($prepare) {
2081             $snap->{snapstate} = 'delete';
2082         } else {
2083             delete $conf->{snapshots}->{$snapname};
2084             delete $conf->{lock} if $drivehash;
2085             foreach my $volid (@$unused) {
2086                 add_unused_volume($conf, $volid);
2087             }
2088         }
2089
2090         write_config($vmid, $conf);
2091     };
2092
2093     lock_config($vmid, $updatefn);
2094
2095     # now remove vmstate file
2096     # never set for LXC!
2097     my $storecfg = PVE::Storage::config();
2098
2099     if ($snap->{vmstate}) {
2100         die "implement me - saving vmstate\n";
2101     };
2102
2103     # now remove all volume snapshots
2104     foreach_mountpoint($snap, sub {
2105         my ($ms, $mountpoint) = @_;
2106
2107         return if $snapname eq 'vzdump' && $ms ne 'rootfs' && !$mountpoint->{backup};
2108         if (!$drivehash || $drivehash->{$ms}) {
2109             eval { PVE::Storage::volume_snapshot_delete($storecfg, $mountpoint->{volume}, $snapname); };
2110             if (my $err = $@) {
2111                 die $err if !$force;
2112                 warn $err;
2113             }
2114         }
2115
2116         # save changes (remove mp from snapshot)
2117         lock_config($vmid, $updatefn, $ms) if !$force;
2118         push @$unused, $mountpoint->{volume};
2119     });
2120
2121     # now cleanup config
2122     $prepare = 0;
2123     lock_config($vmid, $updatefn);
2124 }
2125
2126 sub snapshot_rollback {
2127     my ($vmid, $snapname) = @_;
2128
2129     my $prepare = 1;
2130
2131     my $storecfg = PVE::Storage::config();
2132
2133     my $conf = load_config($vmid);
2134
2135     my $get_snapshot_config = sub {
2136
2137         die "you can't rollback if vm is a template\n" if is_template($conf);
2138
2139         my $res = $conf->{snapshots}->{$snapname};
2140
2141         die "snapshot '$snapname' does not exist\n" if !defined($res);
2142
2143         return $res;
2144     };
2145
2146     my $snap = &$get_snapshot_config();
2147
2148     foreach_mountpoint($snap, sub {
2149         my ($ms, $mountpoint) = @_;
2150
2151         PVE::Storage::volume_rollback_is_possible($storecfg, $mountpoint->{volume}, $snapname);
2152     });
2153
2154     my $updatefn = sub {
2155
2156         $conf = load_config($vmid);
2157
2158         $snap = &$get_snapshot_config();
2159
2160         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
2161             if $snap->{snapstate};
2162
2163         if ($prepare) {
2164             check_lock($conf);
2165             PVE::Tools::run_command(['/usr/bin/lxc-stop', '-n', $vmid, '--kill'])
2166                 if check_running($vmid);
2167         }
2168
2169         die "unable to rollback vm $vmid: vm is running\n"
2170             if check_running($vmid);
2171
2172         if ($prepare) {
2173             $conf->{lock} = 'rollback';
2174         } else {
2175             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
2176             delete $conf->{lock};
2177         }
2178
2179         my $forcemachine;
2180
2181         if (!$prepare) {
2182             # copy snapshot config to current config
2183             $conf = &$snapshot_apply_config($conf, $snap);
2184             $conf->{parent} = $snapname;
2185         }
2186
2187         write_config($vmid, $conf);
2188
2189         if (!$prepare && $snap->{vmstate}) {
2190             die "implement me - save vmstate\n";
2191         }
2192     };
2193
2194     lock_config($vmid, $updatefn);
2195
2196     foreach_mountpoint($snap, sub {
2197         my ($ms, $mountpoint) = @_;
2198
2199         PVE::Storage::volume_snapshot_rollback($storecfg, $mountpoint->{volume}, $snapname);
2200     });
2201
2202     $prepare = 0;
2203     lock_config($vmid, $updatefn);
2204 }
2205
2206 sub template_create {
2207     my ($vmid, $conf) = @_;
2208
2209     my $storecfg = PVE::Storage::config();
2210
2211     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
2212     my $volid = $rootinfo->{volume};
2213
2214     die "Template feature is not available for '$volid'\n"
2215         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
2216
2217     PVE::Storage::activate_volumes($storecfg, [$volid]);
2218
2219     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
2220     $rootinfo->{volume} = $template_volid;
2221     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
2222
2223     write_config($vmid, $conf);
2224 }
2225
2226 sub is_template {
2227     my ($conf) = @_;
2228
2229     return 1 if defined $conf->{template} && $conf->{template} == 1;
2230 }
2231
2232 sub mountpoint_names {
2233     my ($reverse) = @_;
2234
2235     my @names = ('rootfs');
2236
2237     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
2238         push @names, "mp$i";
2239     }
2240
2241     return $reverse ? reverse @names : @names;
2242 }
2243
2244
2245 sub foreach_mountpoint_full {
2246     my ($conf, $reverse, $func) = @_;
2247
2248     foreach my $key (mountpoint_names($reverse)) {
2249         my $value = $conf->{$key};
2250         next if !defined($value);
2251         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2252         next if !defined($mountpoint);
2253
2254         &$func($key, $mountpoint);
2255     }
2256 }
2257
2258 sub foreach_mountpoint {
2259     my ($conf, $func) = @_;
2260
2261     foreach_mountpoint_full($conf, 0, $func);
2262 }
2263
2264 sub foreach_mountpoint_reverse {
2265     my ($conf, $func) = @_;
2266
2267     foreach_mountpoint_full($conf, 1, $func);
2268 }
2269
2270 sub check_ct_modify_config_perm {
2271     my ($rpcenv, $authuser, $vmid, $pool, $newconf, $delete) = @_;
2272
2273     return 1 if $authuser eq 'root@pam';
2274
2275     my $check = sub {
2276         my ($opt, $delete) = @_;
2277         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2278             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2279         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2280             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2281             return if $delete;
2282             my $data = $opt eq 'rootfs' ? parse_ct_rootfs($newconf->{$opt})
2283                                         : parse_ct_mountpoint($newconf->{$opt});
2284             raise_perm_exc("mountpoint type $data->{type}") if $data->{type} ne 'volume';
2285         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2286             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2287         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2288                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2289             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2290         } else {
2291             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2292         }
2293     };
2294
2295     foreach my $opt (keys %$newconf) {
2296         &$check($opt, 0);
2297     }
2298     foreach my $opt (@$delete) {
2299         &$check($opt, 1);
2300     }
2301
2302     return 1;
2303 }
2304
2305 sub umount_all {
2306     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2307
2308     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2309     my $volid_list = get_vm_volumes($conf);
2310
2311     foreach_mountpoint_reverse($conf, sub {
2312         my ($ms, $mountpoint) = @_;
2313
2314         my $volid = $mountpoint->{volume};
2315         my $mount = $mountpoint->{mp};
2316
2317         return if !$volid || !$mount;
2318
2319         my $mount_path = "$rootdir/$mount";
2320         $mount_path =~ s!/+!/!g;
2321
2322         return if !PVE::ProcFSTools::is_mounted($mount_path);
2323
2324         eval {
2325             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2326         };
2327         if (my $err = $@) {
2328             if ($noerr) {
2329                 warn $err;
2330             } else {
2331                 die $err;
2332             }
2333         }
2334     });
2335 }
2336
2337 sub mount_all {
2338     my ($vmid, $storage_cfg, $conf) = @_;
2339
2340     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2341     File::Path::make_path($rootdir);
2342
2343     my $volid_list = get_vm_volumes($conf);
2344     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2345
2346     eval {
2347         foreach_mountpoint($conf, sub {
2348             my ($ms, $mountpoint) = @_;
2349
2350             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2351         });
2352     };
2353     if (my $err = $@) {
2354         warn "mounting container failed\n";
2355         umount_all($vmid, $storage_cfg, $conf, 1);
2356         die $err;
2357     }
2358
2359     return $rootdir;
2360 }
2361
2362
2363 sub mountpoint_mount_path {
2364     my ($mountpoint, $storage_cfg, $snapname) = @_;
2365
2366     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2367 }
2368
2369 my $check_mount_path = sub {
2370     my ($path) = @_;
2371     $path = File::Spec->canonpath($path);
2372     my $real = Cwd::realpath($path);
2373     if ($real ne $path) {
2374         die "mount path modified by symlink: $path != $real";
2375     }
2376 };
2377
2378 sub query_loopdev {
2379     my ($path) = @_;
2380     my $found;
2381     my $parser = sub {
2382         my $line = shift;
2383         if ($line =~ m@^(/dev/loop\d+):@) {
2384             $found = $1;
2385         }
2386     };
2387     my $cmd = ['losetup', '--associated', $path];
2388     PVE::Tools::run_command($cmd, outfunc => $parser);
2389     return $found;
2390 }
2391
2392 # Run a function with a file attached to a loop device.
2393 # The loop device is always detached afterwards (or set to autoclear).
2394 # Returns the loop device.
2395 sub run_with_loopdev {
2396     my ($func, $file) = @_;
2397     my $device = query_loopdev($file);
2398     # Try to reuse an existing device
2399     if ($device) {
2400         # We assume that whoever setup the loop device is responsible for
2401         # detaching it.
2402         &$func($device);
2403         return $device;
2404     }
2405
2406     my $parser = sub {
2407         my $line = shift;
2408         if ($line =~ m@^(/dev/loop\d+)$@) {
2409             $device = $1;
2410         }
2411     };
2412     PVE::Tools::run_command(['losetup', '--show', '-f', $file], outfunc => $parser);
2413     die "failed to setup loop device for $file\n" if !$device;
2414     eval { &$func($device); };
2415     my $err = $@;
2416     PVE::Tools::run_command(['losetup', '-d', $device]);
2417     die $err if $err;
2418     return $device;
2419 }
2420
2421 sub bindmount {
2422     my ($dir, $dest, $ro, @extra_opts) = @_;
2423     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $dir, $dest]);
2424     if ($ro) {
2425         eval { PVE::Tools::run_command(['mount', '-o', 'bind,remount,ro', $dest]); };
2426         if (my $err = $@) {
2427             warn "bindmount error\n";
2428             # don't leave writable bind-mounts behind...
2429             PVE::Tools::run_command(['umount', $dest]);
2430             die $err;
2431         }
2432     }
2433 }
2434
2435 # use $rootdir = undef to just return the corresponding mount path
2436 sub mountpoint_mount {
2437     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2438
2439     my $volid = $mountpoint->{volume};
2440     my $mount = $mountpoint->{mp};
2441     my $type = $mountpoint->{type};
2442     my $quota = !$snapname && !$mountpoint->{ro} && $mountpoint->{quota};
2443     my $mounted_dev;
2444
2445     return if !$volid || !$mount;
2446
2447     my $mount_path;
2448
2449     if (defined($rootdir)) {
2450         $rootdir =~ s!/+$!!;
2451         $mount_path = "$rootdir/$mount";
2452         $mount_path =~ s!/+!/!g;
2453         &$check_mount_path($mount_path);
2454         File::Path::mkpath($mount_path);
2455     }
2456
2457     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2458
2459     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2460
2461     my $optstring = '';
2462     if (defined($mountpoint->{acl})) {
2463         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
2464     }
2465     my $readonly = $mountpoint->{ro};
2466
2467     my @extra_opts = ('-o', $optstring);
2468
2469     if ($storage) {
2470
2471         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2472         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2473
2474         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2475             PVE::Storage::parse_volname($storage_cfg, $volid);
2476
2477         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2478
2479         if ($format eq 'subvol') {
2480             if ($mount_path) {
2481                 if ($snapname) {
2482                     if ($scfg->{type} eq 'zfspool') {
2483                         my $path_arg = $path;
2484                         $path_arg =~ s!^/+!!;
2485                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
2486                     } else {
2487                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2488                     }
2489                 } else {
2490                     bindmount($path, $mount_path, $readonly, @extra_opts);
2491                     warn "cannot enable quota control for bind mounted subvolumes\n" if $quota;
2492                 }
2493             }
2494             return wantarray ? ($path, 0, $mounted_dev) : $path;
2495         } elsif ($format eq 'raw' || $format eq 'iso') {
2496             my $domount = sub {
2497                 my ($path) = @_;
2498                 if ($mount_path) {
2499                     if ($format eq 'iso') {
2500                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2501                     } elsif ($isBase || defined($snapname)) {
2502                         PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2503                     } else {
2504                         if ($quota) {
2505                             push @extra_opts, '-o', 'usrjquota=aquota.user,grpjquota=aquota.group,jqfmt=vfsv0';
2506                         }
2507                         push @extra_opts, '-o', 'ro' if $readonly;
2508                         PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2509                     }
2510                 }
2511             };
2512             my $use_loopdev = 0;
2513             if ($scfg->{path}) {
2514                 $mounted_dev = run_with_loopdev($domount, $path);
2515                 $use_loopdev = 1;
2516             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2517                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2518                 $mounted_dev = $path;
2519                 &$domount($path);
2520             } else {
2521                 die "unsupported storage type '$scfg->{type}'\n";
2522             }
2523             return wantarray ? ($path, $use_loopdev, $mounted_dev) : $path;
2524         } else {
2525             die "unsupported image format '$format'\n";
2526         }
2527     } elsif ($type eq 'device') {
2528                         push @extra_opts, '-o', 'ro' if $readonly;
2529         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
2530         return wantarray ? ($volid, 0, $volid) : $volid;
2531     } elsif ($type eq 'bind') {
2532         die "directory '$volid' does not exist\n" if ! -d $volid;
2533         &$check_mount_path($volid);
2534         bindmount($volid, $mount_path, $readonly, @extra_opts) if $mount_path;
2535         warn "cannot enable quota control for bind mounts\n" if $quota;
2536         return wantarray ? ($volid, 0, undef) : $volid;
2537     }
2538
2539     die "unsupported storage";
2540 }
2541
2542 sub get_vm_volumes {
2543     my ($conf, $excludes) = @_;
2544
2545     my $vollist = [];
2546
2547     foreach_mountpoint($conf, sub {
2548         my ($ms, $mountpoint) = @_;
2549
2550         return if $excludes && $ms eq $excludes;
2551
2552         my $volid = $mountpoint->{volume};
2553
2554         return if !$volid || $mountpoint->{type} ne 'volume';
2555
2556         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2557         return if !$sid;
2558
2559         push @$vollist, $volid;
2560     });
2561
2562     return $vollist;
2563 }
2564
2565 sub mkfs {
2566     my ($dev, $rootuid, $rootgid) = @_;
2567
2568     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2569                              '-E', "root_owner=$rootuid:$rootgid",
2570                              $dev]);
2571 }
2572
2573 sub format_disk {
2574     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2575
2576     if ($volid =~ m!^/dev/.+!) {
2577         mkfs($volid);
2578         return;
2579     }
2580
2581     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2582
2583     die "cannot format volume '$volid' with no storage\n" if !$storage;
2584
2585     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2586
2587     my $path = PVE::Storage::path($storage_cfg, $volid);
2588
2589     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2590         PVE::Storage::parse_volname($storage_cfg, $volid);
2591
2592     die "cannot format volume '$volid' (format == $format)\n"
2593         if $format ne 'raw';
2594
2595     mkfs($path, $rootuid, $rootgid);
2596 }
2597
2598 sub destroy_disks {
2599     my ($storecfg, $vollist) = @_;
2600
2601     foreach my $volid (@$vollist) {
2602         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2603         warn $@ if $@;
2604     }
2605 }
2606
2607 sub create_disks {
2608     my ($storecfg, $vmid, $settings, $conf) = @_;
2609
2610     my $vollist = [];
2611
2612     eval {
2613         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2614         my $chown_vollist = [];
2615
2616         foreach_mountpoint($settings, sub {
2617             my ($ms, $mountpoint) = @_;
2618
2619             my $volid = $mountpoint->{volume};
2620             my $mp = $mountpoint->{mp};
2621
2622             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2623
2624             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2625                 my ($storeid, $size_gb) = ($1, $2);
2626
2627                 my $size_kb = int(${size_gb}*1024) * 1024;
2628
2629                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2630                 # fixme: use better naming ct-$vmid-disk-X.raw?
2631
2632                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2633                     if ($size_kb > 0) {
2634                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2635                                                            undef, $size_kb);
2636                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2637                     } else {
2638                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2639                                                            undef, 0);
2640                         push @$chown_vollist, $volid;
2641                     }
2642                 } elsif ($scfg->{type} eq 'zfspool') {
2643
2644                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2645                                                        undef, $size_kb);
2646                     push @$chown_vollist, $volid;
2647                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2648
2649                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2650                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2651
2652                 } elsif ($scfg->{type} eq 'rbd') {
2653
2654                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2655                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2656                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2657                 } else {
2658                     die "unable to create containers on storage type '$scfg->{type}'\n";
2659                 }
2660                 push @$vollist, $volid;
2661                 $mountpoint->{volume} = $volid;
2662                 $mountpoint->{size} = $size_kb * 1024;
2663                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2664             } else {
2665                 # use specified/existing volid/dir/device
2666                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2667             }
2668         });
2669
2670         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2671         foreach my $volid (@$chown_vollist) {
2672             my $path = PVE::Storage::path($storecfg, $volid, undef);
2673             chown($rootuid, $rootgid, $path);
2674         }
2675         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2676     };
2677     # free allocated images on error
2678     if (my $err = $@) {
2679         destroy_disks($storecfg, $vollist);
2680         die $err;
2681     }
2682     return $vollist;
2683 }
2684
2685 # bash completion helper
2686
2687 sub complete_os_templates {
2688     my ($cmdname, $pname, $cvalue) = @_;
2689
2690     my $cfg = PVE::Storage::config();
2691
2692     my $storeid;
2693
2694     if ($cvalue =~ m/^([^:]+):/) {
2695         $storeid = $1;
2696     }
2697
2698     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2699     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2700
2701     my $res = [];
2702     foreach my $id (keys %$data) {
2703         foreach my $item (@{$data->{$id}}) {
2704             push @$res, $item->{volid} if defined($item->{volid});
2705         }
2706     }
2707
2708     return $res;
2709 }
2710
2711 my $complete_ctid_full = sub {
2712     my ($running) = @_;
2713
2714     my $idlist = vmstatus();
2715
2716     my $active_hash = list_active_containers();
2717
2718     my $res = [];
2719
2720     foreach my $id (keys %$idlist) {
2721         my $d = $idlist->{$id};
2722         if (defined($running)) {
2723             next if $d->{template};
2724             next if $running && !$active_hash->{$id};
2725             next if !$running && $active_hash->{$id};
2726         }
2727         push @$res, $id;
2728
2729     }
2730     return $res;
2731 };
2732
2733 sub complete_ctid {
2734     return &$complete_ctid_full();
2735 }
2736
2737 sub complete_ctid_stopped {
2738     return &$complete_ctid_full(0);
2739 }
2740
2741 sub complete_ctid_running {
2742     return &$complete_ctid_full(1);
2743 }
2744
2745 sub parse_id_maps {
2746     my ($conf) = @_;
2747
2748     my $id_map = [];
2749     my $rootuid = 0;
2750     my $rootgid = 0;
2751
2752     my $lxc = $conf->{lxc};
2753     foreach my $entry (@$lxc) {
2754         my ($key, $value) = @$entry;
2755         next if $key ne 'lxc.id_map';
2756         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2757             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2758             push @$id_map, [$type, $ct, $host, $length];
2759             if ($ct == 0) {
2760                 $rootuid = $host if $type eq 'u';
2761                 $rootgid = $host if $type eq 'g';
2762             }
2763         } else {
2764             die "failed to parse id_map: $value\n";
2765         }
2766     }
2767
2768     if (!@$id_map && $conf->{unprivileged}) {
2769         # Should we read them from /etc/subuid?
2770         $id_map = [ ['u', '0', '100000', '65536'],
2771                     ['g', '0', '100000', '65536'] ];
2772         $rootuid = $rootgid = 100000;
2773     }
2774
2775     return ($id_map, $rootuid, $rootgid);
2776 }
2777
2778 sub userns_command {
2779     my ($id_map) = @_;
2780     if (@$id_map) {
2781         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2782     }
2783     return [];
2784 }
2785
2786 sub set_lock {
2787     my ($vmid, $lock) = @_;
2788     my $conf;
2789     lock_config($vmid, sub {
2790         $conf = load_config($vmid);
2791         check_lock($conf);
2792         $conf->{lock} = $lock;
2793         write_config($vmid, $conf);
2794     });
2795     return $conf;
2796 }
2797
2798 sub remove_lock {
2799     my ($vmid, $lock) = @_;
2800     lock_config($vmid, sub {
2801         my $conf = load_config($vmid);
2802         if (!$conf->{lock}) {
2803             die "no lock found trying to remove lock '$lock'\n";
2804         } elsif (defined($lock) && $conf->{lock} ne $lock) {
2805             die "found lock '$conf->{lock}' trying to remove lock '$lock'\n";
2806         }
2807         delete $conf->{lock};
2808         write_config($vmid, $conf);
2809     });
2810 }
2811
2812 1;