src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format => 'pve-lxc-mp-string',
  42         format_description => 'volume',
  43         description => 'Volume, device or directory to mount into the container.',
  44     },
  45     backup => {
  46         type => 'boolean',
  47         format_description => '[1|0]',
  48         description => 'Whether to include the mountpoint in backups.',
  49         optional => 1,
  50     },
  51     size => {
  52         type => 'string',
  53         format => 'disk-size',
  54         format_description => 'DiskSize',
  55         description => 'Volume size (read only value).',
  56         optional => 1,
  57     },
  58     acl => {
  59         type => 'boolean',
  60         format_description => 'acl',
  61         description => 'Explicitly enable or disable ACL support.',
  62         optional => 1,
  63     },
  64     ro => {
  65         type => 'boolean',
  66         format_description => 'ro',
  67         description => 'Read-only mountpoint (not supported with bind mounts)',
  68         optional => 1,
  69     },
  70 };
  71
  72 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  73     type => 'string', format => $rootfs_desc,
  74     description => "Use volume as container root.",
  75     optional => 1,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  79     description => "The name of the snapshot.",
  80     type => 'string', format => 'pve-configid',
  81     maxLength => 40,
  82 });
  83
  84 my $confdesc = {
  85     lock => {
  86         optional => 1,
  87         type => 'string',
  88         description => "Lock/unlock the VM.",
  89         enum => [qw(migrate backup snapshot rollback)],
  90     },
  91     onboot => {
  92         optional => 1,
  93         type => 'boolean',
  94         description => "Specifies whether a VM will be started during system bootup.",
  95         default => 0,
  96     },
  97     startup => get_standard_option('pve-startup-order'),
  98     template => {
  99         optional => 1,
 100         type => 'boolean',
 101         description => "Enable/disable Template.",
 102         default => 0,
 103     },
 104     arch => {
 105         optional => 1,
 106         type => 'string',
 107         enum => ['amd64', 'i386'],
 108         description => "OS architecture type.",
 109         default => 'amd64',
 110     },
 111     ostype => {
 112         optional => 1,
 113         type => 'string',
 114         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux'],
 115         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 116     },
 117     console => {
 118         optional => 1,
 119         type => 'boolean',
 120         description => "Attach a console device (/dev/console) to the container.",
 121         default => 1,
 122     },
 123     tty => {
 124         optional => 1,
 125         type => 'integer',
 126         description => "Specify the number of tty available to the container",
 127         minimum => 0,
 128         maximum => 6,
 129         default => 2,
 130     },
 131     cpulimit => {
 132         optional => 1,
 133         type => 'number',
 134         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 135         minimum => 0,
 136         maximum => 128,
 137         default => 0,
 138     },
 139     cpuunits => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 143         minimum => 0,
 144         maximum => 500000,
 145         default => 1024,
 146     },
 147     memory => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of RAM for the VM in MB.",
 151         minimum => 16,
 152         default => 512,
 153     },
 154     swap => {
 155         optional => 1,
 156         type => 'integer',
 157         description => "Amount of SWAP for the VM in MB.",
 158         minimum => 0,
 159         default => 512,
 160     },
 161     hostname => {
 162         optional => 1,
 163         description => "Set a host name for the container.",
 164         type => 'string', format => 'dns-name',
 165         maxLength => 255,
 166     },
 167     description => {
 168         optional => 1,
 169         type => 'string',
 170         description => "Container description. Only used on the configuration web interface.",
 171     },
 172     searchdomain => {
 173         optional => 1,
 174         type => 'string', format => 'dns-name-list',
 175         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 176     },
 177     nameserver => {
 178         optional => 1,
 179         type => 'string', format => 'address-list',
 180         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 181     },
 182     rootfs => get_standard_option('pve-ct-rootfs'),
 183     parent => {
 184         optional => 1,
 185         type => 'string', format => 'pve-configid',
 186         maxLength => 40,
 187         description => "Parent snapshot name. This is used internally, and should not be modified.",
 188     },
 189     snaptime => {
 190         optional => 1,
 191         description => "Timestamp for snapshots.",
 192         type => 'integer',
 193         minimum => 0,
 194     },
 195     cmode => {
 196         optional => 1,
 197         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 198         type => 'string',
 199         enum => ['shell', 'console', 'tty'],
 200         default => 'tty',
 201     },
 202     protection => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 206         default => 0,
 207     },
 208     unprivileged => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 212         default => 0,
 213     },
 214 };
 215
 216 my $valid_lxc_conf_keys = {
 217     'lxc.include' => 1,
 218     'lxc.arch' => 1,
 219     'lxc.utsname' => 1,
 220     'lxc.haltsignal' => 1,
 221     'lxc.rebootsignal' => 1,
 222     'lxc.stopsignal' => 1,
 223     'lxc.init_cmd' => 1,
 224     'lxc.network.type' => 1,
 225     'lxc.network.flags' => 1,
 226     'lxc.network.link' => 1,
 227     'lxc.network.mtu' => 1,
 228     'lxc.network.name' => 1,
 229     'lxc.network.hwaddr' => 1,
 230     'lxc.network.ipv4' => 1,
 231     'lxc.network.ipv4.gateway' => 1,
 232     'lxc.network.ipv6' => 1,
 233     'lxc.network.ipv6.gateway' => 1,
 234     'lxc.network.script.up' => 1,
 235     'lxc.network.script.down' => 1,
 236     'lxc.pts' => 1,
 237     'lxc.console.logfile' => 1,
 238     'lxc.console' => 1,
 239     'lxc.tty' => 1,
 240     'lxc.devttydir' => 1,
 241     'lxc.hook.autodev' => 1,
 242     'lxc.autodev' => 1,
 243     'lxc.kmsg' => 1,
 244     'lxc.mount' => 1,
 245     'lxc.mount.entry' => 1,
 246     'lxc.mount.auto' => 1,
 247     'lxc.rootfs' => 'lxc.rootfs is auto generated from rootfs',
 248     'lxc.rootfs.mount' => 1,
 249     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 250                             ', please use mountpoint options in the "rootfs" key',
 251     # lxc.cgroup.*
 252     'lxc.cap.drop' => 1,
 253     'lxc.cap.keep' => 1,
 254     'lxc.aa_profile' => 1,
 255     'lxc.aa_allow_incomplete' => 1,
 256     'lxc.se_context' => 1,
 257     'lxc.seccomp' => 1,
 258     'lxc.id_map' => 1,
 259     'lxc.hook.pre-start' => 1,
 260     'lxc.hook.pre-mount' => 1,
 261     'lxc.hook.mount' => 1,
 262     'lxc.hook.start' => 1,
 263     'lxc.hook.stop' => 1,
 264     'lxc.hook.post-stop' => 1,
 265     'lxc.hook.clone' => 1,
 266     'lxc.hook.destroy' => 1,
 267     'lxc.loglevel' => 1,
 268     'lxc.logfile' => 1,
 269     'lxc.start.auto' => 1,
 270     'lxc.start.delay' => 1,
 271     'lxc.start.order' => 1,
 272     'lxc.group' => 1,
 273     'lxc.environment' => 1,
 274 };
 275
 276 my $netconf_desc = {
 277     type => {
 278         type => 'string',
 279         optional => 1,
 280         description => "Network interface type.",
 281         enum => [qw(veth)],
 282     },
 283     name => {
 284         type => 'string',
 285         format_description => 'String',
 286         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 287         pattern => '[-_.\w\d]+',
 288     },
 289     bridge => {
 290         type => 'string',
 291         format_description => 'vmbr<Number>',
 292         description => 'Bridge to attach the network device to.',
 293         pattern => '[-_.\w\d]+',
 294         optional => 1,
 295     },
 296     hwaddr => {
 297         type => 'string',
 298         format_description => 'MAC',
 299         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 300         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 301         optional => 1,
 302     },
 303     mtu => {
 304         type => 'integer',
 305         format_description => 'Number',
 306         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 307         minimum => 64, # minimum ethernet frame is 64 bytes
 308         optional => 1,
 309     },
 310     ip => {
 311         type => 'string',
 312         format => 'pve-ipv4-config',
 313         format_description => 'IPv4Format/CIDR',
 314         description => 'IPv4 address in CIDR format.',
 315         optional => 1,
 316     },
 317     gw => {
 318         type => 'string',
 319         format => 'ipv4',
 320         format_description => 'GatewayIPv4',
 321         description => 'Default gateway for IPv4 traffic.',
 322         optional => 1,
 323     },
 324     ip6 => {
 325         type => 'string',
 326         format => 'pve-ipv6-config',
 327         format_description => 'IPv6Format/CIDR',
 328         description => 'IPv6 address in CIDR format.',
 329         optional => 1,
 330     },
 331     gw6 => {
 332         type => 'string',
 333         format => 'ipv6',
 334         format_description => 'GatewayIPv6',
 335         description => 'Default gateway for IPv6 traffic.',
 336         optional => 1,
 337     },
 338     firewall => {
 339         type => 'boolean',
 340         format_description => '[1|0]',
 341         description => "Controls whether this interface's firewall rules should be used.",
 342         optional => 1,
 343     },
 344     tag => {
 345         type => 'integer',
 346         format_description => 'VlanNo',
 347         minimum => '2',
 348         maximum => '4094',
 349         description => "VLAN tag for this interface.",
 350         optional => 1,
 351     },
 352     trunks => {
 353         type => 'string',
 354         pattern => qr/\d+(?:;\d+)*/,
 355         format_description => 'vlanid[;vlanid...]',
 356         description => "VLAN ids to pass through the interface",
 357         optional => 1,
 358     },
 359 };
 360 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 361
 362 my $MAX_LXC_NETWORKS = 10;
 363 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 364     $confdesc->{"net$i"} = {
 365         optional => 1,
 366         type => 'string', format => $netconf_desc,
 367         description => "Specifies network interfaces for the container.",
 368     };
 369 }
 370
 371 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 372 sub verify_lxc_mp_string{
 373     my ($mp, $noerr) = @_;
 374
 375     # do not allow:
 376     # /./ or /../
 377     # /. or /.. at the end
 378     # ../ at the beginning
 379
 380     if($mp =~ m@/\.\.?/@ ||
 381        $mp =~ m@/\.\.?$@ ||
 382        $mp =~ m@^\.\./@){
 383         return undef if $noerr;
 384         die "$mp contains illegal character sequences\n";
 385     }
 386     return $mp;
 387 }
 388
 389 my $mp_desc = {
 390     %$rootfs_desc,
 391     mp => {
 392         type => 'string',
 393         format => 'pve-lxc-mp-string',
 394         format_description => 'Path',
 395         description => 'Path to the mountpoint as seen from inside the container.',
 396     },
 397 };
 398 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 399
 400 my $unuseddesc = {
 401     optional => 1,
 402     type => 'string', format => 'pve-volume-id',
 403     description => "Reference to unused volumes.",
 404 };
 405
 406 my $MAX_MOUNT_POINTS = 10;
 407 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 408     $confdesc->{"mp$i"} = {
 409         optional => 1,
 410         type => 'string', format => $mp_desc,
 411         description => "Use volume as container mount point (experimental feature).",
 412         optional => 1,
 413     };
 414 }
 415
 416 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 417 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 418     $confdesc->{"unused$i"} = $unuseddesc;
 419 }
 420
 421 sub write_pct_config {
 422     my ($filename, $conf) = @_;
 423
 424     delete $conf->{snapstate}; # just to be sure
 425
 426     my $generate_raw_config = sub {
 427         my ($conf) = @_;
 428
 429         my $raw = '';
 430
 431         # add description as comment to top of file
 432         my $descr = $conf->{description} || '';
 433         foreach my $cl (split(/\n/, $descr)) {
 434             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 435         }
 436
 437         foreach my $key (sort keys %$conf) {
 438             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 439                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 440             my $value = $conf->{$key};
 441             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 442             $raw .= "$key: $value\n";
 443         }
 444
 445         if (my $lxcconf = $conf->{lxc}) {
 446             foreach my $entry (@$lxcconf) {
 447                 my ($k, $v) = @$entry;
 448                 $raw .= "$k: $v\n";
 449             }
 450         }
 451
 452         return $raw;
 453     };
 454
 455     my $raw = &$generate_raw_config($conf);
 456
 457     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 458         $raw .= "\n[$snapname]\n";
 459         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 460     }
 461
 462     return $raw;
 463 }
 464
 465 sub check_type {
 466     my ($key, $value) = @_;
 467
 468     die "unknown setting '$key'\n" if !$confdesc->{$key};
 469
 470     my $type = $confdesc->{$key}->{type};
 471
 472     if (!defined($value)) {
 473         die "got undefined value\n";
 474     }
 475
 476     if ($value =~ m/[\n\r]/) {
 477         die "property contains a line feed\n";
 478     }
 479
 480     if ($type eq 'boolean') {
 481         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 482         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 483         die "type check ('boolean') failed - got '$value'\n";
 484     } elsif ($type eq 'integer') {
 485         return int($1) if $value =~ m/^(\d+)$/;
 486         die "type check ('integer') failed - got '$value'\n";
 487     } elsif ($type eq 'number') {
 488         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 489         die "type check ('number') failed - got '$value'\n";
 490     } elsif ($type eq 'string') {
 491         if (my $fmt = $confdesc->{$key}->{format}) {
 492             PVE::JSONSchema::check_format($fmt, $value);
 493             return $value;
 494         }
 495         return $value;
 496     } else {
 497         die "internal error"
 498     }
 499 }
 500
 501 sub parse_pct_config {
 502     my ($filename, $raw) = @_;
 503
 504     return undef if !defined($raw);
 505
 506     my $res = {
 507         digest => Digest::SHA::sha1_hex($raw),
 508         snapshots => {},
 509     };
 510
 511     $filename =~ m|/lxc/(\d+).conf$|
 512         || die "got strange filename '$filename'";
 513
 514     my $vmid = $1;
 515
 516     my $conf = $res;
 517     my $descr = '';
 518     my $section = '';
 519
 520     my @lines = split(/\n/, $raw);
 521     foreach my $line (@lines) {
 522         next if $line =~ m/^\s*$/;
 523
 524         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 525             $section = $1;
 526             $conf->{description} = $descr if $descr;
 527             $descr = '';
 528             $conf = $res->{snapshots}->{$section} = {};
 529             next;
 530         }
 531
 532         if ($line =~ m/^\#(.*)\s*$/) {
 533             $descr .= PVE::Tools::decode_text($1) . "\n";
 534             next;
 535         }
 536
 537         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 538             my $key = $1;
 539             my $value = $3;
 540             my $validity = $valid_lxc_conf_keys->{$key} || 0;
 541             if ($validity eq 1 || $key =~ m/^lxc\.cgroup\./) {
 542                 push @{$conf->{lxc}}, [$key, $value];
 543             } elsif (my $errmsg = $validity) {
 544                 warn "vm $vmid - $key: $errmsg\n";
 545             } else {
 546                 warn "vm $vmid - unable to parse config: $line\n";
 547             }
 548         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 549             $descr .= PVE::Tools::decode_text($2);
 550         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 551             $conf->{snapstate} = $1;
 552         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 553             my $key = $1;
 554             my $value = $2;
 555             eval { $value = check_type($key, $value); };
 556             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 557             $conf->{$key} = $value;
 558         } else {
 559             warn "vm $vmid - unable to parse config: $line\n";
 560         }
 561     }
 562
 563     $conf->{description} = $descr if $descr;
 564
 565     delete $res->{snapstate}; # just to be sure
 566
 567     return $res;
 568 }
 569
 570 sub config_list {
 571     my $vmlist = PVE::Cluster::get_vmlist();
 572     my $res = {};
 573     return $res if !$vmlist || !$vmlist->{ids};
 574     my $ids = $vmlist->{ids};
 575
 576     foreach my $vmid (keys %$ids) {
 577         next if !$vmid; # skip CT0
 578         my $d = $ids->{$vmid};
 579         next if !$d->{node} || $d->{node} ne $nodename;
 580         next if !$d->{type} || $d->{type} ne 'lxc';
 581         $res->{$vmid}->{type} = 'lxc';
 582     }
 583     return $res;
 584 }
 585
 586 sub cfs_config_path {
 587     my ($vmid, $node) = @_;
 588
 589     $node = $nodename if !$node;
 590     return "nodes/$node/lxc/$vmid.conf";
 591 }
 592
 593 sub config_file {
 594     my ($vmid, $node) = @_;
 595
 596     my $cfspath = cfs_config_path($vmid, $node);
 597     return "/etc/pve/$cfspath";
 598 }
 599
 600 sub load_config {
 601     my ($vmid, $node) = @_;
 602
 603     $node = $nodename if !$node;
 604     my $cfspath = cfs_config_path($vmid, $node);
 605
 606     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 607     die "container $vmid does not exist\n" if !defined($conf);
 608
 609     return $conf;
 610 }
 611
 612 sub create_config {
 613     my ($vmid, $conf) = @_;
 614
 615     my $dir = "/etc/pve/nodes/$nodename/lxc";
 616     mkdir $dir;
 617
 618     write_config($vmid, $conf);
 619 }
 620
 621 sub destroy_config {
 622     my ($vmid) = @_;
 623
 624     unlink config_file($vmid, $nodename);
 625 }
 626
 627 sub write_config {
 628     my ($vmid, $conf) = @_;
 629
 630     my $cfspath = cfs_config_path($vmid);
 631
 632     PVE::Cluster::cfs_write_file($cfspath, $conf);
 633 }
 634
 635 # flock: we use one file handle per process, so lock file
 636 # can be called multiple times and will succeed for the same process.
 637
 638 my $lock_handles =  {};
 639 my $lockdir = "/run/lock/lxc";
 640
 641 sub lock_filename {
 642     my ($vmid) = @_;
 643
 644     return "$lockdir/pve-config-${vmid}.lock";
 645 }
 646
 647 sub lock_aquire {
 648     my ($vmid, $timeout) = @_;
 649
 650     $timeout = 10 if !$timeout;
 651     my $mode = LOCK_EX;
 652
 653     my $filename = lock_filename($vmid);
 654
 655     mkdir $lockdir if !-d $lockdir;
 656
 657     my $lock_func = sub {
 658         if (!$lock_handles->{$$}->{$filename}) {
 659             my $fh = new IO::File(">>$filename") ||
 660                 die "can't open file - $!\n";
 661             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 662         }
 663
 664         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 665             print STDERR "trying to aquire lock...";
 666             my $success;
 667             while(1) {
 668                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 669                 # try again on EINTR (see bug #273)
 670                 if ($success || ($! != EINTR)) {
 671                     last;
 672                 }
 673             }
 674             if (!$success) {
 675                 print STDERR " failed\n";
 676                 die "can't aquire lock - $!\n";
 677             }
 678
 679             print STDERR " OK\n";
 680         }
 681
 682         $lock_handles->{$$}->{$filename}->{refcount}++;
 683     };
 684
 685     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 686     my $err = $@;
 687     if ($err) {
 688         die "can't lock file '$filename' - $err";
 689     }
 690 }
 691
 692 sub lock_release {
 693     my ($vmid) = @_;
 694
 695     my $filename = lock_filename($vmid);
 696
 697     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 698         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 699         if ($refcount <= 0) {
 700             $lock_handles->{$$}->{$filename} = undef;
 701             close ($fh);
 702         }
 703     }
 704 }
 705
 706 sub lock_container {
 707     my ($vmid, $timeout, $code, @param) = @_;
 708
 709     my $res;
 710
 711     lock_aquire($vmid, $timeout);
 712     eval { $res = &$code(@param) };
 713     my $err = $@;
 714     lock_release($vmid);
 715
 716     die $err if $err;
 717
 718     return $res;
 719 }
 720
 721 sub option_exists {
 722     my ($name) = @_;
 723
 724     return defined($confdesc->{$name});
 725 }
 726
 727 # add JSON properties for create and set function
 728 sub json_config_properties {
 729     my $prop = shift;
 730
 731     foreach my $opt (keys %$confdesc) {
 732         next if $opt eq 'parent' || $opt eq 'snaptime';
 733         next if $prop->{$opt};
 734         $prop->{$opt} = $confdesc->{$opt};
 735     }
 736
 737     return $prop;
 738 }
 739
 740 sub json_config_properties_no_rootfs {
 741     my $prop = shift;
 742
 743     foreach my $opt (keys %$confdesc) {
 744         next if $prop->{$opt};
 745         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 746         $prop->{$opt} = $confdesc->{$opt};
 747     }
 748
 749     return $prop;
 750 }
 751
 752 # container status helpers
 753
 754 sub list_active_containers {
 755
 756     my $filename = "/proc/net/unix";
 757
 758     # similar test is used by lcxcontainers.c: list_active_containers
 759     my $res = {};
 760
 761     my $fh = IO::File->new ($filename, "r");
 762     return $res if !$fh;
 763
 764     while (defined(my $line = <$fh>)) {
 765         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 766             my $path = $1;
 767             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 768                 $res->{$1} = 1;
 769             }
 770         }
 771     }
 772
 773     close($fh);
 774
 775     return $res;
 776 }
 777
 778 # warning: this is slow
 779 sub check_running {
 780     my ($vmid) = @_;
 781
 782     my $active_hash = list_active_containers();
 783
 784     return 1 if defined($active_hash->{$vmid});
 785
 786     return undef;
 787 }
 788
 789 sub get_container_disk_usage {
 790     my ($vmid, $pid) = @_;
 791
 792     return PVE::Tools::df("/proc/$pid/root/", 1);
 793 }
 794
 795 my $last_proc_vmid_stat;
 796
 797 my $parse_cpuacct_stat = sub {
 798     my ($vmid) = @_;
 799
 800     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 801
 802     my $stat = {};
 803
 804     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 805
 806         $stat->{utime} = $1;
 807         $stat->{stime} = $2;
 808
 809     }
 810
 811     return $stat;
 812 };
 813
 814 sub vmstatus {
 815     my ($opt_vmid) = @_;
 816
 817     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 818
 819     my $active_hash = list_active_containers();
 820
 821     my $cpucount = $cpuinfo->{cpus} || 1;
 822
 823     my $cdtime = gettimeofday;
 824
 825     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 826
 827     foreach my $vmid (keys %$list) {
 828         my $d = $list->{$vmid};
 829
 830         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 831         warn $@ if $@; # ignore errors (consider them stopped)
 832
 833         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 834
 835         my $cfspath = cfs_config_path($vmid);
 836         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 837
 838         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 839         $d->{name} =~ s/[\s]//g;
 840
 841         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 842
 843         if ($d->{pid}) {
 844             my $res = get_container_disk_usage($vmid, $d->{pid});
 845             $d->{disk} = $res->{used};
 846             $d->{maxdisk} = $res->{total};
 847         } else {
 848             $d->{disk} = 0;
 849             # use 4GB by default ??
 850             if (my $rootfs = $conf->{rootfs}) {
 851                 my $rootinfo = parse_ct_rootfs($rootfs);
 852                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 853             } else {
 854                 $d->{maxdisk} = 4*1024*1024*1024;
 855             }
 856         }
 857
 858         $d->{mem} = 0;
 859         $d->{swap} = 0;
 860         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 861         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 862
 863         $d->{uptime} = 0;
 864         $d->{cpu} = 0;
 865
 866         $d->{netout} = 0;
 867         $d->{netin} = 0;
 868
 869         $d->{diskread} = 0;
 870         $d->{diskwrite} = 0;
 871
 872         $d->{template} = is_template($conf);
 873     }
 874
 875     foreach my $vmid (keys %$list) {
 876         my $d = $list->{$vmid};
 877         my $pid = $d->{pid};
 878
 879         next if !$pid; # skip stopped CTs
 880
 881         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 882         $d->{uptime} = time - $ctime; # the method lxcfs uses
 883
 884         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 885         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 886
 887         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 888         my @bytes = split(/\n/, $blkio_bytes);
 889         foreach my $byte (@bytes) {
 890             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 891                 $d->{diskread} = $2 if $key eq 'Read';
 892                 $d->{diskwrite} = $2 if $key eq 'Write';
 893             }
 894         }
 895
 896         my $pstat = &$parse_cpuacct_stat($vmid);
 897
 898         my $used = $pstat->{utime} + $pstat->{stime};
 899
 900         my $old = $last_proc_vmid_stat->{$vmid};
 901         if (!$old) {
 902             $last_proc_vmid_stat->{$vmid} = {
 903                 time => $cdtime,
 904                 used => $used,
 905                 cpu => 0,
 906             };
 907             next;
 908         }
 909
 910         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 911
 912         if ($dtime > 1000) {
 913             my $dutime = $used -  $old->{used};
 914
 915             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 916             $last_proc_vmid_stat->{$vmid} = {
 917                 time => $cdtime,
 918                 used => $used,
 919                 cpu => $d->{cpu},
 920             };
 921         } else {
 922             $d->{cpu} = $old->{cpu};
 923         }
 924     }
 925
 926     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 927
 928     foreach my $dev (keys %$netdev) {
 929         next if $dev !~ m/^veth([1-9]\d*)i/;
 930         my $vmid = $1;
 931         my $d = $list->{$vmid};
 932
 933         next if !$d;
 934
 935         $d->{netout} += $netdev->{$dev}->{receive};
 936         $d->{netin} += $netdev->{$dev}->{transmit};
 937
 938     }
 939
 940     return $list;
 941 }
 942
 943 sub classify_mountpoint {
 944     my ($vol) = @_;
 945     if ($vol =~ m!^/!) {
 946         return 'device' if $vol =~ m!^/dev/!;
 947         return 'bind';
 948     }
 949     return 'volume';
 950 }
 951
 952 my $parse_ct_mountpoint_full = sub {
 953     my ($desc, $data, $noerr) = @_;
 954
 955     $data //= '';
 956
 957     my $res;
 958     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
 959     if ($@) {
 960         return undef if $noerr;
 961         die $@;
 962     }
 963
 964     if (defined(my $size = $res->{size})) {
 965         $size = PVE::JSONSchema::parse_size($size);
 966         if (!defined($size)) {
 967             return undef if $noerr;
 968             die "invalid size: $size\n";
 969         }
 970         $res->{size} = $size;
 971     }
 972
 973     $res->{type} = classify_mountpoint($res->{volume});
 974
 975     return $res;
 976 };
 977
 978 sub parse_ct_rootfs {
 979     my ($data, $noerr) = @_;
 980
 981     my $res =  &$parse_ct_mountpoint_full($rootfs_desc, $data, $noerr);
 982
 983     $res->{mp} = '/' if defined($res);
 984
 985     return $res;
 986 }
 987
 988 sub parse_ct_mountpoint {
 989     my ($data, $noerr) = @_;
 990
 991     return &$parse_ct_mountpoint_full($mp_desc, $data, $noerr);
 992 }
 993
 994 sub print_ct_mountpoint {
 995     my ($info, $nomp) = @_;
 996     my $skip = [ 'type' ];
 997     push @$skip, 'mp' if $nomp;
 998     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 999 }
1000
1001 sub print_lxc_network {
1002     my $net = shift;
1003     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
1004 }
1005
1006 sub parse_lxc_network {
1007     my ($data) = @_;
1008
1009     my $res = {};
1010
1011     return $res if !$data;
1012
1013     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
1014
1015     $res->{type} = 'veth';
1016     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
1017
1018     return $res;
1019 }
1020
1021 sub read_cgroup_value {
1022     my ($group, $vmid, $name, $full) = @_;
1023
1024     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
1025
1026     return PVE::Tools::file_get_contents($path) if $full;
1027
1028     return PVE::Tools::file_read_firstline($path);
1029 }
1030
1031 sub write_cgroup_value {
1032    my ($group, $vmid, $name, $value) = @_;
1033
1034    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
1035    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
1036
1037 }
1038
1039 sub find_lxc_console_pids {
1040
1041     my $res = {};
1042
1043     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
1044         my ($pid) = @_;
1045
1046         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
1047         return if !$cmdline;
1048
1049         my @args = split(/\0/, $cmdline);
1050
1051         # search for lxc-console -n <vmid>
1052         return if scalar(@args) != 3;
1053         return if $args[1] ne '-n';
1054         return if $args[2] !~ m/^\d+$/;
1055         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1056
1057         my $vmid = $args[2];
1058
1059         push @{$res->{$vmid}}, $pid;
1060     });
1061
1062     return $res;
1063 }
1064
1065 sub find_lxc_pid {
1066     my ($vmid) = @_;
1067
1068     my $pid = undef;
1069     my $parser = sub {
1070         my $line = shift;
1071         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1072     };
1073     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1074
1075     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1076
1077     return $pid;
1078 }
1079
1080 # Note: we cannot use Net:IP, because that only allows strict
1081 # CIDR networks
1082 sub parse_ipv4_cidr {
1083     my ($cidr, $noerr) = @_;
1084
1085     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1086         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1087     }
1088
1089     return undef if $noerr;
1090
1091     die "unable to parse ipv4 address/mask\n";
1092 }
1093
1094 sub check_lock {
1095     my ($conf) = @_;
1096
1097     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1098 }
1099
1100 sub check_protection {
1101     my ($vm_conf, $err_msg) = @_;
1102
1103     if ($vm_conf->{protection}) {
1104         die "$err_msg - protection mode enabled\n";
1105     }
1106 }
1107
1108 sub update_lxc_config {
1109     my ($storage_cfg, $vmid, $conf) = @_;
1110
1111     my $dir = "/var/lib/lxc/$vmid";
1112
1113     if ($conf->{template}) {
1114
1115         unlink "$dir/config";
1116
1117         return;
1118     }
1119
1120     my $raw = '';
1121
1122     die "missing 'arch' - internal error" if !$conf->{arch};
1123     $raw .= "lxc.arch = $conf->{arch}\n";
1124
1125     my $unprivileged = $conf->{unprivileged};
1126     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1127
1128     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1129     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux)$/x) {
1130         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1131         if ($unprivileged || $custom_idmap) {
1132             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1133         }
1134     } else {
1135         die "implement me (ostype $ostype)";
1136     }
1137
1138     $raw .= "lxc.monitor.unshare = 1\n";
1139
1140     # Should we read them from /etc/subuid?
1141     if ($unprivileged && !$custom_idmap) {
1142         $raw .= "lxc.id_map = u 0 100000 65536\n";
1143         $raw .= "lxc.id_map = g 0 100000 65536\n";
1144     }
1145
1146     if (!has_dev_console($conf)) {
1147         $raw .= "lxc.console = none\n";
1148         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1149     }
1150
1151     my $ttycount = get_tty_count($conf);
1152     $raw .= "lxc.tty = $ttycount\n";
1153
1154     # some init scripts expect a linux terminal (turnkey).
1155     $raw .= "lxc.environment = TERM=linux\n";
1156
1157     my $utsname = $conf->{hostname} || "CT$vmid";
1158     $raw .= "lxc.utsname = $utsname\n";
1159
1160     my $memory = $conf->{memory} || 512;
1161     my $swap = $conf->{swap} // 0;
1162
1163     my $lxcmem = int($memory*1024*1024);
1164     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1165
1166     my $lxcswap = int(($memory + $swap)*1024*1024);
1167     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1168
1169     if (my $cpulimit = $conf->{cpulimit}) {
1170         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1171         my $value = int(100000*$cpulimit);
1172         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1173     }
1174
1175     my $shares = $conf->{cpuunits} || 1024;
1176     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1177
1178     my $mountpoint = parse_ct_rootfs($conf->{rootfs});
1179
1180     $raw .= "lxc.rootfs = $dir/rootfs\n";
1181
1182     my $netcount = 0;
1183     foreach my $k (keys %$conf) {
1184         next if $k !~ m/^net(\d+)$/;
1185         my $ind = $1;
1186         my $d = parse_lxc_network($conf->{$k});
1187         $netcount++;
1188         $raw .= "lxc.network.type = veth\n";
1189         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1190         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1191         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1192         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1193     }
1194
1195     if (my $lxcconf = $conf->{lxc}) {
1196         foreach my $entry (@$lxcconf) {
1197             my ($k, $v) = @$entry;
1198             $netcount++ if $k eq 'lxc.network.type';
1199             $raw .= "$k = $v\n";
1200         }
1201     }
1202
1203     $raw .= "lxc.network.type = empty\n" if !$netcount;
1204
1205     File::Path::mkpath("$dir/rootfs");
1206
1207     PVE::Tools::file_set_contents("$dir/config", $raw);
1208 }
1209
1210 # verify and cleanup nameserver list (replace \0 with ' ')
1211 sub verify_nameserver_list {
1212     my ($nameserver_list) = @_;
1213
1214     my @list = ();
1215     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1216         PVE::JSONSchema::pve_verify_ip($server);
1217         push @list, $server;
1218     }
1219
1220     return join(' ', @list);
1221 }
1222
1223 sub verify_searchdomain_list {
1224     my ($searchdomain_list) = @_;
1225
1226     my @list = ();
1227     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1228         # todo: should we add checks for valid dns domains?
1229         push @list, $server;
1230     }
1231
1232     return join(' ', @list);
1233 }
1234
1235 sub add_unused_volume {
1236     my ($config, $volid) = @_;
1237
1238     my $key;
1239     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1240         my $test = "unused$ind";
1241         if (my $vid = $config->{$test}) {
1242             return if $vid eq $volid; # do not add duplicates
1243         } else {
1244             $key = $test;
1245         }
1246     }
1247
1248     die "Too many unused volumes - please delete them first.\n" if !$key;
1249
1250     $config->{$key} = $volid;
1251
1252     return $key;
1253 }
1254
1255 sub update_pct_config {
1256     my ($vmid, $conf, $running, $param, $delete) = @_;
1257
1258     my @nohotplug;
1259
1260     my $new_disks = 0;
1261     my @deleted_volumes;
1262
1263     my $rootdir;
1264     if ($running) {
1265         my $pid = find_lxc_pid($vmid);
1266         $rootdir = "/proc/$pid/root";
1267     }
1268
1269     my $hotplug_error = sub {
1270         if ($running) {
1271             push @nohotplug, @_;
1272             return 1;
1273         } else {
1274             return 0;
1275         }
1276     };
1277
1278     if (defined($delete)) {
1279         foreach my $opt (@$delete) {
1280             if (!exists($conf->{$opt})) {
1281                 warn "no such option: $opt\n";
1282                 next;
1283             }
1284
1285             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1286                 die "unable to delete required option '$opt'\n";
1287             } elsif ($opt eq 'swap') {
1288                 delete $conf->{$opt};
1289                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1290             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1291                 delete $conf->{$opt};
1292             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1293                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1294                 next if $hotplug_error->($opt);
1295                 delete $conf->{$opt};
1296             } elsif ($opt =~ m/^net(\d)$/) {
1297                 delete $conf->{$opt};
1298                 next if !$running;
1299                 my $netid = $1;
1300                 PVE::Network::veth_delete("veth${vmid}i$netid");
1301             } elsif ($opt eq 'protection') {
1302                 delete $conf->{$opt};
1303             } elsif ($opt =~ m/^unused(\d+)$/) {
1304                 next if $hotplug_error->($opt);
1305                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1306                 push @deleted_volumes, $conf->{$opt};
1307                 delete $conf->{$opt};
1308             } elsif ($opt =~ m/^mp(\d+)$/) {
1309                 next if $hotplug_error->($opt);
1310                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1311                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1312                 if ($mountpoint->{type} eq 'volume') {
1313                     add_unused_volume($conf, $mountpoint->{volume})
1314                 }
1315                 delete $conf->{$opt};
1316             } elsif ($opt eq 'unprivileged') {
1317                 die "unable to delete read-only option: '$opt'\n";
1318             } else {
1319                 die "implement me (delete: $opt)"
1320             }
1321             write_config($vmid, $conf) if $running;
1322         }
1323     }
1324
1325     # There's no separate swap size to configure, there's memory and "total"
1326     # memory (iow. memory+swap). This means we have to change them together.
1327     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1328     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1329     if (defined($wanted_memory) || defined($wanted_swap)) {
1330
1331         my $old_memory = ($conf->{memory} || 512);
1332         my $old_swap = ($conf->{swap} || 0);
1333
1334         $wanted_memory //= $old_memory;
1335         $wanted_swap //= $old_swap;
1336
1337         my $total = $wanted_memory + $wanted_swap;
1338         if ($running) {
1339             my $old_total = $old_memory + $old_swap;
1340             if ($total > $old_total) {
1341                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1342                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1343             } else {
1344                 write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1345                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1346             }
1347         }
1348         $conf->{memory} = $wanted_memory;
1349         $conf->{swap} = $wanted_swap;
1350
1351         write_config($vmid, $conf) if $running;
1352     }
1353
1354     foreach my $opt (keys %$param) {
1355         my $value = $param->{$opt};
1356         if ($opt eq 'hostname') {
1357             $conf->{$opt} = $value;
1358         } elsif ($opt eq 'onboot') {
1359             $conf->{$opt} = $value ? 1 : 0;
1360         } elsif ($opt eq 'startup') {
1361             $conf->{$opt} = $value;
1362         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1363             next if $hotplug_error->($opt);
1364             $conf->{$opt} = $value;
1365         } elsif ($opt eq 'nameserver') {
1366             next if $hotplug_error->($opt);
1367             my $list = verify_nameserver_list($value);
1368             $conf->{$opt} = $list;
1369         } elsif ($opt eq 'searchdomain') {
1370             next if $hotplug_error->($opt);
1371             my $list = verify_searchdomain_list($value);
1372             $conf->{$opt} = $list;
1373         } elsif ($opt eq 'cpulimit') {
1374             next if $hotplug_error->($opt); # FIXME: hotplug
1375             $conf->{$opt} = $value;
1376         } elsif ($opt eq 'cpuunits') {
1377             $conf->{$opt} = $value;
1378             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1379         } elsif ($opt eq 'description') {
1380             $conf->{$opt} = PVE::Tools::encode_text($value);
1381         } elsif ($opt =~ m/^net(\d+)$/) {
1382             my $netid = $1;
1383             my $net = parse_lxc_network($value);
1384             if (!$running) {
1385                 $conf->{$opt} = print_lxc_network($net);
1386             } else {
1387                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1388             }
1389         } elsif ($opt eq 'protection') {
1390             $conf->{$opt} = $value ? 1 : 0;
1391         } elsif ($opt =~ m/^mp(\d+)$/) {
1392             next if $hotplug_error->($opt);
1393             check_protection($conf, "can't update CT $vmid drive '$opt'");
1394             $conf->{$opt} = $value;
1395             $new_disks = 1;
1396         } elsif ($opt eq 'rootfs') {
1397             check_protection($conf, "can't update CT $vmid drive '$opt'");
1398             die "implement me: $opt";
1399         } elsif ($opt eq 'unprivileged') {
1400             die "unable to modify read-only option: '$opt'\n";
1401         } else {
1402             die "implement me: $opt";
1403         }
1404         write_config($vmid, $conf) if $running;
1405     }
1406
1407     if (@deleted_volumes) {
1408         my $storage_cfg = PVE::Storage::config();
1409         foreach my $volume (@deleted_volumes) {
1410             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1411         }
1412     }
1413
1414     if ($new_disks) {
1415         my $storage_cfg = PVE::Storage::config();
1416         create_disks($storage_cfg, $vmid, $conf, $conf);
1417     }
1418
1419     # This should be the last thing we do here
1420     if ($running && scalar(@nohotplug)) {
1421         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1422     }
1423 }
1424
1425 sub has_dev_console {
1426     my ($conf) = @_;
1427
1428     return !(defined($conf->{console}) && !$conf->{console});
1429 }
1430
1431 sub get_tty_count {
1432     my ($conf) = @_;
1433
1434     return $conf->{tty} // $confdesc->{tty}->{default};
1435 }
1436
1437 sub get_cmode {
1438     my ($conf) = @_;
1439
1440     return $conf->{cmode} // $confdesc->{cmode}->{default};
1441 }
1442
1443 sub get_console_command {
1444     my ($vmid, $conf) = @_;
1445
1446     my $cmode = get_cmode($conf);
1447
1448     if ($cmode eq 'console') {
1449         return ['lxc-console', '-n',  $vmid, '-t', 0];
1450     } elsif ($cmode eq 'tty') {
1451         return ['lxc-console', '-n',  $vmid];
1452     } elsif ($cmode eq 'shell') {
1453         return ['lxc-attach', '--clear-env', '-n', $vmid];
1454     } else {
1455         die "internal error";
1456     }
1457 }
1458
1459 sub get_primary_ips {
1460     my ($conf) = @_;
1461
1462     # return data from net0
1463
1464     return undef if !defined($conf->{net0});
1465     my $net = parse_lxc_network($conf->{net0});
1466
1467     my $ipv4 = $net->{ip};
1468     if ($ipv4) {
1469         if ($ipv4 =~ /^(dhcp|manual)$/) {
1470             $ipv4 = undef
1471         } else {
1472             $ipv4 =~ s!/\d+$!!;
1473         }
1474     }
1475     my $ipv6 = $net->{ip6};
1476     if ($ipv6) {
1477         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1478             $ipv6 = undef;
1479         } else {
1480             $ipv6 =~ s!/\d+$!!;
1481         }
1482     }
1483
1484     return ($ipv4, $ipv6);
1485 }
1486
1487 sub delete_mountpoint_volume {
1488     my ($storage_cfg, $vmid, $volume) = @_;
1489
1490     return if classify_mountpoint($volume) ne 'volume';
1491
1492     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1493     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1494 }
1495
1496 sub destroy_lxc_container {
1497     my ($storage_cfg, $vmid, $conf) = @_;
1498
1499     foreach_mountpoint($conf, sub {
1500         my ($ms, $mountpoint) = @_;
1501         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1502     });
1503
1504     rmdir "/var/lib/lxc/$vmid/rootfs";
1505     unlink "/var/lib/lxc/$vmid/config";
1506     rmdir "/var/lib/lxc/$vmid";
1507     destroy_config($vmid);
1508
1509     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1510     #PVE::Tools::run_command($cmd);
1511 }
1512
1513 sub vm_stop_cleanup {
1514     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1515
1516     eval {
1517         if (!$keepActive) {
1518
1519             my $vollist = get_vm_volumes($conf);
1520             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1521         }
1522     };
1523     warn $@ if $@; # avoid errors - just warn
1524 }
1525
1526 my $safe_num_ne = sub {
1527     my ($a, $b) = @_;
1528
1529     return 0 if !defined($a) && !defined($b);
1530     return 1 if !defined($a);
1531     return 1 if !defined($b);
1532
1533     return $a != $b;
1534 };
1535
1536 my $safe_string_ne = sub {
1537     my ($a, $b) = @_;
1538
1539     return 0 if !defined($a) && !defined($b);
1540     return 1 if !defined($a);
1541     return 1 if !defined($b);
1542
1543     return $a ne $b;
1544 };
1545
1546 sub update_net {
1547     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1548
1549     if ($newnet->{type} ne 'veth') {
1550         # for when there are physical interfaces
1551         die "cannot update interface of type $newnet->{type}";
1552     }
1553
1554     my $veth = "veth${vmid}i${netid}";
1555     my $eth = $newnet->{name};
1556
1557     if (my $oldnetcfg = $conf->{$opt}) {
1558         my $oldnet = parse_lxc_network($oldnetcfg);
1559
1560         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1561             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1562
1563             PVE::Network::veth_delete($veth);
1564             delete $conf->{$opt};
1565             write_config($vmid, $conf);
1566
1567             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1568
1569         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1570                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1571                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1572
1573                 if ($oldnet->{bridge}) {
1574                     PVE::Network::tap_unplug($veth);
1575                     foreach (qw(bridge tag firewall)) {
1576                         delete $oldnet->{$_};
1577                     }
1578                     $conf->{$opt} = print_lxc_network($oldnet);
1579                     write_config($vmid, $conf);
1580                 }
1581
1582                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1583                 foreach (qw(bridge tag firewall)) {
1584                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1585                 }
1586                 $conf->{$opt} = print_lxc_network($oldnet);
1587                 write_config($vmid, $conf);
1588         }
1589     } else {
1590         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1591     }
1592
1593     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1594 }
1595
1596 sub hotplug_net {
1597     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1598
1599     my $veth = "veth${vmid}i${netid}";
1600     my $vethpeer = $veth . "p";
1601     my $eth = $newnet->{name};
1602
1603     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1604     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
1605
1606     # attach peer in container
1607     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1608     PVE::Tools::run_command($cmd);
1609
1610     # link up peer in container
1611     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1612     PVE::Tools::run_command($cmd);
1613
1614     my $done = { type => 'veth' };
1615     foreach (qw(bridge tag firewall hwaddr name)) {
1616         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1617     }
1618     $conf->{$opt} = print_lxc_network($done);
1619
1620     write_config($vmid, $conf);
1621 }
1622
1623 sub update_ipconfig {
1624     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1625
1626     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1627
1628     my $optdata = parse_lxc_network($conf->{$opt});
1629     my $deleted = [];
1630     my $added = [];
1631     my $nscmd = sub {
1632         my $cmdargs = shift;
1633         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1634     };
1635     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1636
1637     my $change_ip_config = sub {
1638         my ($ipversion) = @_;
1639
1640         my $family_opt = "-$ipversion";
1641         my $suffix = $ipversion == 4 ? '' : $ipversion;
1642         my $gw= "gw$suffix";
1643         my $ip= "ip$suffix";
1644
1645         my $newip = $newnet->{$ip};
1646         my $newgw = $newnet->{$gw};
1647         my $oldip = $optdata->{$ip};
1648
1649         my $change_ip = &$safe_string_ne($oldip, $newip);
1650         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1651
1652         return if !$change_ip && !$change_gw;
1653
1654         # step 1: add new IP, if this fails we cancel
1655         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1656         if ($change_ip && $is_real_ip) {
1657             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1658             if (my $err = $@) {
1659                 warn $err;
1660                 return;
1661             }
1662         }
1663
1664         # step 2: replace gateway
1665         #   If this fails we delete the added IP and cancel.
1666         #   If it succeeds we save the config and delete the old IP, ignoring
1667         #   errors. The config is then saved.
1668         # Note: 'ip route replace' can add
1669         if ($change_gw) {
1670             if ($newgw) {
1671                 eval {
1672                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1673                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1674                     }
1675                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1676                 };
1677                 if (my $err = $@) {
1678                     warn $err;
1679                     # the route was not replaced, the old IP is still available
1680                     # rollback (delete new IP) and cancel
1681                     if ($change_ip) {
1682                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1683                         warn $@ if $@; # no need to die here
1684                     }
1685                     return;
1686                 }
1687             } else {
1688                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1689                 # if the route was not deleted, the guest might have deleted it manually
1690                 # warn and continue
1691                 warn $@ if $@;
1692             }
1693         }
1694
1695         # from this point on we save the configuration
1696         # step 3: delete old IP ignoring errors
1697         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1698             # We need to enable promote_secondaries, otherwise our newly added
1699             # address will be removed along with the old one.
1700             my $promote = 0;
1701             eval {
1702                 if ($ipversion == 4) {
1703                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1704                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1705                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1706                 }
1707                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1708             };
1709             warn $@ if $@; # no need to die here
1710
1711             if ($ipversion == 4) {
1712                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1713             }
1714         }
1715
1716         foreach my $property ($ip, $gw) {
1717             if ($newnet->{$property}) {
1718                 $optdata->{$property} = $newnet->{$property};
1719             } else {
1720                 delete $optdata->{$property};
1721             }
1722         }
1723         $conf->{$opt} = print_lxc_network($optdata);
1724         write_config($vmid, $conf);
1725         $lxc_setup->setup_network($conf);
1726     };
1727
1728     &$change_ip_config(4);
1729     &$change_ip_config(6);
1730
1731 }
1732
1733 # Internal snapshots
1734
1735 # NOTE: Snapshot create/delete involves several non-atomic
1736 # actions, and can take a long time.
1737 # So we try to avoid locking the file and use the 'lock' variable
1738 # inside the config file instead.
1739
1740 my $snapshot_copy_config = sub {
1741     my ($source, $dest) = @_;
1742
1743     foreach my $k (keys %$source) {
1744         next if $k eq 'snapshots';
1745         next if $k eq 'snapstate';
1746         next if $k eq 'snaptime';
1747         next if $k eq 'vmstate';
1748         next if $k eq 'lock';
1749         next if $k eq 'digest';
1750         next if $k eq 'description';
1751
1752         $dest->{$k} = $source->{$k};
1753     }
1754 };
1755
1756 my $snapshot_prepare = sub {
1757     my ($vmid, $snapname, $comment) = @_;
1758
1759     my $snap;
1760
1761     my $updatefn =  sub {
1762
1763         my $conf = load_config($vmid);
1764
1765         die "you can't take a snapshot if it's a template\n"
1766             if is_template($conf);
1767
1768         check_lock($conf);
1769
1770         $conf->{lock} = 'snapshot';
1771
1772         die "snapshot name '$snapname' already used\n"
1773             if defined($conf->{snapshots}->{$snapname});
1774
1775         my $storecfg = PVE::Storage::config();
1776         my $feature = $snapname eq 'vzdump' ? 'vzdump' : 'snapshot';
1777         die "snapshot feature is not available\n" if !has_feature($feature, $conf, $storecfg);
1778
1779         $snap = $conf->{snapshots}->{$snapname} = {};
1780
1781         &$snapshot_copy_config($conf, $snap);
1782
1783         $snap->{'snapstate'} = "prepare";
1784         $snap->{'snaptime'} = time();
1785         $snap->{'description'} = $comment if $comment;
1786         $conf->{snapshots}->{$snapname} = $snap;
1787
1788         write_config($vmid, $conf);
1789     };
1790
1791     lock_container($vmid, 10, $updatefn);
1792
1793     return $snap;
1794 };
1795
1796 my $snapshot_commit = sub {
1797     my ($vmid, $snapname) = @_;
1798
1799     my $updatefn = sub {
1800
1801         my $conf = load_config($vmid);
1802
1803         die "missing snapshot lock\n"
1804             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1805
1806         die "snapshot '$snapname' does not exist\n"
1807             if !defined($conf->{snapshots}->{$snapname});
1808
1809         die "wrong snapshot state\n"
1810             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1811                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1812
1813         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1814         delete $conf->{lock};
1815         $conf->{parent} = $snapname;
1816
1817         write_config($vmid, $conf);
1818     };
1819
1820     lock_container($vmid, 10 ,$updatefn);
1821 };
1822
1823 sub has_feature {
1824     my ($feature, $conf, $storecfg, $snapname) = @_;
1825
1826     my $err;
1827     my $vzdump = $feature eq 'vzdump';
1828     $feature = 'snapshot' if $vzdump;
1829
1830     foreach_mountpoint($conf, sub {
1831         my ($ms, $mountpoint) = @_;
1832
1833         return if $err; # skip further test
1834         return if $vzdump && $ms ne 'rootfs' && !$mountpoint->{backup};
1835
1836         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1837
1838         # TODO: implement support for mountpoints
1839         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1840             if $ms ne 'rootfs';
1841     });
1842
1843     return $err ? 0 : 1;
1844 }
1845
1846 sub snapshot_create {
1847     my ($vmid, $snapname, $comment) = @_;
1848
1849     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1850
1851     my $conf = load_config($vmid);
1852
1853     my $running = check_running($vmid);
1854
1855     my $unfreeze = 0;
1856
1857     eval {
1858         if ($running) {
1859             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1860             $unfreeze = 1;
1861             PVE::Tools::run_command(['/bin/sync']);
1862         };
1863
1864         my $storecfg = PVE::Storage::config();
1865         my $rootinfo = parse_ct_rootfs($conf->{rootfs});
1866         my $volid = $rootinfo->{volume};
1867
1868         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1869         &$snapshot_commit($vmid, $snapname);
1870     };
1871     my $err = $@;
1872
1873     if ($unfreeze) {
1874         eval { PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]); };
1875         warn $@ if $@;
1876     }
1877
1878     if ($err) {
1879         snapshot_delete($vmid, $snapname, 1);
1880         die "$err\n";
1881     }
1882 }
1883
1884 sub snapshot_delete {
1885     my ($vmid, $snapname, $force) = @_;
1886
1887     my $snap;
1888
1889     my $conf;
1890
1891     my $updatefn =  sub {
1892
1893         $conf = load_config($vmid);
1894
1895         die "you can't delete a snapshot if vm is a template\n"
1896             if is_template($conf);
1897
1898         $snap = $conf->{snapshots}->{$snapname};
1899
1900         check_lock($conf);
1901
1902         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1903
1904         $snap->{snapstate} = 'delete';
1905
1906         write_config($vmid, $conf);
1907     };
1908
1909     lock_container($vmid, 10, $updatefn);
1910
1911     my $storecfg = PVE::Storage::config();
1912
1913     my $unlink_parent = sub {
1914
1915         my ($confref, $new_parent) = @_;
1916
1917         if ($confref->{parent} && $confref->{parent} eq $snapname) {
1918             if ($new_parent) {
1919                 $confref->{parent} = $new_parent;
1920             } else {
1921                 delete $confref->{parent};
1922             }
1923         }
1924     };
1925
1926     my $del_snap =  sub {
1927
1928         check_lock($conf);
1929
1930         my $parent = $conf->{snapshots}->{$snapname}->{parent};
1931         foreach my $snapkey (keys %{$conf->{snapshots}}) {
1932             &$unlink_parent($conf->{snapshots}->{$snapkey}, $parent);
1933         }
1934
1935         &$unlink_parent($conf, $parent);
1936
1937         delete $conf->{snapshots}->{$snapname};
1938
1939         write_config($vmid, $conf);
1940     };
1941
1942     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1943     my $rootinfo = parse_ct_rootfs($rootfs);
1944     my $volid = $rootinfo->{volume};
1945
1946     eval {
1947         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1948     };
1949     my $err = $@;
1950
1951     if(!$err || ($err && $force)) {
1952         lock_container($vmid, 10, $del_snap);
1953         if ($err) {
1954             die "Can't delete snapshot: $vmid $snapname $err\n";
1955         }
1956     }
1957 }
1958
1959 sub snapshot_rollback {
1960     my ($vmid, $snapname) = @_;
1961
1962     my $storecfg = PVE::Storage::config();
1963
1964     my $conf = load_config($vmid);
1965
1966     die "you can't rollback if vm is a template\n" if is_template($conf);
1967
1968     my $snap = $conf->{snapshots}->{$snapname};
1969
1970     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1971
1972     my $rootfs = $snap->{rootfs};
1973     my $rootinfo = parse_ct_rootfs($rootfs);
1974     my $volid = $rootinfo->{volume};
1975
1976     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1977
1978     my $updatefn = sub {
1979
1980         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1981             if $snap->{snapstate};
1982
1983         check_lock($conf);
1984
1985         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1986
1987         die "unable to rollback vm $vmid: vm is running\n"
1988             if check_running($vmid);
1989
1990         $conf->{lock} = 'rollback';
1991
1992         my $forcemachine;
1993
1994         # copy snapshot config to current config
1995
1996         my $tmp_conf = $conf;
1997         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1998         $conf->{snapshots} = $tmp_conf->{snapshots};
1999         delete $conf->{snaptime};
2000         delete $conf->{snapname};
2001         $conf->{parent} = $snapname;
2002
2003         write_config($vmid, $conf);
2004     };
2005
2006     my $unlockfn = sub {
2007         delete $conf->{lock};
2008         write_config($vmid, $conf);
2009     };
2010
2011     lock_container($vmid, 10, $updatefn);
2012
2013     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
2014
2015     lock_container($vmid, 5, $unlockfn);
2016 }
2017
2018 sub template_create {
2019     my ($vmid, $conf) = @_;
2020
2021     my $storecfg = PVE::Storage::config();
2022
2023     my $rootinfo = parse_ct_rootfs($conf->{rootfs});
2024     my $volid = $rootinfo->{volume};
2025
2026     die "Template feature is not available for '$volid'\n"
2027         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
2028
2029     PVE::Storage::activate_volumes($storecfg, [$volid]);
2030
2031     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
2032     $rootinfo->{volume} = $template_volid;
2033     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
2034
2035     write_config($vmid, $conf);
2036 }
2037
2038 sub is_template {
2039     my ($conf) = @_;
2040
2041     return 1 if defined $conf->{template} && $conf->{template} == 1;
2042 }
2043
2044 sub mountpoint_names {
2045     my ($reverse) = @_;
2046
2047     my @names = ('rootfs');
2048
2049     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
2050         push @names, "mp$i";
2051     }
2052
2053     return $reverse ? reverse @names : @names;
2054 }
2055
2056
2057 sub foreach_mountpoint_full {
2058     my ($conf, $reverse, $func) = @_;
2059
2060     foreach my $key (mountpoint_names($reverse)) {
2061         my $value = $conf->{$key};
2062         next if !defined($value);
2063         my $mountpoint = $key eq 'rootfs' ? parse_ct_rootfs($value, 1) : parse_ct_mountpoint($value, 1);
2064         next if !defined($mountpoint);
2065
2066         &$func($key, $mountpoint);
2067     }
2068 }
2069
2070 sub foreach_mountpoint {
2071     my ($conf, $func) = @_;
2072
2073     foreach_mountpoint_full($conf, 0, $func);
2074 }
2075
2076 sub foreach_mountpoint_reverse {
2077     my ($conf, $func) = @_;
2078
2079     foreach_mountpoint_full($conf, 1, $func);
2080 }
2081
2082 sub check_ct_modify_config_perm {
2083     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2084
2085     return 1 if $authuser ne 'root@pam';
2086
2087     foreach my $opt (@$key_list) {
2088
2089         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2090             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2091         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2092             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2093         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2094             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2095         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2096                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2097             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2098         } else {
2099             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2100         }
2101     }
2102
2103     return 1;
2104 }
2105
2106 sub umount_all {
2107     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2108
2109     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2110     my $volid_list = get_vm_volumes($conf);
2111
2112     foreach_mountpoint_reverse($conf, sub {
2113         my ($ms, $mountpoint) = @_;
2114
2115         my $volid = $mountpoint->{volume};
2116         my $mount = $mountpoint->{mp};
2117
2118         return if !$volid || !$mount;
2119
2120         my $mount_path = "$rootdir/$mount";
2121         $mount_path =~ s!/+!/!g;
2122
2123         return if !PVE::ProcFSTools::is_mounted($mount_path);
2124
2125         eval {
2126             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2127         };
2128         if (my $err = $@) {
2129             if ($noerr) {
2130                 warn $err;
2131             } else {
2132                 die $err;
2133             }
2134         }
2135     });
2136 }
2137
2138 sub mount_all {
2139     my ($vmid, $storage_cfg, $conf) = @_;
2140
2141     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2142     File::Path::make_path($rootdir);
2143
2144     my $volid_list = get_vm_volumes($conf);
2145     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2146
2147     eval {
2148         foreach_mountpoint($conf, sub {
2149             my ($ms, $mountpoint) = @_;
2150
2151             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2152         });
2153     };
2154     if (my $err = $@) {
2155         warn "mounting container failed\n";
2156         umount_all($vmid, $storage_cfg, $conf, 1);
2157         die $err;
2158     }
2159
2160     return $rootdir;
2161 }
2162
2163
2164 sub mountpoint_mount_path {
2165     my ($mountpoint, $storage_cfg, $snapname) = @_;
2166
2167     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2168 }
2169
2170 my $check_mount_path = sub {
2171     my ($path) = @_;
2172     $path = File::Spec->canonpath($path);
2173     my $real = Cwd::realpath($path);
2174     if ($real ne $path) {
2175         die "mount path modified by symlink: $path != $real";
2176     }
2177 };
2178
2179 sub query_loopdev {
2180     my ($path) = @_;
2181     my $found;
2182     my $parser = sub {
2183         my $line = shift;
2184         if ($line =~ m@^(/dev/loop\d+):@) {
2185             $found = $1;
2186         }
2187     };
2188     my $cmd = ['losetup', '--associated', $path];
2189     PVE::Tools::run_command($cmd, outfunc => $parser);
2190     return $found;
2191 }
2192
2193 # use $rootdir = undef to just return the corresponding mount path
2194 sub mountpoint_mount {
2195     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2196
2197     my $volid = $mountpoint->{volume};
2198     my $mount = $mountpoint->{mp};
2199     my $type = $mountpoint->{type};
2200
2201     return if !$volid || !$mount;
2202
2203     my $mount_path;
2204
2205     if (defined($rootdir)) {
2206         $rootdir =~ s!/+$!!;
2207         $mount_path = "$rootdir/$mount";
2208         $mount_path =~ s!/+!/!g;
2209         &$check_mount_path($mount_path);
2210         File::Path::mkpath($mount_path);
2211     }
2212
2213     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2214
2215     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2216
2217     my $optstring = '';
2218     if (defined($mountpoint->{acl})) {
2219         $optstring .= ($mountpoint->{acl} ? 'acl' : 'noacl');
2220     }
2221     if ($mountpoint->{ro}) {
2222         $optstring .= ',' if $optstring;
2223         $optstring .= 'ro';
2224     }
2225
2226     my @extra_opts = ('-o', $optstring);
2227
2228     if ($storage) {
2229
2230         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2231         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2232
2233         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2234             PVE::Storage::parse_volname($storage_cfg, $volid);
2235
2236         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2237
2238         if ($format eq 'subvol') {
2239             if ($mount_path) {
2240                 if ($snapname) {
2241                     if ($scfg->{type} eq 'zfspool') {
2242                         my $path_arg = $path;
2243                         $path_arg =~ s!^/+!!;
2244                         PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, '-t', 'zfs', $path_arg, $mount_path]);
2245                     } else {
2246                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2247                     }
2248                 } else {
2249                     if ($mountpoint->{ro}) {
2250                         die "read-only bind mounts not supported\n";
2251                     }
2252                     PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $path, $mount_path]);
2253                 }
2254             }
2255             return wantarray ? ($path, 0) : $path;
2256         } elsif ($format eq 'raw' || $format eq 'iso') {
2257             my $use_loopdev = 0;
2258             if ($scfg->{path}) {
2259                 push @extra_opts, '-o', 'loop';
2260                 $use_loopdev = 1;
2261             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2262                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2263                 # do nothing
2264             } else {
2265                 die "unsupported storage type '$scfg->{type}'\n";
2266             }
2267             if ($mount_path) {
2268                 if ($format eq 'iso') {
2269                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2270                 } elsif ($isBase || defined($snapname)) {
2271                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2272                 } else {
2273                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2274                 }
2275             }
2276             return wantarray ? ($path, $use_loopdev) : $path;
2277         } else {
2278             die "unsupported image format '$format'\n";
2279         }
2280     } elsif ($type eq 'device') {
2281         PVE::Tools::run_command(['mount', @extra_opts, $volid, $mount_path]) if $mount_path;
2282         return wantarray ? ($volid, 0) : $volid;
2283     } elsif ($type eq 'bind') {
2284         if ($mountpoint->{ro}) {
2285             die "read-only bind mounts not supported\n";
2286             # Theoretically we'd have to execute both:
2287             # mount -o bind $a $b
2288             # mount -o bind,remount,ro $a $b
2289         }
2290         die "directory '$volid' does not exist\n" if ! -d $volid;
2291         &$check_mount_path($volid);
2292         PVE::Tools::run_command(['mount', '-o', 'bind', @extra_opts, $volid, $mount_path]) if $mount_path;
2293         return wantarray ? ($volid, 0) : $volid;
2294     }
2295
2296     die "unsupported storage";
2297 }
2298
2299 sub get_vm_volumes {
2300     my ($conf, $excludes) = @_;
2301
2302     my $vollist = [];
2303
2304     foreach_mountpoint($conf, sub {
2305         my ($ms, $mountpoint) = @_;
2306
2307         return if $excludes && $ms eq $excludes;
2308
2309         my $volid = $mountpoint->{volume};
2310
2311         return if !$volid || $mountpoint->{type} ne 'volume';
2312
2313         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2314         return if !$sid;
2315
2316         push @$vollist, $volid;
2317     });
2318
2319     return $vollist;
2320 }
2321
2322 sub mkfs {
2323     my ($dev, $rootuid, $rootgid) = @_;
2324
2325     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2326                              '-E', "root_owner=$rootuid:$rootgid",
2327                              $dev]);
2328 }
2329
2330 sub format_disk {
2331     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2332
2333     if ($volid =~ m!^/dev/.+!) {
2334         mkfs($volid);
2335         return;
2336     }
2337
2338     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2339
2340     die "cannot format volume '$volid' with no storage\n" if !$storage;
2341
2342     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2343
2344     my $path = PVE::Storage::path($storage_cfg, $volid);
2345
2346     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2347         PVE::Storage::parse_volname($storage_cfg, $volid);
2348
2349     die "cannot format volume '$volid' (format == $format)\n"
2350         if $format ne 'raw';
2351
2352     mkfs($path, $rootuid, $rootgid);
2353 }
2354
2355 sub destroy_disks {
2356     my ($storecfg, $vollist) = @_;
2357
2358     foreach my $volid (@$vollist) {
2359         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2360         warn $@ if $@;
2361     }
2362 }
2363
2364 sub create_disks {
2365     my ($storecfg, $vmid, $settings, $conf) = @_;
2366
2367     my $vollist = [];
2368
2369     eval {
2370         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2371         my $chown_vollist = [];
2372
2373         foreach_mountpoint($settings, sub {
2374             my ($ms, $mountpoint) = @_;
2375
2376             my $volid = $mountpoint->{volume};
2377             my $mp = $mountpoint->{mp};
2378
2379             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2380
2381             if ($storage && ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/)) {
2382                 my ($storeid, $size_gb) = ($1, $2);
2383
2384                 my $size_kb = int(${size_gb}*1024) * 1024;
2385
2386                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2387                 # fixme: use better naming ct-$vmid-disk-X.raw?
2388
2389                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2390                     if ($size_kb > 0) {
2391                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2392                                                            undef, $size_kb);
2393                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2394                     } else {
2395                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2396                                                            undef, 0);
2397                         push @$chown_vollist, $volid;
2398                     }
2399                 } elsif ($scfg->{type} eq 'zfspool') {
2400
2401                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2402                                                        undef, $size_kb);
2403                     push @$chown_vollist, $volid;
2404                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2405
2406                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2407                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2408
2409                 } elsif ($scfg->{type} eq 'rbd') {
2410
2411                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2412                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2413                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2414                 } else {
2415                     die "unable to create containers on storage type '$scfg->{type}'\n";
2416                 }
2417                 push @$vollist, $volid;
2418                 $mountpoint->{volume} = $volid;
2419                 $mountpoint->{size} = $size_kb * 1024;
2420                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2421             } else {
2422                 # use specified/existing volid/dir/device
2423                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2424             }
2425         });
2426
2427         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2428         foreach my $volid (@$chown_vollist) {
2429             my $path = PVE::Storage::path($storecfg, $volid, undef);
2430             chown($rootuid, $rootgid, $path);
2431         }
2432         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2433     };
2434     # free allocated images on error
2435     if (my $err = $@) {
2436         destroy_disks($storecfg, $vollist);
2437         die $err;
2438     }
2439     return $vollist;
2440 }
2441
2442 # bash completion helper
2443
2444 sub complete_os_templates {
2445     my ($cmdname, $pname, $cvalue) = @_;
2446
2447     my $cfg = PVE::Storage::config();
2448
2449     my $storeid;
2450
2451     if ($cvalue =~ m/^([^:]+):/) {
2452         $storeid = $1;
2453     }
2454
2455     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2456     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2457
2458     my $res = [];
2459     foreach my $id (keys %$data) {
2460         foreach my $item (@{$data->{$id}}) {
2461             push @$res, $item->{volid} if defined($item->{volid});
2462         }
2463     }
2464
2465     return $res;
2466 }
2467
2468 my $complete_ctid_full = sub {
2469     my ($running) = @_;
2470
2471     my $idlist = vmstatus();
2472
2473     my $active_hash = list_active_containers();
2474
2475     my $res = [];
2476
2477     foreach my $id (keys %$idlist) {
2478         my $d = $idlist->{$id};
2479         if (defined($running)) {
2480             next if $d->{template};
2481             next if $running && !$active_hash->{$id};
2482             next if !$running && $active_hash->{$id};
2483         }
2484         push @$res, $id;
2485
2486     }
2487     return $res;
2488 };
2489
2490 sub complete_ctid {
2491     return &$complete_ctid_full();
2492 }
2493
2494 sub complete_ctid_stopped {
2495     return &$complete_ctid_full(0);
2496 }
2497
2498 sub complete_ctid_running {
2499     return &$complete_ctid_full(1);
2500 }
2501
2502 sub parse_id_maps {
2503     my ($conf) = @_;
2504
2505     my $id_map = [];
2506     my $rootuid = 0;
2507     my $rootgid = 0;
2508
2509     my $lxc = $conf->{lxc};
2510     foreach my $entry (@$lxc) {
2511         my ($key, $value) = @$entry;
2512         next if $key ne 'lxc.id_map';
2513         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2514             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2515             push @$id_map, [$type, $ct, $host, $length];
2516             if ($ct == 0) {
2517                 $rootuid = $host if $type eq 'u';
2518                 $rootgid = $host if $type eq 'g';
2519             }
2520         } else {
2521             die "failed to parse id_map: $value\n";
2522         }
2523     }
2524
2525     if (!@$id_map && $conf->{unprivileged}) {
2526         # Should we read them from /etc/subuid?
2527         $id_map = [ ['u', '0', '100000', '65536'],
2528                     ['g', '0', '100000', '65536'] ];
2529         $rootuid = $rootgid = 100000;
2530     }
2531
2532     return ($id_map, $rootuid, $rootgid);
2533 }
2534
2535 sub userns_command {
2536     my ($id_map) = @_;
2537     if (@$id_map) {
2538         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2539     }
2540     return [];
2541 }
2542
2543 1;