src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format_description => 'volume',
  42         description => 'Volume, device or directory to mount into the container.',
  43     },
  44     backup => {
  45         type => 'boolean',
  46         format_description => '[1|0]',
  47         description => 'Whether to include the mountpoint in backups.',
  48         optional => 1,
  49     },
  50     size => {
  51         type => 'string',
  52         format => 'disk-size',
  53         format_description => 'DiskSize',
  54         description => 'Volume size (read only value).',
  55         optional => 1,
  56     },
  57 };
  58
  59 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  60     type => 'string', format => $rootfs_desc,
  61     description => "Use volume as container root.",
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 my $confdesc = {
  72     lock => {
  73         optional => 1,
  74         type => 'string',
  75         description => "Lock/unlock the VM.",
  76         enum => [qw(migrate backup snapshot rollback)],
  77     },
  78     onboot => {
  79         optional => 1,
  80         type => 'boolean',
  81         description => "Specifies whether a VM will be started during system bootup.",
  82         default => 0,
  83     },
  84     startup => get_standard_option('pve-startup-order'),
  85     template => {
  86         optional => 1,
  87         type => 'boolean',
  88         description => "Enable/disable Template.",
  89         default => 0,
  90     },
  91     arch => {
  92         optional => 1,
  93         type => 'string',
  94         enum => ['amd64', 'i386'],
  95         description => "OS architecture type.",
  96         default => 'amd64',
  97     },
  98     ostype => {
  99         optional => 1,
 100         type => 'string',
 101         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
 102         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 103     },
 104     console => {
 105         optional => 1,
 106         type => 'boolean',
 107         description => "Attach a console device (/dev/console) to the container.",
 108         default => 1,
 109     },
 110     tty => {
 111         optional => 1,
 112         type => 'integer',
 113         description => "Specify the number of tty available to the container",
 114         minimum => 0,
 115         maximum => 6,
 116         default => 2,
 117     },
 118     cpulimit => {
 119         optional => 1,
 120         type => 'number',
 121         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 122         minimum => 0,
 123         maximum => 128,
 124         default => 0,
 125     },
 126     cpuunits => {
 127         optional => 1,
 128         type => 'integer',
 129         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 130         minimum => 0,
 131         maximum => 500000,
 132         default => 1024,
 133     },
 134     memory => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Amount of RAM for the VM in MB.",
 138         minimum => 16,
 139         default => 512,
 140     },
 141     swap => {
 142         optional => 1,
 143         type => 'integer',
 144         description => "Amount of SWAP for the VM in MB.",
 145         minimum => 0,
 146         default => 512,
 147     },
 148     hostname => {
 149         optional => 1,
 150         description => "Set a host name for the container.",
 151         type => 'string', format => 'dns-name',
 152         maxLength => 255,
 153     },
 154     description => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Container description. Only used on the configuration web interface.",
 158     },
 159     searchdomain => {
 160         optional => 1,
 161         type => 'string', format => 'dns-name-list',
 162         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 163     },
 164     nameserver => {
 165         optional => 1,
 166         type => 'string', format => 'address-list',
 167         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 168     },
 169     rootfs => get_standard_option('pve-ct-rootfs'),
 170     parent => {
 171         optional => 1,
 172         type => 'string', format => 'pve-configid',
 173         maxLength => 40,
 174         description => "Parent snapshot name. This is used internally, and should not be modified.",
 175     },
 176     snaptime => {
 177         optional => 1,
 178         description => "Timestamp for snapshots.",
 179         type => 'integer',
 180         minimum => 0,
 181     },
 182     cmode => {
 183         optional => 1,
 184         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 185         type => 'string',
 186         enum => ['shell', 'console', 'tty'],
 187         default => 'tty',
 188     },
 189     protection => {
 190         optional => 1,
 191         type => 'boolean',
 192         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 193         default => 0,
 194     },
 195     unprivileged => {
 196         optional => 1,
 197         type => 'boolean',
 198         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 199         default => 0,
 200     },
 201 };
 202
 203 my $valid_lxc_conf_keys = {
 204     'lxc.include' => 1,
 205     'lxc.arch' => 1,
 206     'lxc.utsname' => 1,
 207     'lxc.haltsignal' => 1,
 208     'lxc.rebootsignal' => 1,
 209     'lxc.stopsignal' => 1,
 210     'lxc.init_cmd' => 1,
 211     'lxc.network.type' => 1,
 212     'lxc.network.flags' => 1,
 213     'lxc.network.link' => 1,
 214     'lxc.network.mtu' => 1,
 215     'lxc.network.name' => 1,
 216     'lxc.network.hwaddr' => 1,
 217     'lxc.network.ipv4' => 1,
 218     'lxc.network.ipv4.gateway' => 1,
 219     'lxc.network.ipv6' => 1,
 220     'lxc.network.ipv6.gateway' => 1,
 221     'lxc.network.script.up' => 1,
 222     'lxc.network.script.down' => 1,
 223     'lxc.pts' => 1,
 224     'lxc.console.logfile' => 1,
 225     'lxc.console' => 1,
 226     'lxc.tty' => 1,
 227     'lxc.devttydir' => 1,
 228     'lxc.hook.autodev' => 1,
 229     'lxc.autodev' => 1,
 230     'lxc.kmsg' => 1,
 231     'lxc.mount' => 1,
 232     'lxc.mount.entry' => 1,
 233     'lxc.mount.auto' => 1,
 234     'lxc.rootfs' => 1,
 235     'lxc.rootfs.mount' => 1,
 236     'lxc.rootfs.options' => 1,
 237     # lxc.cgroup.*
 238     'lxc.cap.drop' => 1,
 239     'lxc.cap.keep' => 1,
 240     'lxc.aa_profile' => 1,
 241     'lxc.aa_allow_incomplete' => 1,
 242     'lxc.se_context' => 1,
 243     'lxc.seccomp' => 1,
 244     'lxc.id_map' => 1,
 245     'lxc.hook.pre-start' => 1,
 246     'lxc.hook.pre-mount' => 1,
 247     'lxc.hook.mount' => 1,
 248     'lxc.hook.start' => 1,
 249     'lxc.hook.stop' => 1,
 250     'lxc.hook.post-stop' => 1,
 251     'lxc.hook.clone' => 1,
 252     'lxc.hook.destroy' => 1,
 253     'lxc.loglevel' => 1,
 254     'lxc.logfile' => 1,
 255     'lxc.start.auto' => 1,
 256     'lxc.start.delay' => 1,
 257     'lxc.start.order' => 1,
 258     'lxc.group' => 1,
 259     'lxc.environment' => 1,
 260     'lxc.' => 1,
 261     'lxc.' => 1,
 262     'lxc.' => 1,
 263     'lxc.' => 1,
 264 };
 265
 266 my $netconf_desc = {
 267     type => {
 268         type => 'string',
 269         optional => 1,
 270         description => "Network interface type.",
 271         enum => [qw(veth)],
 272     },
 273     name => {
 274         type => 'string',
 275         format_description => 'String',
 276         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 277         pattern => '[-_.\w\d]+',
 278     },
 279     bridge => {
 280         type => 'string',
 281         format_description => 'vmbr<Number>',
 282         description => 'Bridge to attach the network device to.',
 283         pattern => '[-_.\w\d]+',
 284         optional => 1,
 285     },
 286     hwaddr => {
 287         type => 'string',
 288         format_description => 'MAC',
 289         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 290         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 291         optional => 1,
 292     },
 293     mtu => {
 294         type => 'integer',
 295         format_description => 'Number',
 296         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 297         minimum => 64, # minimum ethernet frame is 64 bytes
 298         optional => 1,
 299     },
 300     ip => {
 301         type => 'string',
 302         format => 'pve-ipv4-config',
 303         format_description => 'IPv4Format/CIDR',
 304         description => 'IPv4 address in CIDR format.',
 305         optional => 1,
 306     },
 307     gw => {
 308         type => 'string',
 309         format => 'ipv4',
 310         format_description => 'GatewayIPv4',
 311         description => 'Default gateway for IPv4 traffic.',
 312         optional => 1,
 313     },
 314     ip6 => {
 315         type => 'string',
 316         format => 'pve-ipv6-config',
 317         format_description => 'IPv6Format/CIDR',
 318         description => 'IPv6 address in CIDR format.',
 319         optional => 1,
 320     },
 321     gw6 => {
 322         type => 'string',
 323         format => 'ipv6',
 324         format_description => 'GatewayIPv6',
 325         description => 'Default gateway for IPv6 traffic.',
 326         optional => 1,
 327     },
 328     firewall => {
 329         type => 'boolean',
 330         format_description => '[1|0]',
 331         description => "Controls whether this interface's firewall rules should be used.",
 332         optional => 1,
 333     },
 334     tag => {
 335         type => 'integer',
 336         format_description => 'VlanNo',
 337         minimum => '2',
 338         maximum => '4094',
 339         description => "VLAN tag foro this interface.",
 340         optional => 1,
 341     },
 342 };
 343 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 344
 345 my $MAX_LXC_NETWORKS = 10;
 346 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 347     $confdesc->{"net$i"} = {
 348         optional => 1,
 349         type => 'string', format => $netconf_desc,
 350         description => "Specifies network interfaces for the container.",
 351     };
 352 }
 353
 354 my $mp_desc = {
 355     %$rootfs_desc,
 356     mp => {
 357         type => 'string',
 358         format_description => 'Path',
 359         description => 'Path to the mountpoint as seen from inside the container.',
 360         optional => 1,
 361     },
 362 };
 363 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 364
 365 my $unuseddesc = {
 366     optional => 1,
 367     type => 'string', format => 'pve-volume-id',
 368     description => "Reference to unused volumes.",
 369 };
 370
 371 my $MAX_MOUNT_POINTS = 10;
 372 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 373     $confdesc->{"mp$i"} = {
 374         optional => 1,
 375         type => 'string', format => $mp_desc,
 376         description => "Use volume as container mount point (experimental feature).",
 377         optional => 1,
 378     };
 379 }
 380
 381 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 382 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 383     $confdesc->{"unused$i"} = $unuseddesc;
 384 }
 385
 386 sub write_pct_config {
 387     my ($filename, $conf) = @_;
 388
 389     delete $conf->{snapstate}; # just to be sure
 390
 391     my $generate_raw_config = sub {
 392         my ($conf) = @_;
 393
 394         my $raw = '';
 395
 396         # add description as comment to top of file
 397         my $descr = $conf->{description} || '';
 398         foreach my $cl (split(/\n/, $descr)) {
 399             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 400         }
 401
 402         foreach my $key (sort keys %$conf) {
 403             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 404                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 405             my $value = $conf->{$key};
 406             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 407             $raw .= "$key: $value\n";
 408         }
 409
 410         if (my $lxcconf = $conf->{lxc}) {
 411             foreach my $entry (@$lxcconf) {
 412                 my ($k, $v) = @$entry;
 413                 $raw .= "$k: $v\n";
 414             }
 415         }
 416
 417         return $raw;
 418     };
 419
 420     my $raw = &$generate_raw_config($conf);
 421
 422     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 423         $raw .= "\n[$snapname]\n";
 424         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 425     }
 426
 427     return $raw;
 428 }
 429
 430 sub check_type {
 431     my ($key, $value) = @_;
 432
 433     die "unknown setting '$key'\n" if !$confdesc->{$key};
 434
 435     my $type = $confdesc->{$key}->{type};
 436
 437     if (!defined($value)) {
 438         die "got undefined value\n";
 439     }
 440
 441     if ($value =~ m/[\n\r]/) {
 442         die "property contains a line feed\n";
 443     }
 444
 445     if ($type eq 'boolean') {
 446         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 447         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 448         die "type check ('boolean') failed - got '$value'\n";
 449     } elsif ($type eq 'integer') {
 450         return int($1) if $value =~ m/^(\d+)$/;
 451         die "type check ('integer') failed - got '$value'\n";
 452     } elsif ($type eq 'number') {
 453         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 454         die "type check ('number') failed - got '$value'\n";
 455     } elsif ($type eq 'string') {
 456         if (my $fmt = $confdesc->{$key}->{format}) {
 457             PVE::JSONSchema::check_format($fmt, $value);
 458             return $value;
 459         }
 460         return $value;
 461     } else {
 462         die "internal error"
 463     }
 464 }
 465
 466 sub parse_pct_config {
 467     my ($filename, $raw) = @_;
 468
 469     return undef if !defined($raw);
 470
 471     my $res = {
 472         digest => Digest::SHA::sha1_hex($raw),
 473         snapshots => {},
 474     };
 475
 476     $filename =~ m|/lxc/(\d+).conf$|
 477         || die "got strange filename '$filename'";
 478
 479     my $vmid = $1;
 480
 481     my $conf = $res;
 482     my $descr = '';
 483     my $section = '';
 484
 485     my @lines = split(/\n/, $raw);
 486     foreach my $line (@lines) {
 487         next if $line =~ m/^\s*$/;
 488
 489         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 490             $section = $1;
 491             $conf->{description} = $descr if $descr;
 492             $descr = '';
 493             $conf = $res->{snapshots}->{$section} = {};
 494             next;
 495         }
 496
 497         if ($line =~ m/^\#(.*)\s*$/) {
 498             $descr .= PVE::Tools::decode_text($1) . "\n";
 499             next;
 500         }
 501
 502         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 503             my $key = $1;
 504             my $value = $3;
 505             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 506                 push @{$conf->{lxc}}, [$key, $value];
 507             } else {
 508                 warn "vm $vmid - unable to parse config: $line\n";
 509             }
 510         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 511             $descr .= PVE::Tools::decode_text($2);
 512         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 513             $conf->{snapstate} = $1;
 514         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 515             my $key = $1;
 516             my $value = $2;
 517             eval { $value = check_type($key, $value); };
 518             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 519             $conf->{$key} = $value;
 520         } else {
 521             warn "vm $vmid - unable to parse config: $line\n";
 522         }
 523     }
 524
 525     $conf->{description} = $descr if $descr;
 526
 527     delete $res->{snapstate}; # just to be sure
 528
 529     return $res;
 530 }
 531
 532 sub config_list {
 533     my $vmlist = PVE::Cluster::get_vmlist();
 534     my $res = {};
 535     return $res if !$vmlist || !$vmlist->{ids};
 536     my $ids = $vmlist->{ids};
 537
 538     foreach my $vmid (keys %$ids) {
 539         next if !$vmid; # skip CT0
 540         my $d = $ids->{$vmid};
 541         next if !$d->{node} || $d->{node} ne $nodename;
 542         next if !$d->{type} || $d->{type} ne 'lxc';
 543         $res->{$vmid}->{type} = 'lxc';
 544     }
 545     return $res;
 546 }
 547
 548 sub cfs_config_path {
 549     my ($vmid, $node) = @_;
 550
 551     $node = $nodename if !$node;
 552     return "nodes/$node/lxc/$vmid.conf";
 553 }
 554
 555 sub config_file {
 556     my ($vmid, $node) = @_;
 557
 558     my $cfspath = cfs_config_path($vmid, $node);
 559     return "/etc/pve/$cfspath";
 560 }
 561
 562 sub load_config {
 563     my ($vmid, $node) = @_;
 564
 565     $node = $nodename if !$node;
 566     my $cfspath = cfs_config_path($vmid, $node);
 567
 568     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 569     die "container $vmid does not exists\n" if !defined($conf);
 570
 571     return $conf;
 572 }
 573
 574 sub create_config {
 575     my ($vmid, $conf) = @_;
 576
 577     my $dir = "/etc/pve/nodes/$nodename/lxc";
 578     mkdir $dir;
 579
 580     write_config($vmid, $conf);
 581 }
 582
 583 sub destroy_config {
 584     my ($vmid) = @_;
 585
 586     unlink config_file($vmid, $nodename);
 587 }
 588
 589 sub write_config {
 590     my ($vmid, $conf) = @_;
 591
 592     my $cfspath = cfs_config_path($vmid);
 593
 594     PVE::Cluster::cfs_write_file($cfspath, $conf);
 595 }
 596
 597 # flock: we use one file handle per process, so lock file
 598 # can be called multiple times and succeeds for the same process.
 599
 600 my $lock_handles =  {};
 601 my $lockdir = "/run/lock/lxc";
 602
 603 sub lock_filename {
 604     my ($vmid) = @_;
 605
 606     return "$lockdir/pve-config-${vmid}.lock";
 607 }
 608
 609 sub lock_aquire {
 610     my ($vmid, $timeout) = @_;
 611
 612     $timeout = 10 if !$timeout;
 613     my $mode = LOCK_EX;
 614
 615     my $filename = lock_filename($vmid);
 616
 617     mkdir $lockdir if !-d $lockdir;
 618
 619     my $lock_func = sub {
 620         if (!$lock_handles->{$$}->{$filename}) {
 621             my $fh = new IO::File(">>$filename") ||
 622                 die "can't open file - $!\n";
 623             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 624         }
 625
 626         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 627             print STDERR "trying to aquire lock...";
 628             my $success;
 629             while(1) {
 630                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 631                 # try again on EINTR (see bug #273)
 632                 if ($success || ($! != EINTR)) {
 633                     last;
 634                 }
 635             }
 636             if (!$success) {
 637                 print STDERR " failed\n";
 638                 die "can't aquire lock - $!\n";
 639             }
 640
 641             print STDERR " OK\n";
 642         }
 643
 644         $lock_handles->{$$}->{$filename}->{refcount}++;
 645     };
 646
 647     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 648     my $err = $@;
 649     if ($err) {
 650         die "can't lock file '$filename' - $err";
 651     }
 652 }
 653
 654 sub lock_release {
 655     my ($vmid) = @_;
 656
 657     my $filename = lock_filename($vmid);
 658
 659     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 660         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 661         if ($refcount <= 0) {
 662             $lock_handles->{$$}->{$filename} = undef;
 663             close ($fh);
 664         }
 665     }
 666 }
 667
 668 sub lock_container {
 669     my ($vmid, $timeout, $code, @param) = @_;
 670
 671     my $res;
 672
 673     lock_aquire($vmid, $timeout);
 674     eval { $res = &$code(@param) };
 675     my $err = $@;
 676     lock_release($vmid);
 677
 678     die $err if $err;
 679
 680     return $res;
 681 }
 682
 683 sub option_exists {
 684     my ($name) = @_;
 685
 686     return defined($confdesc->{$name});
 687 }
 688
 689 # add JSON properties for create and set function
 690 sub json_config_properties {
 691     my $prop = shift;
 692
 693     foreach my $opt (keys %$confdesc) {
 694         next if $opt eq 'parent' || $opt eq 'snaptime';
 695         next if $prop->{$opt};
 696         $prop->{$opt} = $confdesc->{$opt};
 697     }
 698
 699     return $prop;
 700 }
 701
 702 sub json_config_properties_no_rootfs {
 703     my $prop = shift;
 704
 705     foreach my $opt (keys %$confdesc) {
 706         next if $prop->{$opt};
 707         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 708         $prop->{$opt} = $confdesc->{$opt};
 709     }
 710
 711     return $prop;
 712 }
 713
 714 # container status helpers
 715
 716 sub list_active_containers {
 717
 718     my $filename = "/proc/net/unix";
 719
 720     # similar test is used by lcxcontainers.c: list_active_containers
 721     my $res = {};
 722
 723     my $fh = IO::File->new ($filename, "r");
 724     return $res if !$fh;
 725
 726     while (defined(my $line = <$fh>)) {
 727         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 728             my $path = $1;
 729             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 730                 $res->{$1} = 1;
 731             }
 732         }
 733     }
 734
 735     close($fh);
 736
 737     return $res;
 738 }
 739
 740 # warning: this is slow
 741 sub check_running {
 742     my ($vmid) = @_;
 743
 744     my $active_hash = list_active_containers();
 745
 746     return 1 if defined($active_hash->{$vmid});
 747
 748     return undef;
 749 }
 750
 751 sub get_container_disk_usage {
 752     my ($vmid, $pid) = @_;
 753
 754     return PVE::Tools::df("/proc/$pid/root/", 1);
 755 }
 756
 757 my $last_proc_vmid_stat;
 758
 759 my $parse_cpuacct_stat = sub {
 760     my ($vmid) = @_;
 761
 762     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 763
 764     my $stat = {};
 765
 766     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 767
 768         $stat->{utime} = $1;
 769         $stat->{stime} = $2;
 770
 771     }
 772
 773     return $stat;
 774 };
 775
 776 sub vmstatus {
 777     my ($opt_vmid) = @_;
 778
 779     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 780
 781     my $active_hash = list_active_containers();
 782
 783     my $cpucount = $cpuinfo->{cpus} || 1;
 784
 785     my $cdtime = gettimeofday;
 786
 787     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 788
 789     foreach my $vmid (keys %$list) {
 790         my $d = $list->{$vmid};
 791
 792         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 793         warn $@ if $@; # ignore errors (consider them stopped)
 794
 795         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 796
 797         my $cfspath = cfs_config_path($vmid);
 798         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 799
 800         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 801         $d->{name} =~ s/[\s]//g;
 802
 803         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 804
 805         if ($d->{pid}) {
 806             my $res = get_container_disk_usage($vmid, $d->{pid});
 807             $d->{disk} = $res->{used};
 808             $d->{maxdisk} = $res->{total};
 809         } else {
 810             $d->{disk} = 0;
 811             # use 4GB by default ??
 812             if (my $rootfs = $conf->{rootfs}) {
 813                 my $rootinfo = parse_ct_mountpoint($rootfs);
 814                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 815             } else {
 816                 $d->{maxdisk} = 4*1024*1024*1024;
 817             }
 818         }
 819
 820         $d->{mem} = 0;
 821         $d->{swap} = 0;
 822         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 823         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 824
 825         $d->{uptime} = 0;
 826         $d->{cpu} = 0;
 827
 828         $d->{netout} = 0;
 829         $d->{netin} = 0;
 830
 831         $d->{diskread} = 0;
 832         $d->{diskwrite} = 0;
 833
 834         $d->{template} = is_template($conf);
 835     }
 836
 837     foreach my $vmid (keys %$list) {
 838         my $d = $list->{$vmid};
 839         my $pid = $d->{pid};
 840
 841         next if !$pid; # skip stopped CTs
 842
 843         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 844         $d->{uptime} = time - $ctime; # the method lxcfs uses
 845
 846         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 847         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 848
 849         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 850         my @bytes = split(/\n/, $blkio_bytes);
 851         foreach my $byte (@bytes) {
 852             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 853                 $d->{diskread} = $2 if $key eq 'Read';
 854                 $d->{diskwrite} = $2 if $key eq 'Write';
 855             }
 856         }
 857
 858         my $pstat = &$parse_cpuacct_stat($vmid);
 859
 860         my $used = $pstat->{utime} + $pstat->{stime};
 861
 862         my $old = $last_proc_vmid_stat->{$vmid};
 863         if (!$old) {
 864             $last_proc_vmid_stat->{$vmid} = {
 865                 time => $cdtime,
 866                 used => $used,
 867                 cpu => 0,
 868             };
 869             next;
 870         }
 871
 872         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 873
 874         if ($dtime > 1000) {
 875             my $dutime = $used -  $old->{used};
 876
 877             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 878             $last_proc_vmid_stat->{$vmid} = {
 879                 time => $cdtime,
 880                 used => $used,
 881                 cpu => $d->{cpu},
 882             };
 883         } else {
 884             $d->{cpu} = $old->{cpu};
 885         }
 886     }
 887
 888     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 889
 890     foreach my $dev (keys %$netdev) {
 891         next if $dev !~ m/^veth([1-9]\d*)i/;
 892         my $vmid = $1;
 893         my $d = $list->{$vmid};
 894
 895         next if !$d;
 896
 897         $d->{netout} += $netdev->{$dev}->{receive};
 898         $d->{netin} += $netdev->{$dev}->{transmit};
 899
 900     }
 901
 902     return $list;
 903 }
 904
 905 sub parse_ct_mountpoint {
 906     my ($data, $noerr) = @_;
 907
 908     $data //= '';
 909
 910     my $res;
 911     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 912     if ($@) {
 913         return undef if $noerr;
 914         die $@;
 915     }
 916
 917     if (defined(my $size = $res->{size})) {
 918         $size = PVE::JSONSchema::parse_size($size);
 919         if (!defined($size)) {
 920             return undef if $noerr;
 921             die "invalid size: $size\n";
 922         }
 923         $res->{size} = $size;
 924     }
 925
 926     return $res;
 927 }
 928
 929 sub print_ct_mountpoint {
 930     my ($info, $nomp) = @_;
 931     my $skip = $nomp ? ['mp'] : [];
 932     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 933 }
 934
 935 sub print_lxc_network {
 936     my $net = shift;
 937     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 938 }
 939
 940 sub parse_lxc_network {
 941     my ($data) = @_;
 942
 943     my $res = {};
 944
 945     return $res if !$data;
 946
 947     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 948
 949     $res->{type} = 'veth';
 950     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 951
 952     return $res;
 953 }
 954
 955 sub read_cgroup_value {
 956     my ($group, $vmid, $name, $full) = @_;
 957
 958     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 959
 960     return PVE::Tools::file_get_contents($path) if $full;
 961
 962     return PVE::Tools::file_read_firstline($path);
 963 }
 964
 965 sub write_cgroup_value {
 966    my ($group, $vmid, $name, $value) = @_;
 967
 968    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 969    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 970
 971 }
 972
 973 sub find_lxc_console_pids {
 974
 975     my $res = {};
 976
 977     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 978         my ($pid) = @_;
 979
 980         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 981         return if !$cmdline;
 982
 983         my @args = split(/\0/, $cmdline);
 984
 985         # serach for lxc-console -n <vmid>
 986         return if scalar(@args) != 3;
 987         return if $args[1] ne '-n';
 988         return if $args[2] !~ m/^\d+$/;
 989         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 990
 991         my $vmid = $args[2];
 992
 993         push @{$res->{$vmid}}, $pid;
 994     });
 995
 996     return $res;
 997 }
 998
 999 sub find_lxc_pid {
1000     my ($vmid) = @_;
1001
1002     my $pid = undef;
1003     my $parser = sub {
1004         my $line = shift;
1005         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1006     };
1007     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1008
1009     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1010
1011     return $pid;
1012 }
1013
1014 # Note: we cannot use Net:IP, because that only allows strict
1015 # CIDR networks
1016 sub parse_ipv4_cidr {
1017     my ($cidr, $noerr) = @_;
1018
1019     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1020         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1021     }
1022
1023     return undef if $noerr;
1024
1025     die "unable to parse ipv4 address/mask\n";
1026 }
1027
1028 sub check_lock {
1029     my ($conf) = @_;
1030
1031     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1032 }
1033
1034 sub check_protection {
1035     my ($vm_conf, $err_msg) = @_;
1036
1037     if ($vm_conf->{protection}) {
1038         die "$err_msg - protection mode enabled\n";
1039     }
1040 }
1041
1042 sub update_lxc_config {
1043     my ($storage_cfg, $vmid, $conf) = @_;
1044
1045     my $dir = "/var/lib/lxc/$vmid";
1046
1047     if ($conf->{template}) {
1048
1049         unlink "$dir/config";
1050
1051         return;
1052     }
1053
1054     my $raw = '';
1055
1056     die "missing 'arch' - internal error" if !$conf->{arch};
1057     $raw .= "lxc.arch = $conf->{arch}\n";
1058
1059     my $unprivileged = $conf->{unprivileged};
1060     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1061
1062     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1063     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1064         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1065         if ($unprivileged || $custom_idmap) {
1066             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1067         }
1068     } else {
1069         die "implement me";
1070     }
1071
1072     $raw .= "lxc.start.unshare = 1\n";
1073
1074     # Should we read them from /etc/subuid?
1075     if ($unprivileged && !$custom_idmap) {
1076         $raw .= "lxc.id_map = u 0 100000 65536\n";
1077         $raw .= "lxc.id_map = g 0 100000 65536\n";
1078     }
1079
1080     if (!has_dev_console($conf)) {
1081         $raw .= "lxc.console = none\n";
1082         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1083     }
1084
1085     my $ttycount = get_tty_count($conf);
1086     $raw .= "lxc.tty = $ttycount\n";
1087
1088     # some init scripts expects a linux terminal (turnkey).
1089     $raw .= "lxc.environment = TERM=linux\n";
1090
1091     my $utsname = $conf->{hostname} || "CT$vmid";
1092     $raw .= "lxc.utsname = $utsname\n";
1093
1094     my $memory = $conf->{memory} || 512;
1095     my $swap = $conf->{swap} // 0;
1096
1097     my $lxcmem = int($memory*1024*1024);
1098     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1099     $raw .= "lxc.cgroup.memory.kmem.limit_in_bytes = $lxcmem\n";
1100
1101     my $lxcswap = int(($memory + $swap)*1024*1024);
1102     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1103
1104     if (my $cpulimit = $conf->{cpulimit}) {
1105         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1106         my $value = int(100000*$cpulimit);
1107         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1108     }
1109
1110     my $shares = $conf->{cpuunits} || 1024;
1111     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1112
1113     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1114     $mountpoint->{mp} = '/';
1115
1116     $raw .= "lxc.rootfs = $dir/rootfs\n";
1117
1118     my $netcount = 0;
1119     foreach my $k (keys %$conf) {
1120         next if $k !~ m/^net(\d+)$/;
1121         my $ind = $1;
1122         my $d = parse_lxc_network($conf->{$k});
1123         $netcount++;
1124         $raw .= "lxc.network.type = veth\n";
1125         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1126         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1127         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1128         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1129     }
1130
1131     if (my $lxcconf = $conf->{lxc}) {
1132         foreach my $entry (@$lxcconf) {
1133             my ($k, $v) = @$entry;
1134             $netcount++ if $k eq 'lxc.network.type';
1135             $raw .= "$k = $v\n";
1136         }
1137     }
1138
1139     $raw .= "lxc.network.type = empty\n" if !$netcount;
1140
1141     File::Path::mkpath("$dir/rootfs");
1142
1143     PVE::Tools::file_set_contents("$dir/config", $raw);
1144 }
1145
1146 # verify and cleanup nameserver list (replace \0 with ' ')
1147 sub verify_nameserver_list {
1148     my ($nameserver_list) = @_;
1149
1150     my @list = ();
1151     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1152         PVE::JSONSchema::pve_verify_ip($server);
1153         push @list, $server;
1154     }
1155
1156     return join(' ', @list);
1157 }
1158
1159 sub verify_searchdomain_list {
1160     my ($searchdomain_list) = @_;
1161
1162     my @list = ();
1163     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1164         # todo: should we add checks for valid dns domains?
1165         push @list, $server;
1166     }
1167
1168     return join(' ', @list);
1169 }
1170
1171 sub add_unused_volume {
1172     my ($config, $volid) = @_;
1173
1174     # skip bind mounts and block devices
1175     return if $volid =~ m|^/|;
1176
1177     my $key;
1178     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1179         my $test = "unused$ind";
1180         if (my $vid = $config->{$test}) {
1181             return if $vid eq $volid; # do not add duplicates
1182         } else {
1183             $key = $test;
1184         }
1185     }
1186
1187     die "To many unused volume - please delete them first.\n" if !$key;
1188
1189     $config->{$key} = $volid;
1190
1191     return $key;
1192 }
1193
1194 sub update_pct_config {
1195     my ($vmid, $conf, $running, $param, $delete) = @_;
1196
1197     my @nohotplug;
1198
1199     my $new_disks = 0;
1200     my @deleted_volumes;
1201
1202     my $rootdir;
1203     if ($running) {
1204         my $pid = find_lxc_pid($vmid);
1205         $rootdir = "/proc/$pid/root";
1206     }
1207
1208     my $hotplug_error = sub {
1209         if ($running) {
1210             push @nohotplug, @_;
1211             return 1;
1212         } else {
1213             return 0;
1214         }
1215     };
1216
1217     if (defined($delete)) {
1218         foreach my $opt (@$delete) {
1219             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1220                 die "unable to delete required option '$opt'\n";
1221             } elsif ($opt eq 'swap') {
1222                 delete $conf->{$opt};
1223                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1224             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1225                 delete $conf->{$opt};
1226             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1227                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1228                 next if $hotplug_error->($opt);
1229                 delete $conf->{$opt};
1230             } elsif ($opt =~ m/^net(\d)$/) {
1231                 delete $conf->{$opt};
1232                 next if !$running;
1233                 my $netid = $1;
1234                 PVE::Network::veth_delete("veth${vmid}i$netid");
1235             } elsif ($opt eq 'protection') {
1236                 delete $conf->{$opt};
1237             } elsif ($opt =~ m/^unused(\d+)$/) {
1238                 next if $hotplug_error->($opt);
1239                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1240                 push @deleted_volumes, $conf->{$opt};
1241                 delete $conf->{$opt};
1242             } elsif ($opt =~ m/^mp(\d+)$/) {
1243                 next if $hotplug_error->($opt);
1244                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1245                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1246                 add_unused_volume($conf, $mountpoint->{volume});
1247                 delete $conf->{$opt};
1248             } elsif ($opt eq 'unprivileged') {
1249                 die "unable to delete read-only option: '$opt'\n";
1250             } else {
1251                 die "implement me"
1252             }
1253             write_config($vmid, $conf) if $running;
1254         }
1255     }
1256
1257     # There's no separate swap size to configure, there's memory and "total"
1258     # memory (iow. memory+swap). This means we have to change them together.
1259     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1260     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1261     if (defined($wanted_memory) || defined($wanted_swap)) {
1262
1263         $wanted_memory //= ($conf->{memory} || 512);
1264         $wanted_swap //=  ($conf->{swap} || 0);
1265
1266         my $total = $wanted_memory + $wanted_swap;
1267         if ($running) {
1268             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1269             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1270         }
1271         $conf->{memory} = $wanted_memory;
1272         $conf->{swap} = $wanted_swap;
1273
1274         write_config($vmid, $conf) if $running;
1275     }
1276
1277     foreach my $opt (keys %$param) {
1278         my $value = $param->{$opt};
1279         if ($opt eq 'hostname') {
1280             $conf->{$opt} = $value;
1281         } elsif ($opt eq 'onboot') {
1282             $conf->{$opt} = $value ? 1 : 0;
1283         } elsif ($opt eq 'startup') {
1284             $conf->{$opt} = $value;
1285         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1286             next if $hotplug_error->($opt);
1287             $conf->{$opt} = $value;
1288         } elsif ($opt eq 'nameserver') {
1289             next if $hotplug_error->($opt);
1290             my $list = verify_nameserver_list($value);
1291             $conf->{$opt} = $list;
1292         } elsif ($opt eq 'searchdomain') {
1293             next if $hotplug_error->($opt);
1294             my $list = verify_searchdomain_list($value);
1295             $conf->{$opt} = $list;
1296         } elsif ($opt eq 'cpulimit') {
1297             next if $hotplug_error->($opt); # FIXME: hotplug
1298             $conf->{$opt} = $value;
1299         } elsif ($opt eq 'cpuunits') {
1300             $conf->{$opt} = $value;
1301             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1302         } elsif ($opt eq 'description') {
1303             $conf->{$opt} = PVE::Tools::encode_text($value);
1304         } elsif ($opt =~ m/^net(\d+)$/) {
1305             my $netid = $1;
1306             my $net = parse_lxc_network($value);
1307             if (!$running) {
1308                 $conf->{$opt} = print_lxc_network($net);
1309             } else {
1310                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1311             }
1312         } elsif ($opt eq 'protection') {
1313             $conf->{$opt} = $value ? 1 : 0;
1314         } elsif ($opt =~ m/^mp(\d+)$/) {
1315             next if $hotplug_error->($opt);
1316             check_protection($conf, "can't update CT $vmid drive '$opt'");
1317             $conf->{$opt} = $value;
1318             $new_disks = 1;
1319         } elsif ($opt eq 'rootfs') {
1320             check_protection($conf, "can't update CT $vmid drive '$opt'");
1321             die "implement me: $opt";
1322         } elsif ($opt eq 'unprivileged') {
1323             die "unable to modify read-only option: '$opt'\n";
1324         } else {
1325             die "implement me: $opt";
1326         }
1327         write_config($vmid, $conf) if $running;
1328     }
1329
1330     if (@deleted_volumes) {
1331         my $storage_cfg = PVE::Storage::config();
1332         foreach my $volume (@deleted_volumes) {
1333             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1334         }
1335     }
1336
1337     if ($new_disks) {
1338         my $storage_cfg = PVE::Storage::config();
1339         create_disks($storage_cfg, $vmid, $conf, $conf);
1340     }
1341
1342     # This should be the last thing we do here
1343     if ($running && scalar(@nohotplug)) {
1344         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1345     }
1346 }
1347
1348 sub has_dev_console {
1349     my ($conf) = @_;
1350
1351     return !(defined($conf->{console}) && !$conf->{console});
1352 }
1353
1354 sub get_tty_count {
1355     my ($conf) = @_;
1356
1357     return $conf->{tty} // $confdesc->{tty}->{default};
1358 }
1359
1360 sub get_cmode {
1361     my ($conf) = @_;
1362
1363     return $conf->{cmode} // $confdesc->{cmode}->{default};
1364 }
1365
1366 sub get_console_command {
1367     my ($vmid, $conf) = @_;
1368
1369     my $cmode = get_cmode($conf);
1370
1371     if ($cmode eq 'console') {
1372         return ['lxc-console', '-n',  $vmid, '-t', 0];
1373     } elsif ($cmode eq 'tty') {
1374         return ['lxc-console', '-n',  $vmid];
1375     } elsif ($cmode eq 'shell') {
1376         return ['lxc-attach', '--clear-env', '-n', $vmid];
1377     } else {
1378         die "internal error";
1379     }
1380 }
1381
1382 sub get_primary_ips {
1383     my ($conf) = @_;
1384
1385     # return data from net0
1386
1387     return undef if !defined($conf->{net0});
1388     my $net = parse_lxc_network($conf->{net0});
1389
1390     my $ipv4 = $net->{ip};
1391     if ($ipv4) {
1392         if ($ipv4 =~ /^(dhcp|manual)$/) {
1393             $ipv4 = undef
1394         } else {
1395             $ipv4 =~ s!/\d+$!!;
1396         }
1397     }
1398     my $ipv6 = $net->{ip6};
1399     if ($ipv6) {
1400         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1401             $ipv6 = undef;
1402         } else {
1403             $ipv6 =~ s!/\d+$!!;
1404         }
1405     }
1406
1407     return ($ipv4, $ipv6);
1408 }
1409
1410 sub delete_mountpoint_volume {
1411     my ($storage_cfg, $vmid, $volume) = @_;
1412
1413     # skip bind mounts and block devices
1414     if ($volume =~ m|^/|) {
1415             return;
1416     }
1417
1418     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1419     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1420 }
1421
1422 sub destroy_lxc_container {
1423     my ($storage_cfg, $vmid, $conf) = @_;
1424
1425     foreach_mountpoint($conf, sub {
1426         my ($ms, $mountpoint) = @_;
1427         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1428     });
1429
1430     rmdir "/var/lib/lxc/$vmid/rootfs";
1431     unlink "/var/lib/lxc/$vmid/config";
1432     rmdir "/var/lib/lxc/$vmid";
1433     destroy_config($vmid);
1434
1435     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1436     #PVE::Tools::run_command($cmd);
1437 }
1438
1439 sub vm_stop_cleanup {
1440     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1441
1442     eval {
1443         if (!$keepActive) {
1444
1445             my $vollist = get_vm_volumes($conf);
1446             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1447         }
1448     };
1449     warn $@ if $@; # avoid errors - just warn
1450 }
1451
1452 my $safe_num_ne = sub {
1453     my ($a, $b) = @_;
1454
1455     return 0 if !defined($a) && !defined($b);
1456     return 1 if !defined($a);
1457     return 1 if !defined($b);
1458
1459     return $a != $b;
1460 };
1461
1462 my $safe_string_ne = sub {
1463     my ($a, $b) = @_;
1464
1465     return 0 if !defined($a) && !defined($b);
1466     return 1 if !defined($a);
1467     return 1 if !defined($b);
1468
1469     return $a ne $b;
1470 };
1471
1472 sub update_net {
1473     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1474
1475     if ($newnet->{type} ne 'veth') {
1476         # for when there are physical interfaces
1477         die "cannot update interface of type $newnet->{type}";
1478     }
1479
1480     my $veth = "veth${vmid}i${netid}";
1481     my $eth = $newnet->{name};
1482
1483     if (my $oldnetcfg = $conf->{$opt}) {
1484         my $oldnet = parse_lxc_network($oldnetcfg);
1485
1486         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1487             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1488
1489             PVE::Network::veth_delete($veth);
1490             delete $conf->{$opt};
1491             write_config($vmid, $conf);
1492
1493             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1494
1495         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1496                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1497                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1498
1499                 if ($oldnet->{bridge}) {
1500                     PVE::Network::tap_unplug($veth);
1501                     foreach (qw(bridge tag firewall)) {
1502                         delete $oldnet->{$_};
1503                     }
1504                     $conf->{$opt} = print_lxc_network($oldnet);
1505                     write_config($vmid, $conf);
1506                 }
1507
1508                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1509                 foreach (qw(bridge tag firewall)) {
1510                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1511                 }
1512                 $conf->{$opt} = print_lxc_network($oldnet);
1513                 write_config($vmid, $conf);
1514         }
1515     } else {
1516         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1517     }
1518
1519     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1520 }
1521
1522 sub hotplug_net {
1523     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1524
1525     my $veth = "veth${vmid}i${netid}";
1526     my $vethpeer = $veth . "p";
1527     my $eth = $newnet->{name};
1528
1529     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1530     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1531
1532     # attach peer in container
1533     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1534     PVE::Tools::run_command($cmd);
1535
1536     # link up peer in container
1537     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1538     PVE::Tools::run_command($cmd);
1539
1540     my $done = { type => 'veth' };
1541     foreach (qw(bridge tag firewall hwaddr name)) {
1542         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1543     }
1544     $conf->{$opt} = print_lxc_network($done);
1545
1546     write_config($vmid, $conf);
1547 }
1548
1549 sub update_ipconfig {
1550     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1551
1552     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1553
1554     my $optdata = parse_lxc_network($conf->{$opt});
1555     my $deleted = [];
1556     my $added = [];
1557     my $nscmd = sub {
1558         my $cmdargs = shift;
1559         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1560     };
1561     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1562
1563     my $change_ip_config = sub {
1564         my ($ipversion) = @_;
1565
1566         my $family_opt = "-$ipversion";
1567         my $suffix = $ipversion == 4 ? '' : $ipversion;
1568         my $gw= "gw$suffix";
1569         my $ip= "ip$suffix";
1570
1571         my $newip = $newnet->{$ip};
1572         my $newgw = $newnet->{$gw};
1573         my $oldip = $optdata->{$ip};
1574
1575         my $change_ip = &$safe_string_ne($oldip, $newip);
1576         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1577
1578         return if !$change_ip && !$change_gw;
1579
1580         # step 1: add new IP, if this fails we cancel
1581         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1582         if ($change_ip && $is_real_ip) {
1583             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1584             if (my $err = $@) {
1585                 warn $err;
1586                 return;
1587             }
1588         }
1589
1590         # step 2: replace gateway
1591         #   If this fails we delete the added IP and cancel.
1592         #   If it succeeds we save the config and delete the old IP, ignoring
1593         #   errors. The config is then saved.
1594         # Note: 'ip route replace' can add
1595         if ($change_gw) {
1596             if ($newgw) {
1597                 eval {
1598                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1599                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1600                     }
1601                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1602                 };
1603                 if (my $err = $@) {
1604                     warn $err;
1605                     # the route was not replaced, the old IP is still available
1606                     # rollback (delete new IP) and cancel
1607                     if ($change_ip) {
1608                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1609                         warn $@ if $@; # no need to die here
1610                     }
1611                     return;
1612                 }
1613             } else {
1614                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1615                 # if the route was not deleted, the guest might have deleted it manually
1616                 # warn and continue
1617                 warn $@ if $@;
1618             }
1619         }
1620
1621         # from this point on we save the configuration
1622         # step 3: delete old IP ignoring errors
1623         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1624             # We need to enable promote_secondaries, otherwise our newly added
1625             # address will be removed along with the old one.
1626             my $promote = 0;
1627             eval {
1628                 if ($ipversion == 4) {
1629                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1630                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1631                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1632                 }
1633                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1634             };
1635             warn $@ if $@; # no need to die here
1636
1637             if ($ipversion == 4) {
1638                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1639             }
1640         }
1641
1642         foreach my $property ($ip, $gw) {
1643             if ($newnet->{$property}) {
1644                 $optdata->{$property} = $newnet->{$property};
1645             } else {
1646                 delete $optdata->{$property};
1647             }
1648         }
1649         $conf->{$opt} = print_lxc_network($optdata);
1650         write_config($vmid, $conf);
1651         $lxc_setup->setup_network($conf);
1652     };
1653
1654     &$change_ip_config(4);
1655     &$change_ip_config(6);
1656
1657 }
1658
1659 # Internal snapshots
1660
1661 # NOTE: Snapshot create/delete involves several non-atomic
1662 # action, and can take a long time.
1663 # So we try to avoid locking the file and use 'lock' variable
1664 # inside the config file instead.
1665
1666 my $snapshot_copy_config = sub {
1667     my ($source, $dest) = @_;
1668
1669     foreach my $k (keys %$source) {
1670         next if $k eq 'snapshots';
1671         next if $k eq 'snapstate';
1672         next if $k eq 'snaptime';
1673         next if $k eq 'vmstate';
1674         next if $k eq 'lock';
1675         next if $k eq 'digest';
1676         next if $k eq 'description';
1677
1678         $dest->{$k} = $source->{$k};
1679     }
1680 };
1681
1682 my $snapshot_prepare = sub {
1683     my ($vmid, $snapname, $comment) = @_;
1684
1685     my $snap;
1686
1687     my $updatefn =  sub {
1688
1689         my $conf = load_config($vmid);
1690
1691         die "you can't take a snapshot if it's a template\n"
1692             if is_template($conf);
1693
1694         check_lock($conf);
1695
1696         $conf->{lock} = 'snapshot';
1697
1698         die "snapshot name '$snapname' already used\n"
1699             if defined($conf->{snapshots}->{$snapname});
1700
1701         my $storecfg = PVE::Storage::config();
1702         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1703
1704         $snap = $conf->{snapshots}->{$snapname} = {};
1705
1706         &$snapshot_copy_config($conf, $snap);
1707
1708         $snap->{'snapstate'} = "prepare";
1709         $snap->{'snaptime'} = time();
1710         $snap->{'description'} = $comment if $comment;
1711         $conf->{snapshots}->{$snapname} = $snap;
1712
1713         write_config($vmid, $conf);
1714     };
1715
1716     lock_container($vmid, 10, $updatefn);
1717
1718     return $snap;
1719 };
1720
1721 my $snapshot_commit = sub {
1722     my ($vmid, $snapname) = @_;
1723
1724     my $updatefn = sub {
1725
1726         my $conf = load_config($vmid);
1727
1728         die "missing snapshot lock\n"
1729             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1730
1731         die "snapshot '$snapname' does not exist\n"
1732             if !defined($conf->{snapshots}->{$snapname});
1733
1734         die "wrong snapshot state\n"
1735             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1736                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1737
1738         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1739         delete $conf->{lock};
1740         $conf->{parent} = $snapname;
1741
1742         write_config($vmid, $conf);
1743     };
1744
1745     lock_container($vmid, 10 ,$updatefn);
1746 };
1747
1748 sub has_feature {
1749     my ($feature, $conf, $storecfg, $snapname) = @_;
1750
1751     my $err;
1752
1753     foreach_mountpoint($conf, sub {
1754         my ($ms, $mountpoint) = @_;
1755
1756         return if $err; # skip further test
1757
1758         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1759
1760         # TODO: implement support for mountpoints
1761         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1762             if $ms ne 'rootfs';
1763     });
1764
1765     return $err ? 0 : 1;
1766 }
1767
1768 sub snapshot_create {
1769     my ($vmid, $snapname, $comment) = @_;
1770
1771     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1772
1773     my $conf = load_config($vmid);
1774
1775     my $running = check_running($vmid);
1776     eval {
1777         if ($running) {
1778             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1779             PVE::Tools::run_command(['/bin/sync']);
1780         };
1781
1782         my $storecfg = PVE::Storage::config();
1783         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1784         my $volid = $rootinfo->{volume};
1785
1786         if ($running) {
1787             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1788         };
1789
1790         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1791         &$snapshot_commit($vmid, $snapname);
1792     };
1793     if(my $err = $@) {
1794         snapshot_delete($vmid, $snapname, 1);
1795         die "$err\n";
1796     }
1797 }
1798
1799 sub snapshot_delete {
1800     my ($vmid, $snapname, $force) = @_;
1801
1802     my $snap;
1803
1804     my $conf;
1805
1806     my $updatefn =  sub {
1807
1808         $conf = load_config($vmid);
1809
1810         die "you can't delete a snapshot if vm is a template\n"
1811             if is_template($conf);
1812
1813         $snap = $conf->{snapshots}->{$snapname};
1814
1815         check_lock($conf);
1816
1817         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1818
1819         $snap->{snapstate} = 'delete';
1820
1821         write_config($vmid, $conf);
1822     };
1823
1824     lock_container($vmid, 10, $updatefn);
1825
1826     my $storecfg = PVE::Storage::config();
1827
1828     my $del_snap =  sub {
1829
1830         check_lock($conf);
1831
1832         if ($conf->{parent} eq $snapname) {
1833             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1834                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1835             } else {
1836                 delete $conf->{parent};
1837             }
1838         }
1839
1840         delete $conf->{snapshots}->{$snapname};
1841
1842         write_config($vmid, $conf);
1843     };
1844
1845     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1846     my $rootinfo = parse_ct_mountpoint($rootfs);
1847     my $volid = $rootinfo->{volume};
1848
1849     eval {
1850         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1851     };
1852     my $err = $@;
1853
1854     if(!$err || ($err && $force)) {
1855         lock_container($vmid, 10, $del_snap);
1856         if ($err) {
1857             die "Can't delete snapshot: $vmid $snapname $err\n";
1858         }
1859     }
1860 }
1861
1862 sub snapshot_rollback {
1863     my ($vmid, $snapname) = @_;
1864
1865     my $storecfg = PVE::Storage::config();
1866
1867     my $conf = load_config($vmid);
1868
1869     die "you can't rollback if vm is a template\n" if is_template($conf);
1870
1871     my $snap = $conf->{snapshots}->{$snapname};
1872
1873     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1874
1875     my $rootfs = $snap->{rootfs};
1876     my $rootinfo = parse_ct_mountpoint($rootfs);
1877     my $volid = $rootinfo->{volume};
1878
1879     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1880
1881     my $updatefn = sub {
1882
1883         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1884             if $snap->{snapstate};
1885
1886         check_lock($conf);
1887
1888         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1889
1890         die "unable to rollback vm $vmid: vm is running\n"
1891             if check_running($vmid);
1892
1893         $conf->{lock} = 'rollback';
1894
1895         my $forcemachine;
1896
1897         # copy snapshot config to current config
1898
1899         my $tmp_conf = $conf;
1900         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1901         $conf->{snapshots} = $tmp_conf->{snapshots};
1902         delete $conf->{snaptime};
1903         delete $conf->{snapname};
1904         $conf->{parent} = $snapname;
1905
1906         write_config($vmid, $conf);
1907     };
1908
1909     my $unlockfn = sub {
1910         delete $conf->{lock};
1911         write_config($vmid, $conf);
1912     };
1913
1914     lock_container($vmid, 10, $updatefn);
1915
1916     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1917
1918     lock_container($vmid, 5, $unlockfn);
1919 }
1920
1921 sub template_create {
1922     my ($vmid, $conf) = @_;
1923
1924     my $storecfg = PVE::Storage::config();
1925
1926     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1927     my $volid = $rootinfo->{volume};
1928
1929     die "Template feature is not available for '$volid'\n"
1930         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1931
1932     PVE::Storage::activate_volumes($storecfg, [$volid]);
1933
1934     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1935     $rootinfo->{volume} = $template_volid;
1936     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1937
1938     write_config($vmid, $conf);
1939 }
1940
1941 sub is_template {
1942     my ($conf) = @_;
1943
1944     return 1 if defined $conf->{template} && $conf->{template} == 1;
1945 }
1946
1947 sub mountpoint_names {
1948     my ($reverse) = @_;
1949
1950     my @names = ('rootfs');
1951
1952     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1953         push @names, "mp$i";
1954     }
1955
1956     return $reverse ? reverse @names : @names;
1957 }
1958
1959 # The container might have *different* symlinks than the host. realpath/abs_path
1960 # use the actual filesystem to resolve links.
1961 sub sanitize_mountpoint {
1962     my ($mp) = @_;
1963     $mp = '/' . $mp; # we always start with a slash
1964     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1965     $mp =~ s@/\./@@g; # collapse /./
1966     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1967     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1968     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1969     return $mp;
1970 }
1971
1972 sub foreach_mountpoint_full {
1973     my ($conf, $reverse, $func) = @_;
1974
1975     foreach my $key (mountpoint_names($reverse)) {
1976         my $value = $conf->{$key};
1977         next if !defined($value);
1978         my $mountpoint = parse_ct_mountpoint($value, 1);
1979         next if !defined($mountpoint);
1980
1981         # just to be sure: rootfs is /
1982         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1983         $mountpoint->{mp} = sanitize_mountpoint($path);
1984
1985         $path = $mountpoint->{volume};
1986         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1987
1988         &$func($key, $mountpoint);
1989     }
1990 }
1991
1992 sub foreach_mountpoint {
1993     my ($conf, $func) = @_;
1994
1995     foreach_mountpoint_full($conf, 0, $func);
1996 }
1997
1998 sub foreach_mountpoint_reverse {
1999     my ($conf, $func) = @_;
2000
2001     foreach_mountpoint_full($conf, 1, $func);
2002 }
2003
2004 sub check_ct_modify_config_perm {
2005     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2006
2007     return 1 if $authuser ne 'root@pam';
2008
2009     foreach my $opt (@$key_list) {
2010
2011         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2012             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2013         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2014             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2015         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2016             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2017         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2018                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2019             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2020         } else {
2021             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2022         }
2023     }
2024
2025     return 1;
2026 }
2027
2028 sub umount_all {
2029     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2030
2031     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2032     my $volid_list = get_vm_volumes($conf);
2033
2034     foreach_mountpoint_reverse($conf, sub {
2035         my ($ms, $mountpoint) = @_;
2036
2037         my $volid = $mountpoint->{volume};
2038         my $mount = $mountpoint->{mp};
2039
2040         return if !$volid || !$mount;
2041
2042         my $mount_path = "$rootdir/$mount";
2043         $mount_path =~ s!/+!/!g;
2044
2045         return if !PVE::ProcFSTools::is_mounted($mount_path);
2046
2047         eval {
2048             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2049         };
2050         if (my $err = $@) {
2051             if ($noerr) {
2052                 warn $err;
2053             } else {
2054                 die $err;
2055             }
2056         }
2057     });
2058 }
2059
2060 sub mount_all {
2061     my ($vmid, $storage_cfg, $conf) = @_;
2062
2063     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2064     File::Path::make_path($rootdir);
2065
2066     my $volid_list = get_vm_volumes($conf);
2067     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2068
2069     eval {
2070         foreach_mountpoint($conf, sub {
2071             my ($ms, $mountpoint) = @_;
2072
2073             my $volid = $mountpoint->{volume};
2074             my $mount = $mountpoint->{mp};
2075
2076             return if !$volid || !$mount;
2077
2078             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2079             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2080                 PVE::Storage::parse_volname($storage_cfg, $volid);
2081
2082             die "unable to mount base volume - internal error" if $isBase;
2083
2084             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2085         });
2086     };
2087     if (my $err = $@) {
2088         warn "mounting container failed - $err";
2089         umount_all($vmid, $storage_cfg, $conf, 1);
2090     }
2091
2092     return $rootdir;
2093 }
2094
2095
2096 sub mountpoint_mount_path {
2097     my ($mountpoint, $storage_cfg, $snapname) = @_;
2098
2099     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2100 }
2101
2102 my $check_mount_path = sub {
2103     my ($path) = @_;
2104     $path = File::Spec->canonpath($path);
2105     my $real = Cwd::realpath($path);
2106     if ($real ne $path) {
2107         die "mount path modified by symlink: $path != $real";
2108     }
2109 };
2110
2111 # use $rootdir = undef to just return the corresponding mount path
2112 sub mountpoint_mount {
2113     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2114
2115     my $volid = $mountpoint->{volume};
2116     my $mount = $mountpoint->{mp};
2117
2118     return if !$volid || !$mount;
2119
2120     my $mount_path;
2121
2122     if (defined($rootdir)) {
2123         $rootdir =~ s!/+$!!;
2124         $mount_path = "$rootdir/$mount";
2125         $mount_path =~ s!/+!/!g;
2126         &$check_mount_path($mount_path);
2127         File::Path::mkpath($mount_path);
2128     }
2129
2130     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2131
2132     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2133
2134     if ($storage) {
2135
2136         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2137         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2138
2139         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2140             PVE::Storage::parse_volname($storage_cfg, $volid);
2141
2142         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2143
2144         if ($format eq 'subvol') {
2145             if ($mount_path) {
2146                 if ($snapname) {
2147                     if ($scfg->{type} ne 'zfspool') {
2148                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2149                     }
2150                 }
2151                 PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2152             }
2153             return wantarray ? ($path, 0) : $path;
2154         } elsif ($format eq 'raw' || $format eq 'iso') {
2155             my $use_loopdev = 0;
2156             my @extra_opts;
2157             if ($scfg->{path}) {
2158                 push @extra_opts, '-o', 'loop';
2159                 $use_loopdev = 1;
2160             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2161                 # do nothing
2162             } else {
2163                 die "unsupported storage type '$scfg->{type}'\n";
2164             }
2165             if ($mount_path) {
2166                 if ($format eq 'iso') {
2167                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2168                 } elsif ($isBase || defined($snapname)) {
2169                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2170                 } else {
2171                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2172                 }
2173             }
2174             return wantarray ? ($path, $use_loopdev) : $path;
2175         } else {
2176             die "unsupported image format '$format'\n";
2177         }
2178     } elsif ($volid =~ m|^/dev/.+|) {
2179         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2180         return wantarray ? ($volid, 0) : $volid;
2181     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2182         &$check_mount_path($volid);
2183         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2184         return wantarray ? ($volid, 0) : $volid;
2185     }
2186
2187     die "unsupported storage";
2188 }
2189
2190 sub get_vm_volumes {
2191     my ($conf, $excludes) = @_;
2192
2193     my $vollist = [];
2194
2195     foreach_mountpoint($conf, sub {
2196         my ($ms, $mountpoint) = @_;
2197
2198         return if $excludes && $ms eq $excludes;
2199
2200         my $volid = $mountpoint->{volume};
2201
2202         return if !$volid || $volid =~ m|^/|;
2203
2204         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2205         return if !$sid;
2206
2207         push @$vollist, $volid;
2208     });
2209
2210     return $vollist;
2211 }
2212
2213 sub mkfs {
2214     my ($dev, $rootuid, $rootgid) = @_;
2215
2216     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2217                              '-E', "root_owner=$rootuid:$rootgid",
2218                              $dev]);
2219 }
2220
2221 sub format_disk {
2222     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2223
2224     if ($volid =~ m!^/dev/.+!) {
2225         mkfs($volid);
2226         return;
2227     }
2228
2229     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2230
2231     die "cannot format volume '$volid' with no storage\n" if !$storage;
2232
2233     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2234
2235     my $path = PVE::Storage::path($storage_cfg, $volid);
2236
2237     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2238         PVE::Storage::parse_volname($storage_cfg, $volid);
2239
2240     die "cannot format volume '$volid' (format == $format)\n"
2241         if $format ne 'raw';
2242
2243     mkfs($path, $rootuid, $rootgid);
2244 }
2245
2246 sub destroy_disks {
2247     my ($storecfg, $vollist) = @_;
2248
2249     foreach my $volid (@$vollist) {
2250         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2251         warn $@ if $@;
2252     }
2253 }
2254
2255 sub create_disks {
2256     my ($storecfg, $vmid, $settings, $conf) = @_;
2257
2258     my $vollist = [];
2259
2260     eval {
2261         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2262         my $chown_vollist = [];
2263
2264         foreach_mountpoint($settings, sub {
2265             my ($ms, $mountpoint) = @_;
2266
2267             my $volid = $mountpoint->{volume};
2268             my $mp = $mountpoint->{mp};
2269
2270             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2271
2272             return if !$storage;
2273
2274             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2275                 my ($storeid, $size_gb) = ($1, $2);
2276
2277                 my $size_kb = int(${size_gb}*1024) * 1024;
2278
2279                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2280                 # fixme: use better naming ct-$vmid-disk-X.raw?
2281
2282                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2283                     if ($size_kb > 0) {
2284                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2285                                                            undef, $size_kb);
2286                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2287                     } else {
2288                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2289                                                            undef, 0);
2290                         push @$chown_vollist, $volid;
2291                     }
2292                 } elsif ($scfg->{type} eq 'zfspool') {
2293
2294                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2295                                                        undef, $size_kb);
2296                     push @$chown_vollist, $volid;
2297                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2298
2299                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2300                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2301
2302                 } elsif ($scfg->{type} eq 'rbd') {
2303
2304                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2305                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2306                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2307                 } else {
2308                     die "unable to create containers on storage type '$scfg->{type}'\n";
2309                 }
2310                 push @$vollist, $volid;
2311                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2312                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2313             } else {
2314                 # use specified/existing volid
2315             }
2316         });
2317
2318         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2319         foreach my $volid (@$chown_vollist) {
2320             my $path = PVE::Storage::path($storecfg, $volid, undef);
2321             chown($rootuid, $rootgid, $path);
2322         }
2323         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2324     };
2325     # free allocated images on error
2326     if (my $err = $@) {
2327         destroy_disks($storecfg, $vollist);
2328         die $err;
2329     }
2330     return $vollist;
2331 }
2332
2333 # bash completion helper
2334
2335 sub complete_os_templates {
2336     my ($cmdname, $pname, $cvalue) = @_;
2337
2338     my $cfg = PVE::Storage::config();
2339
2340     my $storeid;
2341
2342     if ($cvalue =~ m/^([^:]+):/) {
2343         $storeid = $1;
2344     }
2345
2346     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2347     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2348
2349     my $res = [];
2350     foreach my $id (keys %$data) {
2351         foreach my $item (@{$data->{$id}}) {
2352             push @$res, $item->{volid} if defined($item->{volid});
2353         }
2354     }
2355
2356     return $res;
2357 }
2358
2359 my $complete_ctid_full = sub {
2360     my ($running) = @_;
2361
2362     my $idlist = vmstatus();
2363
2364     my $active_hash = list_active_containers();
2365
2366     my $res = [];
2367
2368     foreach my $id (keys %$idlist) {
2369         my $d = $idlist->{$id};
2370         if (defined($running)) {
2371             next if $d->{template};
2372             next if $running && !$active_hash->{$id};
2373             next if !$running && $active_hash->{$id};
2374         }
2375         push @$res, $id;
2376
2377     }
2378     return $res;
2379 };
2380
2381 sub complete_ctid {
2382     return &$complete_ctid_full();
2383 }
2384
2385 sub complete_ctid_stopped {
2386     return &$complete_ctid_full(0);
2387 }
2388
2389 sub complete_ctid_running {
2390     return &$complete_ctid_full(1);
2391 }
2392
2393 sub parse_id_maps {
2394     my ($conf) = @_;
2395
2396     my $id_map = [];
2397     my $rootuid = 0;
2398     my $rootgid = 0;
2399
2400     my $lxc = $conf->{lxc};
2401     foreach my $entry (@$lxc) {
2402         my ($key, $value) = @$entry;
2403         next if $key ne 'lxc.id_map';
2404         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2405             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2406             push @$id_map, [$type, $ct, $host, $length];
2407             if ($ct == 0) {
2408                 $rootuid = $host if $type eq 'u';
2409                 $rootgid = $host if $type eq 'g';
2410             }
2411         } else {
2412             die "failed to parse id_map: $value\n";
2413         }
2414     }
2415
2416     if (!@$id_map && $conf->{unprivileged}) {
2417         # Should we read them from /etc/subuid?
2418         $id_map = [ ['u', '0', '100000', '65536'],
2419                     ['g', '0', '100000', '65536'] ];
2420         $rootuid = $rootgid = 100000;
2421     }
2422
2423     return ($id_map, $rootuid, $rootgid);
2424 }
2425
2426 sub userns_command {
2427     my ($id_map) = @_;
2428     if (@$id_map) {
2429         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2430     }
2431     return [];
2432 }
2433
2434 1;