src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format_description => 'volume',
  42         description => 'Volume, device or directory to mount into the container.',
  43     },
  44     backup => {
  45         type => 'boolean',
  46         format_description => '[1|0]',
  47         description => 'Whether to include the mountpoint in backups.',
  48         optional => 1,
  49     },
  50     size => {
  51         type => 'string',
  52         format => 'disk-size',
  53         format_description => 'DiskSize',
  54         description => 'Volume size (read only value).',
  55         optional => 1,
  56     },
  57 };
  58
  59 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  60     type => 'string', format => $rootfs_desc,
  61     description => "Use volume as container root.",
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 my $confdesc = {
  72     lock => {
  73         optional => 1,
  74         type => 'string',
  75         description => "Lock/unlock the VM.",
  76         enum => [qw(migrate backup snapshot rollback)],
  77     },
  78     onboot => {
  79         optional => 1,
  80         type => 'boolean',
  81         description => "Specifies whether a VM will be started during system bootup.",
  82         default => 0,
  83     },
  84     startup => get_standard_option('pve-startup-order'),
  85     template => {
  86         optional => 1,
  87         type => 'boolean',
  88         description => "Enable/disable Template.",
  89         default => 0,
  90     },
  91     arch => {
  92         optional => 1,
  93         type => 'string',
  94         enum => ['amd64', 'i386'],
  95         description => "OS architecture type.",
  96         default => 'amd64',
  97     },
  98     ostype => {
  99         optional => 1,
 100         type => 'string',
 101         enum => ['debian', 'ubuntu', 'centos', 'fedora', 'opensuse', 'archlinux'],
 102         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 103     },
 104     console => {
 105         optional => 1,
 106         type => 'boolean',
 107         description => "Attach a console device (/dev/console) to the container.",
 108         default => 1,
 109     },
 110     tty => {
 111         optional => 1,
 112         type => 'integer',
 113         description => "Specify the number of tty available to the container",
 114         minimum => 0,
 115         maximum => 6,
 116         default => 2,
 117     },
 118     cpulimit => {
 119         optional => 1,
 120         type => 'number',
 121         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 122         minimum => 0,
 123         maximum => 128,
 124         default => 0,
 125     },
 126     cpuunits => {
 127         optional => 1,
 128         type => 'integer',
 129         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 130         minimum => 0,
 131         maximum => 500000,
 132         default => 1024,
 133     },
 134     memory => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Amount of RAM for the VM in MB.",
 138         minimum => 16,
 139         default => 512,
 140     },
 141     swap => {
 142         optional => 1,
 143         type => 'integer',
 144         description => "Amount of SWAP for the VM in MB.",
 145         minimum => 0,
 146         default => 512,
 147     },
 148     hostname => {
 149         optional => 1,
 150         description => "Set a host name for the container.",
 151         type => 'string', format => 'dns-name',
 152         maxLength => 255,
 153     },
 154     description => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Container description. Only used on the configuration web interface.",
 158     },
 159     searchdomain => {
 160         optional => 1,
 161         type => 'string', format => 'dns-name-list',
 162         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 163     },
 164     nameserver => {
 165         optional => 1,
 166         type => 'string', format => 'address-list',
 167         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 168     },
 169     rootfs => get_standard_option('pve-ct-rootfs'),
 170     parent => {
 171         optional => 1,
 172         type => 'string', format => 'pve-configid',
 173         maxLength => 40,
 174         description => "Parent snapshot name. This is used internally, and should not be modified.",
 175     },
 176     snaptime => {
 177         optional => 1,
 178         description => "Timestamp for snapshots.",
 179         type => 'integer',
 180         minimum => 0,
 181     },
 182     cmode => {
 183         optional => 1,
 184         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 185         type => 'string',
 186         enum => ['shell', 'console', 'tty'],
 187         default => 'tty',
 188     },
 189     protection => {
 190         optional => 1,
 191         type => 'boolean',
 192         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 193         default => 0,
 194     },
 195     unprivileged => {
 196         optional => 1,
 197         type => 'boolean',
 198         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 199         default => 0,
 200     },
 201 };
 202
 203 my $valid_lxc_conf_keys = {
 204     'lxc.include' => 1,
 205     'lxc.arch' => 1,
 206     'lxc.utsname' => 1,
 207     'lxc.haltsignal' => 1,
 208     'lxc.rebootsignal' => 1,
 209     'lxc.stopsignal' => 1,
 210     'lxc.init_cmd' => 1,
 211     'lxc.network.type' => 1,
 212     'lxc.network.flags' => 1,
 213     'lxc.network.link' => 1,
 214     'lxc.network.mtu' => 1,
 215     'lxc.network.name' => 1,
 216     'lxc.network.hwaddr' => 1,
 217     'lxc.network.ipv4' => 1,
 218     'lxc.network.ipv4.gateway' => 1,
 219     'lxc.network.ipv6' => 1,
 220     'lxc.network.ipv6.gateway' => 1,
 221     'lxc.network.script.up' => 1,
 222     'lxc.network.script.down' => 1,
 223     'lxc.pts' => 1,
 224     'lxc.console.logfile' => 1,
 225     'lxc.console' => 1,
 226     'lxc.tty' => 1,
 227     'lxc.devttydir' => 1,
 228     'lxc.hook.autodev' => 1,
 229     'lxc.autodev' => 1,
 230     'lxc.kmsg' => 1,
 231     'lxc.mount' => 1,
 232     'lxc.mount.entry' => 1,
 233     'lxc.mount.auto' => 1,
 234     'lxc.rootfs' => 1,
 235     'lxc.rootfs.mount' => 1,
 236     'lxc.rootfs.options' => 1,
 237     # lxc.cgroup.*
 238     'lxc.cap.drop' => 1,
 239     'lxc.cap.keep' => 1,
 240     'lxc.aa_profile' => 1,
 241     'lxc.aa_allow_incomplete' => 1,
 242     'lxc.se_context' => 1,
 243     'lxc.seccomp' => 1,
 244     'lxc.id_map' => 1,
 245     'lxc.hook.pre-start' => 1,
 246     'lxc.hook.pre-mount' => 1,
 247     'lxc.hook.mount' => 1,
 248     'lxc.hook.start' => 1,
 249     'lxc.hook.stop' => 1,
 250     'lxc.hook.post-stop' => 1,
 251     'lxc.hook.clone' => 1,
 252     'lxc.hook.destroy' => 1,
 253     'lxc.loglevel' => 1,
 254     'lxc.logfile' => 1,
 255     'lxc.start.auto' => 1,
 256     'lxc.start.delay' => 1,
 257     'lxc.start.order' => 1,
 258     'lxc.group' => 1,
 259     'lxc.environment' => 1,
 260     'lxc.' => 1,
 261     'lxc.' => 1,
 262     'lxc.' => 1,
 263     'lxc.' => 1,
 264 };
 265
 266 my $netconf_desc = {
 267     type => {
 268         type => 'string',
 269         optional => 1,
 270         description => "Network interface type.",
 271         enum => [qw(veth)],
 272     },
 273     name => {
 274         type => 'string',
 275         format_description => 'String',
 276         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 277         pattern => '[-_.\w\d]+',
 278     },
 279     bridge => {
 280         type => 'string',
 281         format_description => 'vmbr<Number>',
 282         description => 'Bridge to attach the network device to.',
 283         pattern => '[-_.\w\d]+',
 284         optional => 1,
 285     },
 286     hwaddr => {
 287         type => 'string',
 288         format_description => 'MAC',
 289         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 290         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 291         optional => 1,
 292     },
 293     mtu => {
 294         type => 'integer',
 295         format_description => 'Number',
 296         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 297         minimum => 64, # minimum ethernet frame is 64 bytes
 298         optional => 1,
 299     },
 300     ip => {
 301         type => 'string',
 302         format => 'pve-ipv4-config',
 303         format_description => 'IPv4Format/CIDR',
 304         description => 'IPv4 address in CIDR format.',
 305         optional => 1,
 306     },
 307     gw => {
 308         type => 'string',
 309         format => 'ipv4',
 310         format_description => 'GatewayIPv4',
 311         description => 'Default gateway for IPv4 traffic.',
 312         optional => 1,
 313     },
 314     ip6 => {
 315         type => 'string',
 316         format => 'pve-ipv6-config',
 317         format_description => 'IPv6Format/CIDR',
 318         description => 'IPv6 address in CIDR format.',
 319         optional => 1,
 320     },
 321     gw6 => {
 322         type => 'string',
 323         format => 'ipv6',
 324         format_description => 'GatewayIPv6',
 325         description => 'Default gateway for IPv6 traffic.',
 326         optional => 1,
 327     },
 328     firewall => {
 329         type => 'boolean',
 330         format_description => '[1|0]',
 331         description => "Controls whether this interface's firewall rules should be used.",
 332         optional => 1,
 333     },
 334     tag => {
 335         type => 'integer',
 336         format_description => 'VlanNo',
 337         minimum => '2',
 338         maximum => '4094',
 339         description => "VLAN tag foro this interface.",
 340         optional => 1,
 341     },
 342 };
 343 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 344
 345 my $MAX_LXC_NETWORKS = 10;
 346 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 347     $confdesc->{"net$i"} = {
 348         optional => 1,
 349         type => 'string', format => $netconf_desc,
 350         description => "Specifies network interfaces for the container.",
 351     };
 352 }
 353
 354 my $mp_desc = {
 355     %$rootfs_desc,
 356     mp => {
 357         type => 'string',
 358         format_description => 'Path',
 359         description => 'Path to the mountpoint as seen from inside the container.',
 360         optional => 1,
 361     },
 362 };
 363 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 364
 365 my $unuseddesc = {
 366     optional => 1,
 367     type => 'string', format => 'pve-volume-id',
 368     description => "Reference to unused volumes.",
 369 };
 370
 371 my $MAX_MOUNT_POINTS = 10;
 372 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 373     $confdesc->{"mp$i"} = {
 374         optional => 1,
 375         type => 'string', format => $mp_desc,
 376         description => "Use volume as container mount point (experimental feature).",
 377         optional => 1,
 378     };
 379 }
 380
 381 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 382 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 383     $confdesc->{"unused$i"} = $unuseddesc;
 384 }
 385
 386 sub write_pct_config {
 387     my ($filename, $conf) = @_;
 388
 389     delete $conf->{snapstate}; # just to be sure
 390
 391     my $generate_raw_config = sub {
 392         my ($conf) = @_;
 393
 394         my $raw = '';
 395
 396         # add description as comment to top of file
 397         my $descr = $conf->{description} || '';
 398         foreach my $cl (split(/\n/, $descr)) {
 399             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 400         }
 401
 402         foreach my $key (sort keys %$conf) {
 403             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 404                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 405             my $value = $conf->{$key};
 406             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 407             $raw .= "$key: $value\n";
 408         }
 409
 410         if (my $lxcconf = $conf->{lxc}) {
 411             foreach my $entry (@$lxcconf) {
 412                 my ($k, $v) = @$entry;
 413                 $raw .= "$k: $v\n";
 414             }
 415         }
 416
 417         return $raw;
 418     };
 419
 420     my $raw = &$generate_raw_config($conf);
 421
 422     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 423         $raw .= "\n[$snapname]\n";
 424         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 425     }
 426
 427     return $raw;
 428 }
 429
 430 sub check_type {
 431     my ($key, $value) = @_;
 432
 433     die "unknown setting '$key'\n" if !$confdesc->{$key};
 434
 435     my $type = $confdesc->{$key}->{type};
 436
 437     if (!defined($value)) {
 438         die "got undefined value\n";
 439     }
 440
 441     if ($value =~ m/[\n\r]/) {
 442         die "property contains a line feed\n";
 443     }
 444
 445     if ($type eq 'boolean') {
 446         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 447         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 448         die "type check ('boolean') failed - got '$value'\n";
 449     } elsif ($type eq 'integer') {
 450         return int($1) if $value =~ m/^(\d+)$/;
 451         die "type check ('integer') failed - got '$value'\n";
 452     } elsif ($type eq 'number') {
 453         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 454         die "type check ('number') failed - got '$value'\n";
 455     } elsif ($type eq 'string') {
 456         if (my $fmt = $confdesc->{$key}->{format}) {
 457             PVE::JSONSchema::check_format($fmt, $value);
 458             return $value;
 459         }
 460         return $value;
 461     } else {
 462         die "internal error"
 463     }
 464 }
 465
 466 sub parse_pct_config {
 467     my ($filename, $raw) = @_;
 468
 469     return undef if !defined($raw);
 470
 471     my $res = {
 472         digest => Digest::SHA::sha1_hex($raw),
 473         snapshots => {},
 474     };
 475
 476     $filename =~ m|/lxc/(\d+).conf$|
 477         || die "got strange filename '$filename'";
 478
 479     my $vmid = $1;
 480
 481     my $conf = $res;
 482     my $descr = '';
 483     my $section = '';
 484
 485     my @lines = split(/\n/, $raw);
 486     foreach my $line (@lines) {
 487         next if $line =~ m/^\s*$/;
 488
 489         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 490             $section = $1;
 491             $conf->{description} = $descr if $descr;
 492             $descr = '';
 493             $conf = $res->{snapshots}->{$section} = {};
 494             next;
 495         }
 496
 497         if ($line =~ m/^\#(.*)\s*$/) {
 498             $descr .= PVE::Tools::decode_text($1) . "\n";
 499             next;
 500         }
 501
 502         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 503             my $key = $1;
 504             my $value = $3;
 505             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 506                 push @{$conf->{lxc}}, [$key, $value];
 507             } else {
 508                 warn "vm $vmid - unable to parse config: $line\n";
 509             }
 510         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 511             $descr .= PVE::Tools::decode_text($2);
 512         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 513             $conf->{snapstate} = $1;
 514         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 515             my $key = $1;
 516             my $value = $2;
 517             eval { $value = check_type($key, $value); };
 518             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 519             $conf->{$key} = $value;
 520         } else {
 521             warn "vm $vmid - unable to parse config: $line\n";
 522         }
 523     }
 524
 525     $conf->{description} = $descr if $descr;
 526
 527     delete $res->{snapstate}; # just to be sure
 528
 529     return $res;
 530 }
 531
 532 sub config_list {
 533     my $vmlist = PVE::Cluster::get_vmlist();
 534     my $res = {};
 535     return $res if !$vmlist || !$vmlist->{ids};
 536     my $ids = $vmlist->{ids};
 537
 538     foreach my $vmid (keys %$ids) {
 539         next if !$vmid; # skip CT0
 540         my $d = $ids->{$vmid};
 541         next if !$d->{node} || $d->{node} ne $nodename;
 542         next if !$d->{type} || $d->{type} ne 'lxc';
 543         $res->{$vmid}->{type} = 'lxc';
 544     }
 545     return $res;
 546 }
 547
 548 sub cfs_config_path {
 549     my ($vmid, $node) = @_;
 550
 551     $node = $nodename if !$node;
 552     return "nodes/$node/lxc/$vmid.conf";
 553 }
 554
 555 sub config_file {
 556     my ($vmid, $node) = @_;
 557
 558     my $cfspath = cfs_config_path($vmid, $node);
 559     return "/etc/pve/$cfspath";
 560 }
 561
 562 sub load_config {
 563     my ($vmid, $node) = @_;
 564
 565     $node = $nodename if !$node;
 566     my $cfspath = cfs_config_path($vmid, $node);
 567
 568     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 569     die "container $vmid does not exists\n" if !defined($conf);
 570
 571     return $conf;
 572 }
 573
 574 sub create_config {
 575     my ($vmid, $conf) = @_;
 576
 577     my $dir = "/etc/pve/nodes/$nodename/lxc";
 578     mkdir $dir;
 579
 580     write_config($vmid, $conf);
 581 }
 582
 583 sub destroy_config {
 584     my ($vmid) = @_;
 585
 586     unlink config_file($vmid, $nodename);
 587 }
 588
 589 sub write_config {
 590     my ($vmid, $conf) = @_;
 591
 592     my $cfspath = cfs_config_path($vmid);
 593
 594     PVE::Cluster::cfs_write_file($cfspath, $conf);
 595 }
 596
 597 # flock: we use one file handle per process, so lock file
 598 # can be called multiple times and succeeds for the same process.
 599
 600 my $lock_handles =  {};
 601 my $lockdir = "/run/lock/lxc";
 602
 603 sub lock_filename {
 604     my ($vmid) = @_;
 605
 606     return "$lockdir/pve-config-${vmid}.lock";
 607 }
 608
 609 sub lock_aquire {
 610     my ($vmid, $timeout) = @_;
 611
 612     $timeout = 10 if !$timeout;
 613     my $mode = LOCK_EX;
 614
 615     my $filename = lock_filename($vmid);
 616
 617     mkdir $lockdir if !-d $lockdir;
 618
 619     my $lock_func = sub {
 620         if (!$lock_handles->{$$}->{$filename}) {
 621             my $fh = new IO::File(">>$filename") ||
 622                 die "can't open file - $!\n";
 623             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 624         }
 625
 626         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 627             print STDERR "trying to aquire lock...";
 628             my $success;
 629             while(1) {
 630                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 631                 # try again on EINTR (see bug #273)
 632                 if ($success || ($! != EINTR)) {
 633                     last;
 634                 }
 635             }
 636             if (!$success) {
 637                 print STDERR " failed\n";
 638                 die "can't aquire lock - $!\n";
 639             }
 640
 641             print STDERR " OK\n";
 642         }
 643
 644         $lock_handles->{$$}->{$filename}->{refcount}++;
 645     };
 646
 647     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 648     my $err = $@;
 649     if ($err) {
 650         die "can't lock file '$filename' - $err";
 651     }
 652 }
 653
 654 sub lock_release {
 655     my ($vmid) = @_;
 656
 657     my $filename = lock_filename($vmid);
 658
 659     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 660         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 661         if ($refcount <= 0) {
 662             $lock_handles->{$$}->{$filename} = undef;
 663             close ($fh);
 664         }
 665     }
 666 }
 667
 668 sub lock_container {
 669     my ($vmid, $timeout, $code, @param) = @_;
 670
 671     my $res;
 672
 673     lock_aquire($vmid, $timeout);
 674     eval { $res = &$code(@param) };
 675     my $err = $@;
 676     lock_release($vmid);
 677
 678     die $err if $err;
 679
 680     return $res;
 681 }
 682
 683 sub option_exists {
 684     my ($name) = @_;
 685
 686     return defined($confdesc->{$name});
 687 }
 688
 689 # add JSON properties for create and set function
 690 sub json_config_properties {
 691     my $prop = shift;
 692
 693     foreach my $opt (keys %$confdesc) {
 694         next if $opt eq 'parent' || $opt eq 'snaptime';
 695         next if $prop->{$opt};
 696         $prop->{$opt} = $confdesc->{$opt};
 697     }
 698
 699     return $prop;
 700 }
 701
 702 sub json_config_properties_no_rootfs {
 703     my $prop = shift;
 704
 705     foreach my $opt (keys %$confdesc) {
 706         next if $prop->{$opt};
 707         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 708         $prop->{$opt} = $confdesc->{$opt};
 709     }
 710
 711     return $prop;
 712 }
 713
 714 # container status helpers
 715
 716 sub list_active_containers {
 717
 718     my $filename = "/proc/net/unix";
 719
 720     # similar test is used by lcxcontainers.c: list_active_containers
 721     my $res = {};
 722
 723     my $fh = IO::File->new ($filename, "r");
 724     return $res if !$fh;
 725
 726     while (defined(my $line = <$fh>)) {
 727         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 728             my $path = $1;
 729             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 730                 $res->{$1} = 1;
 731             }
 732         }
 733     }
 734
 735     close($fh);
 736
 737     return $res;
 738 }
 739
 740 # warning: this is slow
 741 sub check_running {
 742     my ($vmid) = @_;
 743
 744     my $active_hash = list_active_containers();
 745
 746     return 1 if defined($active_hash->{$vmid});
 747
 748     return undef;
 749 }
 750
 751 sub get_container_disk_usage {
 752     my ($vmid, $pid) = @_;
 753
 754     return PVE::Tools::df("/proc/$pid/root/", 1);
 755 }
 756
 757 my $last_proc_vmid_stat;
 758
 759 my $parse_cpuacct_stat = sub {
 760     my ($vmid) = @_;
 761
 762     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 763
 764     my $stat = {};
 765
 766     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 767
 768         $stat->{utime} = $1;
 769         $stat->{stime} = $2;
 770
 771     }
 772
 773     return $stat;
 774 };
 775
 776 sub vmstatus {
 777     my ($opt_vmid) = @_;
 778
 779     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 780
 781     my $active_hash = list_active_containers();
 782
 783     my $cpucount = $cpuinfo->{cpus} || 1;
 784
 785     my $cdtime = gettimeofday;
 786
 787     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 788
 789     foreach my $vmid (keys %$list) {
 790         my $d = $list->{$vmid};
 791
 792         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 793         warn $@ if $@; # ignore errors (consider them stopped)
 794
 795         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 796
 797         my $cfspath = cfs_config_path($vmid);
 798         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 799
 800         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 801         $d->{name} =~ s/[\s]//g;
 802
 803         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 804
 805         if ($d->{pid}) {
 806             my $res = get_container_disk_usage($vmid, $d->{pid});
 807             $d->{disk} = $res->{used};
 808             $d->{maxdisk} = $res->{total};
 809         } else {
 810             $d->{disk} = 0;
 811             # use 4GB by default ??
 812             if (my $rootfs = $conf->{rootfs}) {
 813                 my $rootinfo = parse_ct_mountpoint($rootfs);
 814                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 815             } else {
 816                 $d->{maxdisk} = 4*1024*1024*1024;
 817             }
 818         }
 819
 820         $d->{mem} = 0;
 821         $d->{swap} = 0;
 822         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 823         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 824
 825         $d->{uptime} = 0;
 826         $d->{cpu} = 0;
 827
 828         $d->{netout} = 0;
 829         $d->{netin} = 0;
 830
 831         $d->{diskread} = 0;
 832         $d->{diskwrite} = 0;
 833
 834         $d->{template} = is_template($conf);
 835     }
 836
 837     foreach my $vmid (keys %$list) {
 838         my $d = $list->{$vmid};
 839         my $pid = $d->{pid};
 840
 841         next if !$pid; # skip stopped CTs
 842
 843         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 844         $d->{uptime} = time - $ctime; # the method lxcfs uses
 845
 846         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 847         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 848
 849         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 850         my @bytes = split(/\n/, $blkio_bytes);
 851         foreach my $byte (@bytes) {
 852             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 853                 $d->{diskread} = $2 if $key eq 'Read';
 854                 $d->{diskwrite} = $2 if $key eq 'Write';
 855             }
 856         }
 857
 858         my $pstat = &$parse_cpuacct_stat($vmid);
 859
 860         my $used = $pstat->{utime} + $pstat->{stime};
 861
 862         my $old = $last_proc_vmid_stat->{$vmid};
 863         if (!$old) {
 864             $last_proc_vmid_stat->{$vmid} = {
 865                 time => $cdtime,
 866                 used => $used,
 867                 cpu => 0,
 868             };
 869             next;
 870         }
 871
 872         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 873
 874         if ($dtime > 1000) {
 875             my $dutime = $used -  $old->{used};
 876
 877             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 878             $last_proc_vmid_stat->{$vmid} = {
 879                 time => $cdtime,
 880                 used => $used,
 881                 cpu => $d->{cpu},
 882             };
 883         } else {
 884             $d->{cpu} = $old->{cpu};
 885         }
 886     }
 887
 888     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 889
 890     foreach my $dev (keys %$netdev) {
 891         next if $dev !~ m/^veth([1-9]\d*)i/;
 892         my $vmid = $1;
 893         my $d = $list->{$vmid};
 894
 895         next if !$d;
 896
 897         $d->{netout} += $netdev->{$dev}->{receive};
 898         $d->{netin} += $netdev->{$dev}->{transmit};
 899
 900     }
 901
 902     return $list;
 903 }
 904
 905 sub classify_mountpoint {
 906     my ($vol) = @_;
 907     if ($vol =~ m!^/!) {
 908         return 'device' if $vol =~ m!^/dev/!;
 909         return 'bind';
 910     }
 911     return 'volume';
 912 }
 913
 914 sub parse_ct_mountpoint {
 915     my ($data, $noerr) = @_;
 916
 917     $data //= '';
 918
 919     my $res;
 920     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 921     if ($@) {
 922         return undef if $noerr;
 923         die $@;
 924     }
 925
 926     if (defined(my $size = $res->{size})) {
 927         $size = PVE::JSONSchema::parse_size($size);
 928         if (!defined($size)) {
 929             return undef if $noerr;
 930             die "invalid size: $size\n";
 931         }
 932         $res->{size} = $size;
 933     }
 934
 935     $res->{type} = classify_mountpoint($res->{volume});
 936
 937     return $res;
 938 }
 939
 940 sub print_ct_mountpoint {
 941     my ($info, $nomp) = @_;
 942     my $skip = [ 'type' ];
 943     push @$skip, 'mp' if $nomp;
 944     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 945 }
 946
 947 sub print_lxc_network {
 948     my $net = shift;
 949     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 950 }
 951
 952 sub parse_lxc_network {
 953     my ($data) = @_;
 954
 955     my $res = {};
 956
 957     return $res if !$data;
 958
 959     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 960
 961     $res->{type} = 'veth';
 962     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 963
 964     return $res;
 965 }
 966
 967 sub read_cgroup_value {
 968     my ($group, $vmid, $name, $full) = @_;
 969
 970     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 971
 972     return PVE::Tools::file_get_contents($path) if $full;
 973
 974     return PVE::Tools::file_read_firstline($path);
 975 }
 976
 977 sub write_cgroup_value {
 978    my ($group, $vmid, $name, $value) = @_;
 979
 980    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 981    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 982
 983 }
 984
 985 sub find_lxc_console_pids {
 986
 987     my $res = {};
 988
 989     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 990         my ($pid) = @_;
 991
 992         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 993         return if !$cmdline;
 994
 995         my @args = split(/\0/, $cmdline);
 996
 997         # serach for lxc-console -n <vmid>
 998         return if scalar(@args) != 3;
 999         return if $args[1] ne '-n';
1000         return if $args[2] !~ m/^\d+$/;
1001         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1002
1003         my $vmid = $args[2];
1004
1005         push @{$res->{$vmid}}, $pid;
1006     });
1007
1008     return $res;
1009 }
1010
1011 sub find_lxc_pid {
1012     my ($vmid) = @_;
1013
1014     my $pid = undef;
1015     my $parser = sub {
1016         my $line = shift;
1017         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1018     };
1019     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1020
1021     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1022
1023     return $pid;
1024 }
1025
1026 # Note: we cannot use Net:IP, because that only allows strict
1027 # CIDR networks
1028 sub parse_ipv4_cidr {
1029     my ($cidr, $noerr) = @_;
1030
1031     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1032         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1033     }
1034
1035     return undef if $noerr;
1036
1037     die "unable to parse ipv4 address/mask\n";
1038 }
1039
1040 sub check_lock {
1041     my ($conf) = @_;
1042
1043     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1044 }
1045
1046 sub check_protection {
1047     my ($vm_conf, $err_msg) = @_;
1048
1049     if ($vm_conf->{protection}) {
1050         die "$err_msg - protection mode enabled\n";
1051     }
1052 }
1053
1054 sub update_lxc_config {
1055     my ($storage_cfg, $vmid, $conf) = @_;
1056
1057     my $dir = "/var/lib/lxc/$vmid";
1058
1059     if ($conf->{template}) {
1060
1061         unlink "$dir/config";
1062
1063         return;
1064     }
1065
1066     my $raw = '';
1067
1068     die "missing 'arch' - internal error" if !$conf->{arch};
1069     $raw .= "lxc.arch = $conf->{arch}\n";
1070
1071     my $unprivileged = $conf->{unprivileged};
1072     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1073
1074     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1075     if ($ostype =~ /^(?:debian | ubuntu | centos | fedora | opensuse | archlinux)$/x) {
1076         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1077         if ($unprivileged || $custom_idmap) {
1078             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1079         }
1080     } else {
1081         die "implement me (ostype $ostype)";
1082     }
1083
1084     $raw .= "lxc.monitor.unshare = 1\n";
1085
1086     # Should we read them from /etc/subuid?
1087     if ($unprivileged && !$custom_idmap) {
1088         $raw .= "lxc.id_map = u 0 100000 65536\n";
1089         $raw .= "lxc.id_map = g 0 100000 65536\n";
1090     }
1091
1092     if (!has_dev_console($conf)) {
1093         $raw .= "lxc.console = none\n";
1094         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1095     }
1096
1097     my $ttycount = get_tty_count($conf);
1098     $raw .= "lxc.tty = $ttycount\n";
1099
1100     # some init scripts expects a linux terminal (turnkey).
1101     $raw .= "lxc.environment = TERM=linux\n";
1102
1103     my $utsname = $conf->{hostname} || "CT$vmid";
1104     $raw .= "lxc.utsname = $utsname\n";
1105
1106     my $memory = $conf->{memory} || 512;
1107     my $swap = $conf->{swap} // 0;
1108
1109     my $lxcmem = int($memory*1024*1024);
1110     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1111
1112     my $lxcswap = int(($memory + $swap)*1024*1024);
1113     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1114
1115     if (my $cpulimit = $conf->{cpulimit}) {
1116         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1117         my $value = int(100000*$cpulimit);
1118         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1119     }
1120
1121     my $shares = $conf->{cpuunits} || 1024;
1122     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1123
1124     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1125     $mountpoint->{mp} = '/';
1126
1127     $raw .= "lxc.rootfs = $dir/rootfs\n";
1128
1129     my $netcount = 0;
1130     foreach my $k (keys %$conf) {
1131         next if $k !~ m/^net(\d+)$/;
1132         my $ind = $1;
1133         my $d = parse_lxc_network($conf->{$k});
1134         $netcount++;
1135         $raw .= "lxc.network.type = veth\n";
1136         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1137         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1138         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1139         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1140     }
1141
1142     if (my $lxcconf = $conf->{lxc}) {
1143         foreach my $entry (@$lxcconf) {
1144             my ($k, $v) = @$entry;
1145             $netcount++ if $k eq 'lxc.network.type';
1146             $raw .= "$k = $v\n";
1147         }
1148     }
1149
1150     $raw .= "lxc.network.type = empty\n" if !$netcount;
1151
1152     File::Path::mkpath("$dir/rootfs");
1153
1154     PVE::Tools::file_set_contents("$dir/config", $raw);
1155 }
1156
1157 # verify and cleanup nameserver list (replace \0 with ' ')
1158 sub verify_nameserver_list {
1159     my ($nameserver_list) = @_;
1160
1161     my @list = ();
1162     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1163         PVE::JSONSchema::pve_verify_ip($server);
1164         push @list, $server;
1165     }
1166
1167     return join(' ', @list);
1168 }
1169
1170 sub verify_searchdomain_list {
1171     my ($searchdomain_list) = @_;
1172
1173     my @list = ();
1174     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1175         # todo: should we add checks for valid dns domains?
1176         push @list, $server;
1177     }
1178
1179     return join(' ', @list);
1180 }
1181
1182 sub add_unused_volume {
1183     my ($config, $volid) = @_;
1184
1185     my $key;
1186     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1187         my $test = "unused$ind";
1188         if (my $vid = $config->{$test}) {
1189             return if $vid eq $volid; # do not add duplicates
1190         } else {
1191             $key = $test;
1192         }
1193     }
1194
1195     die "To many unused volume - please delete them first.\n" if !$key;
1196
1197     $config->{$key} = $volid;
1198
1199     return $key;
1200 }
1201
1202 sub update_pct_config {
1203     my ($vmid, $conf, $running, $param, $delete) = @_;
1204
1205     my @nohotplug;
1206
1207     my $new_disks = 0;
1208     my @deleted_volumes;
1209
1210     my $rootdir;
1211     if ($running) {
1212         my $pid = find_lxc_pid($vmid);
1213         $rootdir = "/proc/$pid/root";
1214     }
1215
1216     my $hotplug_error = sub {
1217         if ($running) {
1218             push @nohotplug, @_;
1219             return 1;
1220         } else {
1221             return 0;
1222         }
1223     };
1224
1225     if (defined($delete)) {
1226         foreach my $opt (@$delete) {
1227             if (!exists($conf->{$opt})) {
1228                 warn "no such option: $opt\n";
1229                 next;
1230             }
1231
1232             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1233                 die "unable to delete required option '$opt'\n";
1234             } elsif ($opt eq 'swap') {
1235                 delete $conf->{$opt};
1236                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1237             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1238                 delete $conf->{$opt};
1239             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1240                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1241                 next if $hotplug_error->($opt);
1242                 delete $conf->{$opt};
1243             } elsif ($opt =~ m/^net(\d)$/) {
1244                 delete $conf->{$opt};
1245                 next if !$running;
1246                 my $netid = $1;
1247                 PVE::Network::veth_delete("veth${vmid}i$netid");
1248             } elsif ($opt eq 'protection') {
1249                 delete $conf->{$opt};
1250             } elsif ($opt =~ m/^unused(\d+)$/) {
1251                 next if $hotplug_error->($opt);
1252                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1253                 push @deleted_volumes, $conf->{$opt};
1254                 delete $conf->{$opt};
1255             } elsif ($opt =~ m/^mp(\d+)$/) {
1256                 next if $hotplug_error->($opt);
1257                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1258                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1259                 if ($mountpoint->{type} eq 'volume') {
1260                     add_unused_volume($conf, $mountpoint->{volume})
1261                 }
1262                 delete $conf->{$opt};
1263             } elsif ($opt eq 'unprivileged') {
1264                 die "unable to delete read-only option: '$opt'\n";
1265             } else {
1266                 die "implement me (delete: $opt)"
1267             }
1268             write_config($vmid, $conf) if $running;
1269         }
1270     }
1271
1272     # There's no separate swap size to configure, there's memory and "total"
1273     # memory (iow. memory+swap). This means we have to change them together.
1274     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1275     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1276     if (defined($wanted_memory) || defined($wanted_swap)) {
1277
1278         $wanted_memory //= ($conf->{memory} || 512);
1279         $wanted_swap //=  ($conf->{swap} || 0);
1280
1281         my $total = $wanted_memory + $wanted_swap;
1282         if ($running) {
1283             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1284             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1285         }
1286         $conf->{memory} = $wanted_memory;
1287         $conf->{swap} = $wanted_swap;
1288
1289         write_config($vmid, $conf) if $running;
1290     }
1291
1292     foreach my $opt (keys %$param) {
1293         my $value = $param->{$opt};
1294         if ($opt eq 'hostname') {
1295             $conf->{$opt} = $value;
1296         } elsif ($opt eq 'onboot') {
1297             $conf->{$opt} = $value ? 1 : 0;
1298         } elsif ($opt eq 'startup') {
1299             $conf->{$opt} = $value;
1300         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1301             next if $hotplug_error->($opt);
1302             $conf->{$opt} = $value;
1303         } elsif ($opt eq 'nameserver') {
1304             next if $hotplug_error->($opt);
1305             my $list = verify_nameserver_list($value);
1306             $conf->{$opt} = $list;
1307         } elsif ($opt eq 'searchdomain') {
1308             next if $hotplug_error->($opt);
1309             my $list = verify_searchdomain_list($value);
1310             $conf->{$opt} = $list;
1311         } elsif ($opt eq 'cpulimit') {
1312             next if $hotplug_error->($opt); # FIXME: hotplug
1313             $conf->{$opt} = $value;
1314         } elsif ($opt eq 'cpuunits') {
1315             $conf->{$opt} = $value;
1316             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1317         } elsif ($opt eq 'description') {
1318             $conf->{$opt} = PVE::Tools::encode_text($value);
1319         } elsif ($opt =~ m/^net(\d+)$/) {
1320             my $netid = $1;
1321             my $net = parse_lxc_network($value);
1322             if (!$running) {
1323                 $conf->{$opt} = print_lxc_network($net);
1324             } else {
1325                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1326             }
1327         } elsif ($opt eq 'protection') {
1328             $conf->{$opt} = $value ? 1 : 0;
1329         } elsif ($opt =~ m/^mp(\d+)$/) {
1330             next if $hotplug_error->($opt);
1331             check_protection($conf, "can't update CT $vmid drive '$opt'");
1332             $conf->{$opt} = $value;
1333             $new_disks = 1;
1334         } elsif ($opt eq 'rootfs') {
1335             check_protection($conf, "can't update CT $vmid drive '$opt'");
1336             die "implement me: $opt";
1337         } elsif ($opt eq 'unprivileged') {
1338             die "unable to modify read-only option: '$opt'\n";
1339         } else {
1340             die "implement me: $opt";
1341         }
1342         write_config($vmid, $conf) if $running;
1343     }
1344
1345     if (@deleted_volumes) {
1346         my $storage_cfg = PVE::Storage::config();
1347         foreach my $volume (@deleted_volumes) {
1348             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1349         }
1350     }
1351
1352     if ($new_disks) {
1353         my $storage_cfg = PVE::Storage::config();
1354         create_disks($storage_cfg, $vmid, $conf, $conf);
1355     }
1356
1357     # This should be the last thing we do here
1358     if ($running && scalar(@nohotplug)) {
1359         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1360     }
1361 }
1362
1363 sub has_dev_console {
1364     my ($conf) = @_;
1365
1366     return !(defined($conf->{console}) && !$conf->{console});
1367 }
1368
1369 sub get_tty_count {
1370     my ($conf) = @_;
1371
1372     return $conf->{tty} // $confdesc->{tty}->{default};
1373 }
1374
1375 sub get_cmode {
1376     my ($conf) = @_;
1377
1378     return $conf->{cmode} // $confdesc->{cmode}->{default};
1379 }
1380
1381 sub get_console_command {
1382     my ($vmid, $conf) = @_;
1383
1384     my $cmode = get_cmode($conf);
1385
1386     if ($cmode eq 'console') {
1387         return ['lxc-console', '-n',  $vmid, '-t', 0];
1388     } elsif ($cmode eq 'tty') {
1389         return ['lxc-console', '-n',  $vmid];
1390     } elsif ($cmode eq 'shell') {
1391         return ['lxc-attach', '--clear-env', '-n', $vmid];
1392     } else {
1393         die "internal error";
1394     }
1395 }
1396
1397 sub get_primary_ips {
1398     my ($conf) = @_;
1399
1400     # return data from net0
1401
1402     return undef if !defined($conf->{net0});
1403     my $net = parse_lxc_network($conf->{net0});
1404
1405     my $ipv4 = $net->{ip};
1406     if ($ipv4) {
1407         if ($ipv4 =~ /^(dhcp|manual)$/) {
1408             $ipv4 = undef
1409         } else {
1410             $ipv4 =~ s!/\d+$!!;
1411         }
1412     }
1413     my $ipv6 = $net->{ip6};
1414     if ($ipv6) {
1415         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1416             $ipv6 = undef;
1417         } else {
1418             $ipv6 =~ s!/\d+$!!;
1419         }
1420     }
1421
1422     return ($ipv4, $ipv6);
1423 }
1424
1425 sub delete_mountpoint_volume {
1426     my ($storage_cfg, $vmid, $volume) = @_;
1427
1428     return if classify_mountpoint($volume) ne 'volume';
1429
1430     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1431     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1432 }
1433
1434 sub destroy_lxc_container {
1435     my ($storage_cfg, $vmid, $conf) = @_;
1436
1437     foreach_mountpoint($conf, sub {
1438         my ($ms, $mountpoint) = @_;
1439         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1440     });
1441
1442     rmdir "/var/lib/lxc/$vmid/rootfs";
1443     unlink "/var/lib/lxc/$vmid/config";
1444     rmdir "/var/lib/lxc/$vmid";
1445     destroy_config($vmid);
1446
1447     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1448     #PVE::Tools::run_command($cmd);
1449 }
1450
1451 sub vm_stop_cleanup {
1452     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1453
1454     eval {
1455         if (!$keepActive) {
1456
1457             my $vollist = get_vm_volumes($conf);
1458             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1459         }
1460     };
1461     warn $@ if $@; # avoid errors - just warn
1462 }
1463
1464 my $safe_num_ne = sub {
1465     my ($a, $b) = @_;
1466
1467     return 0 if !defined($a) && !defined($b);
1468     return 1 if !defined($a);
1469     return 1 if !defined($b);
1470
1471     return $a != $b;
1472 };
1473
1474 my $safe_string_ne = sub {
1475     my ($a, $b) = @_;
1476
1477     return 0 if !defined($a) && !defined($b);
1478     return 1 if !defined($a);
1479     return 1 if !defined($b);
1480
1481     return $a ne $b;
1482 };
1483
1484 sub update_net {
1485     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1486
1487     if ($newnet->{type} ne 'veth') {
1488         # for when there are physical interfaces
1489         die "cannot update interface of type $newnet->{type}";
1490     }
1491
1492     my $veth = "veth${vmid}i${netid}";
1493     my $eth = $newnet->{name};
1494
1495     if (my $oldnetcfg = $conf->{$opt}) {
1496         my $oldnet = parse_lxc_network($oldnetcfg);
1497
1498         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1499             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1500
1501             PVE::Network::veth_delete($veth);
1502             delete $conf->{$opt};
1503             write_config($vmid, $conf);
1504
1505             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1506
1507         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1508                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1509                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1510
1511                 if ($oldnet->{bridge}) {
1512                     PVE::Network::tap_unplug($veth);
1513                     foreach (qw(bridge tag firewall)) {
1514                         delete $oldnet->{$_};
1515                     }
1516                     $conf->{$opt} = print_lxc_network($oldnet);
1517                     write_config($vmid, $conf);
1518                 }
1519
1520                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1521                 foreach (qw(bridge tag firewall)) {
1522                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1523                 }
1524                 $conf->{$opt} = print_lxc_network($oldnet);
1525                 write_config($vmid, $conf);
1526         }
1527     } else {
1528         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1529     }
1530
1531     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1532 }
1533
1534 sub hotplug_net {
1535     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1536
1537     my $veth = "veth${vmid}i${netid}";
1538     my $vethpeer = $veth . "p";
1539     my $eth = $newnet->{name};
1540
1541     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1542     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1543
1544     # attach peer in container
1545     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1546     PVE::Tools::run_command($cmd);
1547
1548     # link up peer in container
1549     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1550     PVE::Tools::run_command($cmd);
1551
1552     my $done = { type => 'veth' };
1553     foreach (qw(bridge tag firewall hwaddr name)) {
1554         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1555     }
1556     $conf->{$opt} = print_lxc_network($done);
1557
1558     write_config($vmid, $conf);
1559 }
1560
1561 sub update_ipconfig {
1562     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1563
1564     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1565
1566     my $optdata = parse_lxc_network($conf->{$opt});
1567     my $deleted = [];
1568     my $added = [];
1569     my $nscmd = sub {
1570         my $cmdargs = shift;
1571         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1572     };
1573     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1574
1575     my $change_ip_config = sub {
1576         my ($ipversion) = @_;
1577
1578         my $family_opt = "-$ipversion";
1579         my $suffix = $ipversion == 4 ? '' : $ipversion;
1580         my $gw= "gw$suffix";
1581         my $ip= "ip$suffix";
1582
1583         my $newip = $newnet->{$ip};
1584         my $newgw = $newnet->{$gw};
1585         my $oldip = $optdata->{$ip};
1586
1587         my $change_ip = &$safe_string_ne($oldip, $newip);
1588         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1589
1590         return if !$change_ip && !$change_gw;
1591
1592         # step 1: add new IP, if this fails we cancel
1593         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1594         if ($change_ip && $is_real_ip) {
1595             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1596             if (my $err = $@) {
1597                 warn $err;
1598                 return;
1599             }
1600         }
1601
1602         # step 2: replace gateway
1603         #   If this fails we delete the added IP and cancel.
1604         #   If it succeeds we save the config and delete the old IP, ignoring
1605         #   errors. The config is then saved.
1606         # Note: 'ip route replace' can add
1607         if ($change_gw) {
1608             if ($newgw) {
1609                 eval {
1610                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1611                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1612                     }
1613                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1614                 };
1615                 if (my $err = $@) {
1616                     warn $err;
1617                     # the route was not replaced, the old IP is still available
1618                     # rollback (delete new IP) and cancel
1619                     if ($change_ip) {
1620                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1621                         warn $@ if $@; # no need to die here
1622                     }
1623                     return;
1624                 }
1625             } else {
1626                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1627                 # if the route was not deleted, the guest might have deleted it manually
1628                 # warn and continue
1629                 warn $@ if $@;
1630             }
1631         }
1632
1633         # from this point on we save the configuration
1634         # step 3: delete old IP ignoring errors
1635         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1636             # We need to enable promote_secondaries, otherwise our newly added
1637             # address will be removed along with the old one.
1638             my $promote = 0;
1639             eval {
1640                 if ($ipversion == 4) {
1641                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1642                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1643                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1644                 }
1645                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1646             };
1647             warn $@ if $@; # no need to die here
1648
1649             if ($ipversion == 4) {
1650                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1651             }
1652         }
1653
1654         foreach my $property ($ip, $gw) {
1655             if ($newnet->{$property}) {
1656                 $optdata->{$property} = $newnet->{$property};
1657             } else {
1658                 delete $optdata->{$property};
1659             }
1660         }
1661         $conf->{$opt} = print_lxc_network($optdata);
1662         write_config($vmid, $conf);
1663         $lxc_setup->setup_network($conf);
1664     };
1665
1666     &$change_ip_config(4);
1667     &$change_ip_config(6);
1668
1669 }
1670
1671 # Internal snapshots
1672
1673 # NOTE: Snapshot create/delete involves several non-atomic
1674 # action, and can take a long time.
1675 # So we try to avoid locking the file and use 'lock' variable
1676 # inside the config file instead.
1677
1678 my $snapshot_copy_config = sub {
1679     my ($source, $dest) = @_;
1680
1681     foreach my $k (keys %$source) {
1682         next if $k eq 'snapshots';
1683         next if $k eq 'snapstate';
1684         next if $k eq 'snaptime';
1685         next if $k eq 'vmstate';
1686         next if $k eq 'lock';
1687         next if $k eq 'digest';
1688         next if $k eq 'description';
1689
1690         $dest->{$k} = $source->{$k};
1691     }
1692 };
1693
1694 my $snapshot_prepare = sub {
1695     my ($vmid, $snapname, $comment) = @_;
1696
1697     my $snap;
1698
1699     my $updatefn =  sub {
1700
1701         my $conf = load_config($vmid);
1702
1703         die "you can't take a snapshot if it's a template\n"
1704             if is_template($conf);
1705
1706         check_lock($conf);
1707
1708         $conf->{lock} = 'snapshot';
1709
1710         die "snapshot name '$snapname' already used\n"
1711             if defined($conf->{snapshots}->{$snapname});
1712
1713         my $storecfg = PVE::Storage::config();
1714         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1715
1716         $snap = $conf->{snapshots}->{$snapname} = {};
1717
1718         &$snapshot_copy_config($conf, $snap);
1719
1720         $snap->{'snapstate'} = "prepare";
1721         $snap->{'snaptime'} = time();
1722         $snap->{'description'} = $comment if $comment;
1723         $conf->{snapshots}->{$snapname} = $snap;
1724
1725         write_config($vmid, $conf);
1726     };
1727
1728     lock_container($vmid, 10, $updatefn);
1729
1730     return $snap;
1731 };
1732
1733 my $snapshot_commit = sub {
1734     my ($vmid, $snapname) = @_;
1735
1736     my $updatefn = sub {
1737
1738         my $conf = load_config($vmid);
1739
1740         die "missing snapshot lock\n"
1741             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1742
1743         die "snapshot '$snapname' does not exist\n"
1744             if !defined($conf->{snapshots}->{$snapname});
1745
1746         die "wrong snapshot state\n"
1747             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1748                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1749
1750         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1751         delete $conf->{lock};
1752         $conf->{parent} = $snapname;
1753
1754         write_config($vmid, $conf);
1755     };
1756
1757     lock_container($vmid, 10 ,$updatefn);
1758 };
1759
1760 sub has_feature {
1761     my ($feature, $conf, $storecfg, $snapname) = @_;
1762
1763     my $err;
1764
1765     foreach_mountpoint($conf, sub {
1766         my ($ms, $mountpoint) = @_;
1767
1768         return if $err; # skip further test
1769
1770         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1771
1772         # TODO: implement support for mountpoints
1773         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1774             if $ms ne 'rootfs';
1775     });
1776
1777     return $err ? 0 : 1;
1778 }
1779
1780 sub snapshot_create {
1781     my ($vmid, $snapname, $comment) = @_;
1782
1783     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1784
1785     my $conf = load_config($vmid);
1786
1787     my $running = check_running($vmid);
1788     eval {
1789         if ($running) {
1790             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1791             PVE::Tools::run_command(['/bin/sync']);
1792         };
1793
1794         my $storecfg = PVE::Storage::config();
1795         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1796         my $volid = $rootinfo->{volume};
1797
1798         if ($running) {
1799             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1800         };
1801
1802         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1803         &$snapshot_commit($vmid, $snapname);
1804     };
1805     if(my $err = $@) {
1806         snapshot_delete($vmid, $snapname, 1);
1807         die "$err\n";
1808     }
1809 }
1810
1811 sub snapshot_delete {
1812     my ($vmid, $snapname, $force) = @_;
1813
1814     my $snap;
1815
1816     my $conf;
1817
1818     my $updatefn =  sub {
1819
1820         $conf = load_config($vmid);
1821
1822         die "you can't delete a snapshot if vm is a template\n"
1823             if is_template($conf);
1824
1825         $snap = $conf->{snapshots}->{$snapname};
1826
1827         check_lock($conf);
1828
1829         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1830
1831         $snap->{snapstate} = 'delete';
1832
1833         write_config($vmid, $conf);
1834     };
1835
1836     lock_container($vmid, 10, $updatefn);
1837
1838     my $storecfg = PVE::Storage::config();
1839
1840     my $del_snap =  sub {
1841
1842         check_lock($conf);
1843
1844         if ($conf->{parent} eq $snapname) {
1845             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1846                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1847             } else {
1848                 delete $conf->{parent};
1849             }
1850         }
1851
1852         delete $conf->{snapshots}->{$snapname};
1853
1854         write_config($vmid, $conf);
1855     };
1856
1857     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1858     my $rootinfo = parse_ct_mountpoint($rootfs);
1859     my $volid = $rootinfo->{volume};
1860
1861     eval {
1862         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1863     };
1864     my $err = $@;
1865
1866     if(!$err || ($err && $force)) {
1867         lock_container($vmid, 10, $del_snap);
1868         if ($err) {
1869             die "Can't delete snapshot: $vmid $snapname $err\n";
1870         }
1871     }
1872 }
1873
1874 sub snapshot_rollback {
1875     my ($vmid, $snapname) = @_;
1876
1877     my $storecfg = PVE::Storage::config();
1878
1879     my $conf = load_config($vmid);
1880
1881     die "you can't rollback if vm is a template\n" if is_template($conf);
1882
1883     my $snap = $conf->{snapshots}->{$snapname};
1884
1885     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1886
1887     my $rootfs = $snap->{rootfs};
1888     my $rootinfo = parse_ct_mountpoint($rootfs);
1889     my $volid = $rootinfo->{volume};
1890
1891     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1892
1893     my $updatefn = sub {
1894
1895         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1896             if $snap->{snapstate};
1897
1898         check_lock($conf);
1899
1900         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1901
1902         die "unable to rollback vm $vmid: vm is running\n"
1903             if check_running($vmid);
1904
1905         $conf->{lock} = 'rollback';
1906
1907         my $forcemachine;
1908
1909         # copy snapshot config to current config
1910
1911         my $tmp_conf = $conf;
1912         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1913         $conf->{snapshots} = $tmp_conf->{snapshots};
1914         delete $conf->{snaptime};
1915         delete $conf->{snapname};
1916         $conf->{parent} = $snapname;
1917
1918         write_config($vmid, $conf);
1919     };
1920
1921     my $unlockfn = sub {
1922         delete $conf->{lock};
1923         write_config($vmid, $conf);
1924     };
1925
1926     lock_container($vmid, 10, $updatefn);
1927
1928     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1929
1930     lock_container($vmid, 5, $unlockfn);
1931 }
1932
1933 sub template_create {
1934     my ($vmid, $conf) = @_;
1935
1936     my $storecfg = PVE::Storage::config();
1937
1938     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1939     my $volid = $rootinfo->{volume};
1940
1941     die "Template feature is not available for '$volid'\n"
1942         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1943
1944     PVE::Storage::activate_volumes($storecfg, [$volid]);
1945
1946     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1947     $rootinfo->{volume} = $template_volid;
1948     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1949
1950     write_config($vmid, $conf);
1951 }
1952
1953 sub is_template {
1954     my ($conf) = @_;
1955
1956     return 1 if defined $conf->{template} && $conf->{template} == 1;
1957 }
1958
1959 sub mountpoint_names {
1960     my ($reverse) = @_;
1961
1962     my @names = ('rootfs');
1963
1964     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1965         push @names, "mp$i";
1966     }
1967
1968     return $reverse ? reverse @names : @names;
1969 }
1970
1971 # The container might have *different* symlinks than the host. realpath/abs_path
1972 # use the actual filesystem to resolve links.
1973 sub sanitize_mountpoint {
1974     my ($mp) = @_;
1975     $mp = '/' . $mp; # we always start with a slash
1976     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1977     $mp =~ s@/\./@@g; # collapse /./
1978     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1979     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1980     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1981     return $mp;
1982 }
1983
1984 sub foreach_mountpoint_full {
1985     my ($conf, $reverse, $func) = @_;
1986
1987     foreach my $key (mountpoint_names($reverse)) {
1988         my $value = $conf->{$key};
1989         next if !defined($value);
1990         my $mountpoint = parse_ct_mountpoint($value, 1);
1991         next if !defined($mountpoint);
1992
1993         # just to be sure: rootfs is /
1994         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1995         $mountpoint->{mp} = sanitize_mountpoint($path);
1996
1997         $path = $mountpoint->{volume};
1998         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1999
2000         &$func($key, $mountpoint);
2001     }
2002 }
2003
2004 sub foreach_mountpoint {
2005     my ($conf, $func) = @_;
2006
2007     foreach_mountpoint_full($conf, 0, $func);
2008 }
2009
2010 sub foreach_mountpoint_reverse {
2011     my ($conf, $func) = @_;
2012
2013     foreach_mountpoint_full($conf, 1, $func);
2014 }
2015
2016 sub check_ct_modify_config_perm {
2017     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2018
2019     return 1 if $authuser ne 'root@pam';
2020
2021     foreach my $opt (@$key_list) {
2022
2023         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2024             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2025         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2026             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2027         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2028             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2029         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2030                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2031             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2032         } else {
2033             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2034         }
2035     }
2036
2037     return 1;
2038 }
2039
2040 sub umount_all {
2041     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2042
2043     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2044     my $volid_list = get_vm_volumes($conf);
2045
2046     foreach_mountpoint_reverse($conf, sub {
2047         my ($ms, $mountpoint) = @_;
2048
2049         my $volid = $mountpoint->{volume};
2050         my $mount = $mountpoint->{mp};
2051
2052         return if !$volid || !$mount;
2053
2054         my $mount_path = "$rootdir/$mount";
2055         $mount_path =~ s!/+!/!g;
2056
2057         return if !PVE::ProcFSTools::is_mounted($mount_path);
2058
2059         eval {
2060             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2061         };
2062         if (my $err = $@) {
2063             if ($noerr) {
2064                 warn $err;
2065             } else {
2066                 die $err;
2067             }
2068         }
2069     });
2070 }
2071
2072 sub mount_all {
2073     my ($vmid, $storage_cfg, $conf) = @_;
2074
2075     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2076     File::Path::make_path($rootdir);
2077
2078     my $volid_list = get_vm_volumes($conf);
2079     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2080
2081     eval {
2082         foreach_mountpoint($conf, sub {
2083             my ($ms, $mountpoint) = @_;
2084
2085             my $volid = $mountpoint->{volume};
2086             my $mount = $mountpoint->{mp};
2087
2088             return if !$volid || !$mount;
2089
2090             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2091             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2092                 PVE::Storage::parse_volname($storage_cfg, $volid);
2093
2094             die "unable to mount base volume - internal error" if $isBase;
2095
2096             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2097         });
2098     };
2099     if (my $err = $@) {
2100         warn "mounting container failed - $err";
2101         umount_all($vmid, $storage_cfg, $conf, 1);
2102     }
2103
2104     return $rootdir;
2105 }
2106
2107
2108 sub mountpoint_mount_path {
2109     my ($mountpoint, $storage_cfg, $snapname) = @_;
2110
2111     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2112 }
2113
2114 my $check_mount_path = sub {
2115     my ($path) = @_;
2116     $path = File::Spec->canonpath($path);
2117     my $real = Cwd::realpath($path);
2118     if ($real ne $path) {
2119         die "mount path modified by symlink: $path != $real";
2120     }
2121 };
2122
2123 sub query_loopdev {
2124     my ($path) = @_;
2125     my $found;
2126     my $parser = sub {
2127         my $line = shift;
2128         if ($line =~ m@^(/dev/loop\d+):@) {
2129             $found = $1;
2130         }
2131     };
2132     my $cmd = ['losetup', '--associated', $path];
2133     PVE::Tools::run_command($cmd, outfunc => $parser);
2134     return $found;
2135 }
2136
2137 # use $rootdir = undef to just return the corresponding mount path
2138 sub mountpoint_mount {
2139     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2140
2141     my $volid = $mountpoint->{volume};
2142     my $mount = $mountpoint->{mp};
2143     my $type = $mountpoint->{type};
2144
2145     return if !$volid || !$mount;
2146
2147     my $mount_path;
2148
2149     if (defined($rootdir)) {
2150         $rootdir =~ s!/+$!!;
2151         $mount_path = "$rootdir/$mount";
2152         $mount_path =~ s!/+!/!g;
2153         &$check_mount_path($mount_path);
2154         File::Path::mkpath($mount_path);
2155     }
2156
2157     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2158
2159     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2160
2161     if ($storage) {
2162
2163         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2164         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2165
2166         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2167             PVE::Storage::parse_volname($storage_cfg, $volid);
2168
2169         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2170
2171         if ($format eq 'subvol') {
2172             if ($mount_path) {
2173                 if ($snapname) {
2174                     if ($scfg->{type} eq 'zfspool') {
2175                         my $path_arg = $path;
2176                         $path_arg =~ s!^/+!!;
2177                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2178                     } else {
2179                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2180                     }
2181                 } else {
2182                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2183                 }
2184             }
2185             return wantarray ? ($path, 0) : $path;
2186         } elsif ($format eq 'raw' || $format eq 'iso') {
2187             my $use_loopdev = 0;
2188             my @extra_opts;
2189             if ($scfg->{path}) {
2190                 push @extra_opts, '-o', 'loop';
2191                 $use_loopdev = 1;
2192             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' ||
2193                      $scfg->{type} eq 'rbd' || $scfg->{type} eq 'lvmthin') {
2194                 # do nothing
2195             } else {
2196                 die "unsupported storage type '$scfg->{type}'\n";
2197             }
2198             if ($mount_path) {
2199                 if ($format eq 'iso') {
2200                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2201                 } elsif ($isBase || defined($snapname)) {
2202                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2203                 } else {
2204                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2205                 }
2206             }
2207             return wantarray ? ($path, $use_loopdev) : $path;
2208         } else {
2209             die "unsupported image format '$format'\n";
2210         }
2211     } elsif ($type eq 'device') {
2212         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2213         return wantarray ? ($volid, 0) : $volid;
2214     } elsif ($type eq 'bind' && -d $volid) {
2215         &$check_mount_path($volid);
2216         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2217         return wantarray ? ($volid, 0) : $volid;
2218     }
2219
2220     die "unsupported storage";
2221 }
2222
2223 sub get_vm_volumes {
2224     my ($conf, $excludes) = @_;
2225
2226     my $vollist = [];
2227
2228     foreach_mountpoint($conf, sub {
2229         my ($ms, $mountpoint) = @_;
2230
2231         return if $excludes && $ms eq $excludes;
2232
2233         my $volid = $mountpoint->{volume};
2234
2235         return if !$volid || $mountpoint->{type} ne 'volume';
2236
2237         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2238         return if !$sid;
2239
2240         push @$vollist, $volid;
2241     });
2242
2243     return $vollist;
2244 }
2245
2246 sub mkfs {
2247     my ($dev, $rootuid, $rootgid) = @_;
2248
2249     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2250                              '-E', "root_owner=$rootuid:$rootgid",
2251                              $dev]);
2252 }
2253
2254 sub format_disk {
2255     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2256
2257     if ($volid =~ m!^/dev/.+!) {
2258         mkfs($volid);
2259         return;
2260     }
2261
2262     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2263
2264     die "cannot format volume '$volid' with no storage\n" if !$storage;
2265
2266     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2267
2268     my $path = PVE::Storage::path($storage_cfg, $volid);
2269
2270     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2271         PVE::Storage::parse_volname($storage_cfg, $volid);
2272
2273     die "cannot format volume '$volid' (format == $format)\n"
2274         if $format ne 'raw';
2275
2276     mkfs($path, $rootuid, $rootgid);
2277 }
2278
2279 sub destroy_disks {
2280     my ($storecfg, $vollist) = @_;
2281
2282     foreach my $volid (@$vollist) {
2283         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2284         warn $@ if $@;
2285     }
2286 }
2287
2288 sub create_disks {
2289     my ($storecfg, $vmid, $settings, $conf) = @_;
2290
2291     my $vollist = [];
2292
2293     eval {
2294         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2295         my $chown_vollist = [];
2296
2297         foreach_mountpoint($settings, sub {
2298             my ($ms, $mountpoint) = @_;
2299
2300             my $volid = $mountpoint->{volume};
2301             my $mp = $mountpoint->{mp};
2302
2303             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2304
2305             return if !$storage;
2306
2307             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2308                 my ($storeid, $size_gb) = ($1, $2);
2309
2310                 my $size_kb = int(${size_gb}*1024) * 1024;
2311
2312                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2313                 # fixme: use better naming ct-$vmid-disk-X.raw?
2314
2315                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2316                     if ($size_kb > 0) {
2317                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2318                                                            undef, $size_kb);
2319                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2320                     } else {
2321                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2322                                                            undef, 0);
2323                         push @$chown_vollist, $volid;
2324                     }
2325                 } elsif ($scfg->{type} eq 'zfspool') {
2326
2327                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2328                                                        undef, $size_kb);
2329                     push @$chown_vollist, $volid;
2330                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'lvmthin') {
2331
2332                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2333                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2334
2335                 } elsif ($scfg->{type} eq 'rbd') {
2336
2337                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2338                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2339                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2340                 } else {
2341                     die "unable to create containers on storage type '$scfg->{type}'\n";
2342                 }
2343                 push @$vollist, $volid;
2344                 $mountpoint->{volume} = $volid;
2345                 $mountpoint->{size} = $size_kb * 1024;
2346                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2347             } else {
2348                 # use specified/existing volid
2349             }
2350         });
2351
2352         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2353         foreach my $volid (@$chown_vollist) {
2354             my $path = PVE::Storage::path($storecfg, $volid, undef);
2355             chown($rootuid, $rootgid, $path);
2356         }
2357         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2358     };
2359     # free allocated images on error
2360     if (my $err = $@) {
2361         destroy_disks($storecfg, $vollist);
2362         die $err;
2363     }
2364     return $vollist;
2365 }
2366
2367 # bash completion helper
2368
2369 sub complete_os_templates {
2370     my ($cmdname, $pname, $cvalue) = @_;
2371
2372     my $cfg = PVE::Storage::config();
2373
2374     my $storeid;
2375
2376     if ($cvalue =~ m/^([^:]+):/) {
2377         $storeid = $1;
2378     }
2379
2380     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2381     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2382
2383     my $res = [];
2384     foreach my $id (keys %$data) {
2385         foreach my $item (@{$data->{$id}}) {
2386             push @$res, $item->{volid} if defined($item->{volid});
2387         }
2388     }
2389
2390     return $res;
2391 }
2392
2393 my $complete_ctid_full = sub {
2394     my ($running) = @_;
2395
2396     my $idlist = vmstatus();
2397
2398     my $active_hash = list_active_containers();
2399
2400     my $res = [];
2401
2402     foreach my $id (keys %$idlist) {
2403         my $d = $idlist->{$id};
2404         if (defined($running)) {
2405             next if $d->{template};
2406             next if $running && !$active_hash->{$id};
2407             next if !$running && $active_hash->{$id};
2408         }
2409         push @$res, $id;
2410
2411     }
2412     return $res;
2413 };
2414
2415 sub complete_ctid {
2416     return &$complete_ctid_full();
2417 }
2418
2419 sub complete_ctid_stopped {
2420     return &$complete_ctid_full(0);
2421 }
2422
2423 sub complete_ctid_running {
2424     return &$complete_ctid_full(1);
2425 }
2426
2427 sub parse_id_maps {
2428     my ($conf) = @_;
2429
2430     my $id_map = [];
2431     my $rootuid = 0;
2432     my $rootgid = 0;
2433
2434     my $lxc = $conf->{lxc};
2435     foreach my $entry (@$lxc) {
2436         my ($key, $value) = @$entry;
2437         next if $key ne 'lxc.id_map';
2438         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2439             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2440             push @$id_map, [$type, $ct, $host, $length];
2441             if ($ct == 0) {
2442                 $rootuid = $host if $type eq 'u';
2443                 $rootgid = $host if $type eq 'g';
2444             }
2445         } else {
2446             die "failed to parse id_map: $value\n";
2447         }
2448     }
2449
2450     if (!@$id_map && $conf->{unprivileged}) {
2451         # Should we read them from /etc/subuid?
2452         $id_map = [ ['u', '0', '100000', '65536'],
2453                     ['g', '0', '100000', '65536'] ];
2454         $rootuid = $rootgid = 100000;
2455     }
2456
2457     return ($id_map, $rootuid, $rootgid);
2458 }
2459
2460 sub userns_command {
2461     my ($id_map) = @_;
2462     if (@$id_map) {
2463         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2464     }
2465     return [];
2466 }
2467
2468 1;