src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format_description => 'volume',
  42         description => 'Volume, device or directory to mount into the container.',
  43     },
  44     backup => {
  45         type => 'boolean',
  46         format_description => '[1|0]',
  47         description => 'Whether to include the mountpoint in backups.',
  48         optional => 1,
  49     },
  50     size => {
  51         type => 'string',
  52         format => 'disk-size',
  53         format_description => 'DiskSize',
  54         description => 'Volume size (read only value).',
  55         optional => 1,
  56     },
  57 };
  58
  59 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  60     type => 'string', format => $rootfs_desc,
  61     description => "Use volume as container root.",
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 my $confdesc = {
  72     lock => {
  73         optional => 1,
  74         type => 'string',
  75         description => "Lock/unlock the VM.",
  76         enum => [qw(migrate backup snapshot rollback)],
  77     },
  78     onboot => {
  79         optional => 1,
  80         type => 'boolean',
  81         description => "Specifies whether a VM will be started during system bootup.",
  82         default => 0,
  83     },
  84     startup => get_standard_option('pve-startup-order'),
  85     template => {
  86         optional => 1,
  87         type => 'boolean',
  88         description => "Enable/disable Template.",
  89         default => 0,
  90     },
  91     arch => {
  92         optional => 1,
  93         type => 'string',
  94         enum => ['amd64', 'i386'],
  95         description => "OS architecture type.",
  96         default => 'amd64',
  97     },
  98     ostype => {
  99         optional => 1,
 100         type => 'string',
 101         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
 102         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 103     },
 104     console => {
 105         optional => 1,
 106         type => 'boolean',
 107         description => "Attach a console device (/dev/console) to the container.",
 108         default => 1,
 109     },
 110     tty => {
 111         optional => 1,
 112         type => 'integer',
 113         description => "Specify the number of tty available to the container",
 114         minimum => 0,
 115         maximum => 6,
 116         default => 2,
 117     },
 118     cpulimit => {
 119         optional => 1,
 120         type => 'number',
 121         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 122         minimum => 0,
 123         maximum => 128,
 124         default => 0,
 125     },
 126     cpuunits => {
 127         optional => 1,
 128         type => 'integer',
 129         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 130         minimum => 0,
 131         maximum => 500000,
 132         default => 1024,
 133     },
 134     memory => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Amount of RAM for the VM in MB.",
 138         minimum => 16,
 139         default => 512,
 140     },
 141     swap => {
 142         optional => 1,
 143         type => 'integer',
 144         description => "Amount of SWAP for the VM in MB.",
 145         minimum => 0,
 146         default => 512,
 147     },
 148     hostname => {
 149         optional => 1,
 150         description => "Set a host name for the container.",
 151         type => 'string', format => 'dns-name',
 152         maxLength => 255,
 153     },
 154     description => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Container description. Only used on the configuration web interface.",
 158     },
 159     searchdomain => {
 160         optional => 1,
 161         type => 'string', format => 'dns-name-list',
 162         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 163     },
 164     nameserver => {
 165         optional => 1,
 166         type => 'string', format => 'address-list',
 167         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 168     },
 169     rootfs => get_standard_option('pve-ct-rootfs'),
 170     parent => {
 171         optional => 1,
 172         type => 'string', format => 'pve-configid',
 173         maxLength => 40,
 174         description => "Parent snapshot name. This is used internally, and should not be modified.",
 175     },
 176     snaptime => {
 177         optional => 1,
 178         description => "Timestamp for snapshots.",
 179         type => 'integer',
 180         minimum => 0,
 181     },
 182     cmode => {
 183         optional => 1,
 184         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 185         type => 'string',
 186         enum => ['shell', 'console', 'tty'],
 187         default => 'tty',
 188     },
 189     protection => {
 190         optional => 1,
 191         type => 'boolean',
 192         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 193         default => 0,
 194     },
 195     unprivileged => {
 196         optional => 1,
 197         type => 'boolean',
 198         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 199         default => 0,
 200     },
 201 };
 202
 203 my $valid_lxc_conf_keys = {
 204     'lxc.include' => 1,
 205     'lxc.arch' => 1,
 206     'lxc.utsname' => 1,
 207     'lxc.haltsignal' => 1,
 208     'lxc.rebootsignal' => 1,
 209     'lxc.stopsignal' => 1,
 210     'lxc.init_cmd' => 1,
 211     'lxc.network.type' => 1,
 212     'lxc.network.flags' => 1,
 213     'lxc.network.link' => 1,
 214     'lxc.network.mtu' => 1,
 215     'lxc.network.name' => 1,
 216     'lxc.network.hwaddr' => 1,
 217     'lxc.network.ipv4' => 1,
 218     'lxc.network.ipv4.gateway' => 1,
 219     'lxc.network.ipv6' => 1,
 220     'lxc.network.ipv6.gateway' => 1,
 221     'lxc.network.script.up' => 1,
 222     'lxc.network.script.down' => 1,
 223     'lxc.pts' => 1,
 224     'lxc.console.logfile' => 1,
 225     'lxc.console' => 1,
 226     'lxc.tty' => 1,
 227     'lxc.devttydir' => 1,
 228     'lxc.hook.autodev' => 1,
 229     'lxc.autodev' => 1,
 230     'lxc.kmsg' => 1,
 231     'lxc.mount' => 1,
 232     'lxc.mount.entry' => 1,
 233     'lxc.mount.auto' => 1,
 234     'lxc.rootfs' => 1,
 235     'lxc.rootfs.mount' => 1,
 236     'lxc.rootfs.options' => 1,
 237     # lxc.cgroup.*
 238     'lxc.cap.drop' => 1,
 239     'lxc.cap.keep' => 1,
 240     'lxc.aa_profile' => 1,
 241     'lxc.aa_allow_incomplete' => 1,
 242     'lxc.se_context' => 1,
 243     'lxc.seccomp' => 1,
 244     'lxc.id_map' => 1,
 245     'lxc.hook.pre-start' => 1,
 246     'lxc.hook.pre-mount' => 1,
 247     'lxc.hook.mount' => 1,
 248     'lxc.hook.start' => 1,
 249     'lxc.hook.stop' => 1,
 250     'lxc.hook.post-stop' => 1,
 251     'lxc.hook.clone' => 1,
 252     'lxc.hook.destroy' => 1,
 253     'lxc.loglevel' => 1,
 254     'lxc.logfile' => 1,
 255     'lxc.start.auto' => 1,
 256     'lxc.start.delay' => 1,
 257     'lxc.start.order' => 1,
 258     'lxc.group' => 1,
 259     'lxc.environment' => 1,
 260     'lxc.' => 1,
 261     'lxc.' => 1,
 262     'lxc.' => 1,
 263     'lxc.' => 1,
 264 };
 265
 266 my $netconf_desc = {
 267     type => {
 268         type => 'string',
 269         optional => 1,
 270         description => "Network interface type.",
 271         enum => [qw(veth)],
 272     },
 273     name => {
 274         type => 'string',
 275         format_description => 'String',
 276         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 277         pattern => '[-_.\w\d]+',
 278     },
 279     bridge => {
 280         type => 'string',
 281         format_description => 'vmbr<Number>',
 282         description => 'Bridge to attach the network device to.',
 283         pattern => '[-_.\w\d]+',
 284         optional => 1,
 285     },
 286     hwaddr => {
 287         type => 'string',
 288         format_description => 'MAC',
 289         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 290         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 291         optional => 1,
 292     },
 293     mtu => {
 294         type => 'integer',
 295         format_description => 'Number',
 296         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 297         minimum => 64, # minimum ethernet frame is 64 bytes
 298         optional => 1,
 299     },
 300     ip => {
 301         type => 'string',
 302         format => 'pve-ipv4-config',
 303         format_description => 'IPv4Format/CIDR',
 304         description => 'IPv4 address in CIDR format.',
 305         optional => 1,
 306     },
 307     gw => {
 308         type => 'string',
 309         format => 'ipv4',
 310         format_description => 'GatewayIPv4',
 311         description => 'Default gateway for IPv4 traffic.',
 312         optional => 1,
 313     },
 314     ip6 => {
 315         type => 'string',
 316         format => 'pve-ipv6-config',
 317         format_description => 'IPv6Format/CIDR',
 318         description => 'IPv6 address in CIDR format.',
 319         optional => 1,
 320     },
 321     gw6 => {
 322         type => 'string',
 323         format => 'ipv6',
 324         format_description => 'GatewayIPv6',
 325         description => 'Default gateway for IPv6 traffic.',
 326         optional => 1,
 327     },
 328     firewall => {
 329         type => 'boolean',
 330         format_description => '[1|0]',
 331         description => "Controls whether this interface's firewall rules should be used.",
 332         optional => 1,
 333     },
 334     tag => {
 335         type => 'integer',
 336         format_description => 'VlanNo',
 337         minimum => '2',
 338         maximum => '4094',
 339         description => "VLAN tag foro this interface.",
 340         optional => 1,
 341     },
 342 };
 343 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 344
 345 my $MAX_LXC_NETWORKS = 10;
 346 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 347     $confdesc->{"net$i"} = {
 348         optional => 1,
 349         type => 'string', format => $netconf_desc,
 350         description => "Specifies network interfaces for the container.",
 351     };
 352 }
 353
 354 my $mp_desc = {
 355     %$rootfs_desc,
 356     mp => {
 357         type => 'string',
 358         format_description => 'Path',
 359         description => 'Path to the mountpoint as seen from inside the container.',
 360         optional => 1,
 361     },
 362 };
 363 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 364
 365 my $unuseddesc = {
 366     optional => 1,
 367     type => 'string', format => 'pve-volume-id',
 368     description => "Reference to unused volumes.",
 369 };
 370
 371 my $MAX_MOUNT_POINTS = 10;
 372 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 373     $confdesc->{"mp$i"} = {
 374         optional => 1,
 375         type => 'string', format => $mp_desc,
 376         description => "Use volume as container mount point (experimental feature).",
 377         optional => 1,
 378     };
 379 }
 380
 381 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 382 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 383     $confdesc->{"unused$i"} = $unuseddesc;
 384 }
 385
 386 sub write_pct_config {
 387     my ($filename, $conf) = @_;
 388
 389     delete $conf->{snapstate}; # just to be sure
 390
 391     my $generate_raw_config = sub {
 392         my ($conf) = @_;
 393
 394         my $raw = '';
 395
 396         # add description as comment to top of file
 397         my $descr = $conf->{description} || '';
 398         foreach my $cl (split(/\n/, $descr)) {
 399             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 400         }
 401
 402         foreach my $key (sort keys %$conf) {
 403             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 404                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 405             my $value = $conf->{$key};
 406             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 407             $raw .= "$key: $value\n";
 408         }
 409
 410         if (my $lxcconf = $conf->{lxc}) {
 411             foreach my $entry (@$lxcconf) {
 412                 my ($k, $v) = @$entry;
 413                 $raw .= "$k: $v\n";
 414             }
 415         }
 416
 417         return $raw;
 418     };
 419
 420     my $raw = &$generate_raw_config($conf);
 421
 422     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 423         $raw .= "\n[$snapname]\n";
 424         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 425     }
 426
 427     return $raw;
 428 }
 429
 430 sub check_type {
 431     my ($key, $value) = @_;
 432
 433     die "unknown setting '$key'\n" if !$confdesc->{$key};
 434
 435     my $type = $confdesc->{$key}->{type};
 436
 437     if (!defined($value)) {
 438         die "got undefined value\n";
 439     }
 440
 441     if ($value =~ m/[\n\r]/) {
 442         die "property contains a line feed\n";
 443     }
 444
 445     if ($type eq 'boolean') {
 446         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 447         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 448         die "type check ('boolean') failed - got '$value'\n";
 449     } elsif ($type eq 'integer') {
 450         return int($1) if $value =~ m/^(\d+)$/;
 451         die "type check ('integer') failed - got '$value'\n";
 452     } elsif ($type eq 'number') {
 453         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 454         die "type check ('number') failed - got '$value'\n";
 455     } elsif ($type eq 'string') {
 456         if (my $fmt = $confdesc->{$key}->{format}) {
 457             PVE::JSONSchema::check_format($fmt, $value);
 458             return $value;
 459         }
 460         return $value;
 461     } else {
 462         die "internal error"
 463     }
 464 }
 465
 466 sub parse_pct_config {
 467     my ($filename, $raw) = @_;
 468
 469     return undef if !defined($raw);
 470
 471     my $res = {
 472         digest => Digest::SHA::sha1_hex($raw),
 473         snapshots => {},
 474     };
 475
 476     $filename =~ m|/lxc/(\d+).conf$|
 477         || die "got strange filename '$filename'";
 478
 479     my $vmid = $1;
 480
 481     my $conf = $res;
 482     my $descr = '';
 483     my $section = '';
 484
 485     my @lines = split(/\n/, $raw);
 486     foreach my $line (@lines) {
 487         next if $line =~ m/^\s*$/;
 488
 489         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 490             $section = $1;
 491             $conf->{description} = $descr if $descr;
 492             $descr = '';
 493             $conf = $res->{snapshots}->{$section} = {};
 494             next;
 495         }
 496
 497         if ($line =~ m/^\#(.*)\s*$/) {
 498             $descr .= PVE::Tools::decode_text($1) . "\n";
 499             next;
 500         }
 501
 502         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 503             my $key = $1;
 504             my $value = $3;
 505             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 506                 push @{$conf->{lxc}}, [$key, $value];
 507             } else {
 508                 warn "vm $vmid - unable to parse config: $line\n";
 509             }
 510         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 511             $descr .= PVE::Tools::decode_text($2);
 512         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 513             $conf->{snapstate} = $1;
 514         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 515             my $key = $1;
 516             my $value = $2;
 517             eval { $value = check_type($key, $value); };
 518             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 519             $conf->{$key} = $value;
 520         } else {
 521             warn "vm $vmid - unable to parse config: $line\n";
 522         }
 523     }
 524
 525     $conf->{description} = $descr if $descr;
 526
 527     delete $res->{snapstate}; # just to be sure
 528
 529     return $res;
 530 }
 531
 532 sub config_list {
 533     my $vmlist = PVE::Cluster::get_vmlist();
 534     my $res = {};
 535     return $res if !$vmlist || !$vmlist->{ids};
 536     my $ids = $vmlist->{ids};
 537
 538     foreach my $vmid (keys %$ids) {
 539         next if !$vmid; # skip CT0
 540         my $d = $ids->{$vmid};
 541         next if !$d->{node} || $d->{node} ne $nodename;
 542         next if !$d->{type} || $d->{type} ne 'lxc';
 543         $res->{$vmid}->{type} = 'lxc';
 544     }
 545     return $res;
 546 }
 547
 548 sub cfs_config_path {
 549     my ($vmid, $node) = @_;
 550
 551     $node = $nodename if !$node;
 552     return "nodes/$node/lxc/$vmid.conf";
 553 }
 554
 555 sub config_file {
 556     my ($vmid, $node) = @_;
 557
 558     my $cfspath = cfs_config_path($vmid, $node);
 559     return "/etc/pve/$cfspath";
 560 }
 561
 562 sub load_config {
 563     my ($vmid, $node) = @_;
 564
 565     $node = $nodename if !$node;
 566     my $cfspath = cfs_config_path($vmid, $node);
 567
 568     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 569     die "container $vmid does not exists\n" if !defined($conf);
 570
 571     return $conf;
 572 }
 573
 574 sub create_config {
 575     my ($vmid, $conf) = @_;
 576
 577     my $dir = "/etc/pve/nodes/$nodename/lxc";
 578     mkdir $dir;
 579
 580     write_config($vmid, $conf);
 581 }
 582
 583 sub destroy_config {
 584     my ($vmid) = @_;
 585
 586     unlink config_file($vmid, $nodename);
 587 }
 588
 589 sub write_config {
 590     my ($vmid, $conf) = @_;
 591
 592     my $cfspath = cfs_config_path($vmid);
 593
 594     PVE::Cluster::cfs_write_file($cfspath, $conf);
 595 }
 596
 597 # flock: we use one file handle per process, so lock file
 598 # can be called multiple times and succeeds for the same process.
 599
 600 my $lock_handles =  {};
 601 my $lockdir = "/run/lock/lxc";
 602
 603 sub lock_filename {
 604     my ($vmid) = @_;
 605
 606     return "$lockdir/pve-config-${vmid}.lock";
 607 }
 608
 609 sub lock_aquire {
 610     my ($vmid, $timeout) = @_;
 611
 612     $timeout = 10 if !$timeout;
 613     my $mode = LOCK_EX;
 614
 615     my $filename = lock_filename($vmid);
 616
 617     mkdir $lockdir if !-d $lockdir;
 618
 619     my $lock_func = sub {
 620         if (!$lock_handles->{$$}->{$filename}) {
 621             my $fh = new IO::File(">>$filename") ||
 622                 die "can't open file - $!\n";
 623             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 624         }
 625
 626         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 627             print STDERR "trying to aquire lock...";
 628             my $success;
 629             while(1) {
 630                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 631                 # try again on EINTR (see bug #273)
 632                 if ($success || ($! != EINTR)) {
 633                     last;
 634                 }
 635             }
 636             if (!$success) {
 637                 print STDERR " failed\n";
 638                 die "can't aquire lock - $!\n";
 639             }
 640
 641             print STDERR " OK\n";
 642         }
 643
 644         $lock_handles->{$$}->{$filename}->{refcount}++;
 645     };
 646
 647     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 648     my $err = $@;
 649     if ($err) {
 650         die "can't lock file '$filename' - $err";
 651     }
 652 }
 653
 654 sub lock_release {
 655     my ($vmid) = @_;
 656
 657     my $filename = lock_filename($vmid);
 658
 659     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 660         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 661         if ($refcount <= 0) {
 662             $lock_handles->{$$}->{$filename} = undef;
 663             close ($fh);
 664         }
 665     }
 666 }
 667
 668 sub lock_container {
 669     my ($vmid, $timeout, $code, @param) = @_;
 670
 671     my $res;
 672
 673     lock_aquire($vmid, $timeout);
 674     eval { $res = &$code(@param) };
 675     my $err = $@;
 676     lock_release($vmid);
 677
 678     die $err if $err;
 679
 680     return $res;
 681 }
 682
 683 sub option_exists {
 684     my ($name) = @_;
 685
 686     return defined($confdesc->{$name});
 687 }
 688
 689 # add JSON properties for create and set function
 690 sub json_config_properties {
 691     my $prop = shift;
 692
 693     foreach my $opt (keys %$confdesc) {
 694         next if $opt eq 'parent' || $opt eq 'snaptime';
 695         next if $prop->{$opt};
 696         $prop->{$opt} = $confdesc->{$opt};
 697     }
 698
 699     return $prop;
 700 }
 701
 702 sub json_config_properties_no_rootfs {
 703     my $prop = shift;
 704
 705     foreach my $opt (keys %$confdesc) {
 706         next if $prop->{$opt};
 707         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 708         $prop->{$opt} = $confdesc->{$opt};
 709     }
 710
 711     return $prop;
 712 }
 713
 714 # container status helpers
 715
 716 sub list_active_containers {
 717
 718     my $filename = "/proc/net/unix";
 719
 720     # similar test is used by lcxcontainers.c: list_active_containers
 721     my $res = {};
 722
 723     my $fh = IO::File->new ($filename, "r");
 724     return $res if !$fh;
 725
 726     while (defined(my $line = <$fh>)) {
 727         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 728             my $path = $1;
 729             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 730                 $res->{$1} = 1;
 731             }
 732         }
 733     }
 734
 735     close($fh);
 736
 737     return $res;
 738 }
 739
 740 # warning: this is slow
 741 sub check_running {
 742     my ($vmid) = @_;
 743
 744     my $active_hash = list_active_containers();
 745
 746     return 1 if defined($active_hash->{$vmid});
 747
 748     return undef;
 749 }
 750
 751 sub get_container_disk_usage {
 752     my ($vmid, $pid) = @_;
 753
 754     return PVE::Tools::df("/proc/$pid/root/", 1);
 755 }
 756
 757 my $last_proc_vmid_stat;
 758
 759 my $parse_cpuacct_stat = sub {
 760     my ($vmid) = @_;
 761
 762     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 763
 764     my $stat = {};
 765
 766     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 767
 768         $stat->{utime} = $1;
 769         $stat->{stime} = $2;
 770
 771     }
 772
 773     return $stat;
 774 };
 775
 776 sub vmstatus {
 777     my ($opt_vmid) = @_;
 778
 779     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 780
 781     my $active_hash = list_active_containers();
 782
 783     my $cpucount = $cpuinfo->{cpus} || 1;
 784
 785     my $cdtime = gettimeofday;
 786
 787     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 788
 789     foreach my $vmid (keys %$list) {
 790         my $d = $list->{$vmid};
 791
 792         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 793         warn $@ if $@; # ignore errors (consider them stopped)
 794
 795         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 796
 797         my $cfspath = cfs_config_path($vmid);
 798         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 799
 800         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 801         $d->{name} =~ s/[\s]//g;
 802
 803         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 804
 805         if ($d->{pid}) {
 806             my $res = get_container_disk_usage($vmid, $d->{pid});
 807             $d->{disk} = $res->{used};
 808             $d->{maxdisk} = $res->{total};
 809         } else {
 810             $d->{disk} = 0;
 811             # use 4GB by default ??
 812             if (my $rootfs = $conf->{rootfs}) {
 813                 my $rootinfo = parse_ct_mountpoint($rootfs);
 814                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 815             } else {
 816                 $d->{maxdisk} = 4*1024*1024*1024;
 817             }
 818         }
 819
 820         $d->{mem} = 0;
 821         $d->{swap} = 0;
 822         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 823         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 824
 825         $d->{uptime} = 0;
 826         $d->{cpu} = 0;
 827
 828         $d->{netout} = 0;
 829         $d->{netin} = 0;
 830
 831         $d->{diskread} = 0;
 832         $d->{diskwrite} = 0;
 833
 834         $d->{template} = is_template($conf);
 835     }
 836
 837     foreach my $vmid (keys %$list) {
 838         my $d = $list->{$vmid};
 839         my $pid = $d->{pid};
 840
 841         next if !$pid; # skip stopped CTs
 842
 843         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 844         $d->{uptime} = time - $ctime; # the method lxcfs uses
 845
 846         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 847         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 848
 849         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 850         my @bytes = split(/\n/, $blkio_bytes);
 851         foreach my $byte (@bytes) {
 852             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 853                 $d->{diskread} = $2 if $key eq 'Read';
 854                 $d->{diskwrite} = $2 if $key eq 'Write';
 855             }
 856         }
 857
 858         my $pstat = &$parse_cpuacct_stat($vmid);
 859
 860         my $used = $pstat->{utime} + $pstat->{stime};
 861
 862         my $old = $last_proc_vmid_stat->{$vmid};
 863         if (!$old) {
 864             $last_proc_vmid_stat->{$vmid} = {
 865                 time => $cdtime,
 866                 used => $used,
 867                 cpu => 0,
 868             };
 869             next;
 870         }
 871
 872         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 873
 874         if ($dtime > 1000) {
 875             my $dutime = $used -  $old->{used};
 876
 877             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 878             $last_proc_vmid_stat->{$vmid} = {
 879                 time => $cdtime,
 880                 used => $used,
 881                 cpu => $d->{cpu},
 882             };
 883         } else {
 884             $d->{cpu} = $old->{cpu};
 885         }
 886     }
 887
 888     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 889
 890     foreach my $dev (keys %$netdev) {
 891         next if $dev !~ m/^veth([1-9]\d*)i/;
 892         my $vmid = $1;
 893         my $d = $list->{$vmid};
 894
 895         next if !$d;
 896
 897         $d->{netout} += $netdev->{$dev}->{receive};
 898         $d->{netin} += $netdev->{$dev}->{transmit};
 899
 900     }
 901
 902     return $list;
 903 }
 904
 905 sub classify_mountpoint {
 906     my ($vol) = @_;
 907     if ($vol =~ m!^/!) {
 908         return 'device' if $vol =~ m!^/dev/!;
 909         return 'bind';
 910     }
 911     return 'volume';
 912 }
 913
 914 sub parse_ct_mountpoint {
 915     my ($data, $noerr) = @_;
 916
 917     $data //= '';
 918
 919     my $res;
 920     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 921     if ($@) {
 922         return undef if $noerr;
 923         die $@;
 924     }
 925
 926     if (defined(my $size = $res->{size})) {
 927         $size = PVE::JSONSchema::parse_size($size);
 928         if (!defined($size)) {
 929             return undef if $noerr;
 930             die "invalid size: $size\n";
 931         }
 932         $res->{size} = $size;
 933     }
 934
 935     $res->{type} = classify_mountpoint($res->{volume});
 936
 937     return $res;
 938 }
 939
 940 sub print_ct_mountpoint {
 941     my ($info, $nomp) = @_;
 942     my $skip = [ 'type' ];
 943     push @$skip, 'mp' if $nomp;
 944     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 945 }
 946
 947 sub print_lxc_network {
 948     my $net = shift;
 949     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 950 }
 951
 952 sub parse_lxc_network {
 953     my ($data) = @_;
 954
 955     my $res = {};
 956
 957     return $res if !$data;
 958
 959     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 960
 961     $res->{type} = 'veth';
 962     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 963
 964     return $res;
 965 }
 966
 967 sub read_cgroup_value {
 968     my ($group, $vmid, $name, $full) = @_;
 969
 970     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 971
 972     return PVE::Tools::file_get_contents($path) if $full;
 973
 974     return PVE::Tools::file_read_firstline($path);
 975 }
 976
 977 sub write_cgroup_value {
 978    my ($group, $vmid, $name, $value) = @_;
 979
 980    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 981    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 982
 983 }
 984
 985 sub find_lxc_console_pids {
 986
 987     my $res = {};
 988
 989     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 990         my ($pid) = @_;
 991
 992         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 993         return if !$cmdline;
 994
 995         my @args = split(/\0/, $cmdline);
 996
 997         # serach for lxc-console -n <vmid>
 998         return if scalar(@args) != 3;
 999         return if $args[1] ne '-n';
1000         return if $args[2] !~ m/^\d+$/;
1001         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
1002
1003         my $vmid = $args[2];
1004
1005         push @{$res->{$vmid}}, $pid;
1006     });
1007
1008     return $res;
1009 }
1010
1011 sub find_lxc_pid {
1012     my ($vmid) = @_;
1013
1014     my $pid = undef;
1015     my $parser = sub {
1016         my $line = shift;
1017         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1018     };
1019     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1020
1021     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1022
1023     return $pid;
1024 }
1025
1026 # Note: we cannot use Net:IP, because that only allows strict
1027 # CIDR networks
1028 sub parse_ipv4_cidr {
1029     my ($cidr, $noerr) = @_;
1030
1031     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1032         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1033     }
1034
1035     return undef if $noerr;
1036
1037     die "unable to parse ipv4 address/mask\n";
1038 }
1039
1040 sub check_lock {
1041     my ($conf) = @_;
1042
1043     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1044 }
1045
1046 sub check_protection {
1047     my ($vm_conf, $err_msg) = @_;
1048
1049     if ($vm_conf->{protection}) {
1050         die "$err_msg - protection mode enabled\n";
1051     }
1052 }
1053
1054 sub update_lxc_config {
1055     my ($storage_cfg, $vmid, $conf) = @_;
1056
1057     my $dir = "/var/lib/lxc/$vmid";
1058
1059     if ($conf->{template}) {
1060
1061         unlink "$dir/config";
1062
1063         return;
1064     }
1065
1066     my $raw = '';
1067
1068     die "missing 'arch' - internal error" if !$conf->{arch};
1069     $raw .= "lxc.arch = $conf->{arch}\n";
1070
1071     my $unprivileged = $conf->{unprivileged};
1072     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1073
1074     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1075     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1076         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1077         if ($unprivileged || $custom_idmap) {
1078             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1079         }
1080     } else {
1081         die "implement me";
1082     }
1083
1084     $raw .= "lxc.start.unshare = 1\n";
1085
1086     # Should we read them from /etc/subuid?
1087     if ($unprivileged && !$custom_idmap) {
1088         $raw .= "lxc.id_map = u 0 100000 65536\n";
1089         $raw .= "lxc.id_map = g 0 100000 65536\n";
1090     }
1091
1092     if (!has_dev_console($conf)) {
1093         $raw .= "lxc.console = none\n";
1094         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1095     }
1096
1097     my $ttycount = get_tty_count($conf);
1098     $raw .= "lxc.tty = $ttycount\n";
1099
1100     # some init scripts expects a linux terminal (turnkey).
1101     $raw .= "lxc.environment = TERM=linux\n";
1102
1103     my $utsname = $conf->{hostname} || "CT$vmid";
1104     $raw .= "lxc.utsname = $utsname\n";
1105
1106     my $memory = $conf->{memory} || 512;
1107     my $swap = $conf->{swap} // 0;
1108
1109     my $lxcmem = int($memory*1024*1024);
1110     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1111     $raw .= "lxc.cgroup.memory.kmem.limit_in_bytes = $lxcmem\n";
1112
1113     my $lxcswap = int(($memory + $swap)*1024*1024);
1114     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1115
1116     if (my $cpulimit = $conf->{cpulimit}) {
1117         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1118         my $value = int(100000*$cpulimit);
1119         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1120     }
1121
1122     my $shares = $conf->{cpuunits} || 1024;
1123     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1124
1125     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1126     $mountpoint->{mp} = '/';
1127
1128     $raw .= "lxc.rootfs = $dir/rootfs\n";
1129     $raw .= "lxc.hook.stop = /usr/lib/x86_64-linux-gnu/lxc/hooks/unmount-namespace\n";
1130
1131     my $netcount = 0;
1132     foreach my $k (keys %$conf) {
1133         next if $k !~ m/^net(\d+)$/;
1134         my $ind = $1;
1135         my $d = parse_lxc_network($conf->{$k});
1136         $netcount++;
1137         $raw .= "lxc.network.type = veth\n";
1138         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1139         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1140         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1141         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1142     }
1143
1144     if (my $lxcconf = $conf->{lxc}) {
1145         foreach my $entry (@$lxcconf) {
1146             my ($k, $v) = @$entry;
1147             $netcount++ if $k eq 'lxc.network.type';
1148             $raw .= "$k = $v\n";
1149         }
1150     }
1151
1152     $raw .= "lxc.network.type = empty\n" if !$netcount;
1153
1154     File::Path::mkpath("$dir/rootfs");
1155
1156     PVE::Tools::file_set_contents("$dir/config", $raw);
1157 }
1158
1159 # verify and cleanup nameserver list (replace \0 with ' ')
1160 sub verify_nameserver_list {
1161     my ($nameserver_list) = @_;
1162
1163     my @list = ();
1164     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1165         PVE::JSONSchema::pve_verify_ip($server);
1166         push @list, $server;
1167     }
1168
1169     return join(' ', @list);
1170 }
1171
1172 sub verify_searchdomain_list {
1173     my ($searchdomain_list) = @_;
1174
1175     my @list = ();
1176     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1177         # todo: should we add checks for valid dns domains?
1178         push @list, $server;
1179     }
1180
1181     return join(' ', @list);
1182 }
1183
1184 sub add_unused_volume {
1185     my ($config, $volid) = @_;
1186
1187     my $key;
1188     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1189         my $test = "unused$ind";
1190         if (my $vid = $config->{$test}) {
1191             return if $vid eq $volid; # do not add duplicates
1192         } else {
1193             $key = $test;
1194         }
1195     }
1196
1197     die "To many unused volume - please delete them first.\n" if !$key;
1198
1199     $config->{$key} = $volid;
1200
1201     return $key;
1202 }
1203
1204 sub update_pct_config {
1205     my ($vmid, $conf, $running, $param, $delete) = @_;
1206
1207     my @nohotplug;
1208
1209     my $new_disks = 0;
1210     my @deleted_volumes;
1211
1212     my $rootdir;
1213     if ($running) {
1214         my $pid = find_lxc_pid($vmid);
1215         $rootdir = "/proc/$pid/root";
1216     }
1217
1218     my $hotplug_error = sub {
1219         if ($running) {
1220             push @nohotplug, @_;
1221             return 1;
1222         } else {
1223             return 0;
1224         }
1225     };
1226
1227     if (defined($delete)) {
1228         foreach my $opt (@$delete) {
1229             if (!exists($conf->{$opt})) {
1230                 warn "no such option: $opt\n";
1231                 next;
1232             }
1233
1234             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1235                 die "unable to delete required option '$opt'\n";
1236             } elsif ($opt eq 'swap') {
1237                 delete $conf->{$opt};
1238                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1239             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1240                 delete $conf->{$opt};
1241             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1242                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1243                 next if $hotplug_error->($opt);
1244                 delete $conf->{$opt};
1245             } elsif ($opt =~ m/^net(\d)$/) {
1246                 delete $conf->{$opt};
1247                 next if !$running;
1248                 my $netid = $1;
1249                 PVE::Network::veth_delete("veth${vmid}i$netid");
1250             } elsif ($opt eq 'protection') {
1251                 delete $conf->{$opt};
1252             } elsif ($opt =~ m/^unused(\d+)$/) {
1253                 next if $hotplug_error->($opt);
1254                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1255                 push @deleted_volumes, $conf->{$opt};
1256                 delete $conf->{$opt};
1257             } elsif ($opt =~ m/^mp(\d+)$/) {
1258                 next if $hotplug_error->($opt);
1259                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1260                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1261                 if ($mountpoint->{type} eq 'volume') {
1262                     add_unused_volume($conf, $mountpoint->{volume})
1263                 }
1264                 delete $conf->{$opt};
1265             } elsif ($opt eq 'unprivileged') {
1266                 die "unable to delete read-only option: '$opt'\n";
1267             } else {
1268                 die "implement me"
1269             }
1270             write_config($vmid, $conf) if $running;
1271         }
1272     }
1273
1274     # There's no separate swap size to configure, there's memory and "total"
1275     # memory (iow. memory+swap). This means we have to change them together.
1276     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1277     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1278     if (defined($wanted_memory) || defined($wanted_swap)) {
1279
1280         $wanted_memory //= ($conf->{memory} || 512);
1281         $wanted_swap //=  ($conf->{swap} || 0);
1282
1283         my $total = $wanted_memory + $wanted_swap;
1284         if ($running) {
1285             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1286             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1287         }
1288         $conf->{memory} = $wanted_memory;
1289         $conf->{swap} = $wanted_swap;
1290
1291         write_config($vmid, $conf) if $running;
1292     }
1293
1294     foreach my $opt (keys %$param) {
1295         my $value = $param->{$opt};
1296         if ($opt eq 'hostname') {
1297             $conf->{$opt} = $value;
1298         } elsif ($opt eq 'onboot') {
1299             $conf->{$opt} = $value ? 1 : 0;
1300         } elsif ($opt eq 'startup') {
1301             $conf->{$opt} = $value;
1302         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1303             next if $hotplug_error->($opt);
1304             $conf->{$opt} = $value;
1305         } elsif ($opt eq 'nameserver') {
1306             next if $hotplug_error->($opt);
1307             my $list = verify_nameserver_list($value);
1308             $conf->{$opt} = $list;
1309         } elsif ($opt eq 'searchdomain') {
1310             next if $hotplug_error->($opt);
1311             my $list = verify_searchdomain_list($value);
1312             $conf->{$opt} = $list;
1313         } elsif ($opt eq 'cpulimit') {
1314             next if $hotplug_error->($opt); # FIXME: hotplug
1315             $conf->{$opt} = $value;
1316         } elsif ($opt eq 'cpuunits') {
1317             $conf->{$opt} = $value;
1318             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1319         } elsif ($opt eq 'description') {
1320             $conf->{$opt} = PVE::Tools::encode_text($value);
1321         } elsif ($opt =~ m/^net(\d+)$/) {
1322             my $netid = $1;
1323             my $net = parse_lxc_network($value);
1324             if (!$running) {
1325                 $conf->{$opt} = print_lxc_network($net);
1326             } else {
1327                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1328             }
1329         } elsif ($opt eq 'protection') {
1330             $conf->{$opt} = $value ? 1 : 0;
1331         } elsif ($opt =~ m/^mp(\d+)$/) {
1332             next if $hotplug_error->($opt);
1333             check_protection($conf, "can't update CT $vmid drive '$opt'");
1334             $conf->{$opt} = $value;
1335             $new_disks = 1;
1336         } elsif ($opt eq 'rootfs') {
1337             check_protection($conf, "can't update CT $vmid drive '$opt'");
1338             die "implement me: $opt";
1339         } elsif ($opt eq 'unprivileged') {
1340             die "unable to modify read-only option: '$opt'\n";
1341         } else {
1342             die "implement me: $opt";
1343         }
1344         write_config($vmid, $conf) if $running;
1345     }
1346
1347     if (@deleted_volumes) {
1348         my $storage_cfg = PVE::Storage::config();
1349         foreach my $volume (@deleted_volumes) {
1350             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1351         }
1352     }
1353
1354     if ($new_disks) {
1355         my $storage_cfg = PVE::Storage::config();
1356         create_disks($storage_cfg, $vmid, $conf, $conf);
1357     }
1358
1359     # This should be the last thing we do here
1360     if ($running && scalar(@nohotplug)) {
1361         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1362     }
1363 }
1364
1365 sub has_dev_console {
1366     my ($conf) = @_;
1367
1368     return !(defined($conf->{console}) && !$conf->{console});
1369 }
1370
1371 sub get_tty_count {
1372     my ($conf) = @_;
1373
1374     return $conf->{tty} // $confdesc->{tty}->{default};
1375 }
1376
1377 sub get_cmode {
1378     my ($conf) = @_;
1379
1380     return $conf->{cmode} // $confdesc->{cmode}->{default};
1381 }
1382
1383 sub get_console_command {
1384     my ($vmid, $conf) = @_;
1385
1386     my $cmode = get_cmode($conf);
1387
1388     if ($cmode eq 'console') {
1389         return ['lxc-console', '-n',  $vmid, '-t', 0];
1390     } elsif ($cmode eq 'tty') {
1391         return ['lxc-console', '-n',  $vmid];
1392     } elsif ($cmode eq 'shell') {
1393         return ['lxc-attach', '--clear-env', '-n', $vmid];
1394     } else {
1395         die "internal error";
1396     }
1397 }
1398
1399 sub get_primary_ips {
1400     my ($conf) = @_;
1401
1402     # return data from net0
1403
1404     return undef if !defined($conf->{net0});
1405     my $net = parse_lxc_network($conf->{net0});
1406
1407     my $ipv4 = $net->{ip};
1408     if ($ipv4) {
1409         if ($ipv4 =~ /^(dhcp|manual)$/) {
1410             $ipv4 = undef
1411         } else {
1412             $ipv4 =~ s!/\d+$!!;
1413         }
1414     }
1415     my $ipv6 = $net->{ip6};
1416     if ($ipv6) {
1417         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1418             $ipv6 = undef;
1419         } else {
1420             $ipv6 =~ s!/\d+$!!;
1421         }
1422     }
1423
1424     return ($ipv4, $ipv6);
1425 }
1426
1427 sub delete_mountpoint_volume {
1428     my ($storage_cfg, $vmid, $volume) = @_;
1429
1430     return if classify_mountpoint($volume) ne 'volume';
1431
1432     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1433     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1434 }
1435
1436 sub destroy_lxc_container {
1437     my ($storage_cfg, $vmid, $conf) = @_;
1438
1439     foreach_mountpoint($conf, sub {
1440         my ($ms, $mountpoint) = @_;
1441         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1442     });
1443
1444     rmdir "/var/lib/lxc/$vmid/rootfs";
1445     unlink "/var/lib/lxc/$vmid/config";
1446     rmdir "/var/lib/lxc/$vmid";
1447     destroy_config($vmid);
1448
1449     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1450     #PVE::Tools::run_command($cmd);
1451 }
1452
1453 sub vm_stop_cleanup {
1454     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1455
1456     eval {
1457         if (!$keepActive) {
1458
1459             my $vollist = get_vm_volumes($conf);
1460             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1461         }
1462     };
1463     warn $@ if $@; # avoid errors - just warn
1464 }
1465
1466 my $safe_num_ne = sub {
1467     my ($a, $b) = @_;
1468
1469     return 0 if !defined($a) && !defined($b);
1470     return 1 if !defined($a);
1471     return 1 if !defined($b);
1472
1473     return $a != $b;
1474 };
1475
1476 my $safe_string_ne = sub {
1477     my ($a, $b) = @_;
1478
1479     return 0 if !defined($a) && !defined($b);
1480     return 1 if !defined($a);
1481     return 1 if !defined($b);
1482
1483     return $a ne $b;
1484 };
1485
1486 sub update_net {
1487     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1488
1489     if ($newnet->{type} ne 'veth') {
1490         # for when there are physical interfaces
1491         die "cannot update interface of type $newnet->{type}";
1492     }
1493
1494     my $veth = "veth${vmid}i${netid}";
1495     my $eth = $newnet->{name};
1496
1497     if (my $oldnetcfg = $conf->{$opt}) {
1498         my $oldnet = parse_lxc_network($oldnetcfg);
1499
1500         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1501             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1502
1503             PVE::Network::veth_delete($veth);
1504             delete $conf->{$opt};
1505             write_config($vmid, $conf);
1506
1507             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1508
1509         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1510                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1511                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1512
1513                 if ($oldnet->{bridge}) {
1514                     PVE::Network::tap_unplug($veth);
1515                     foreach (qw(bridge tag firewall)) {
1516                         delete $oldnet->{$_};
1517                     }
1518                     $conf->{$opt} = print_lxc_network($oldnet);
1519                     write_config($vmid, $conf);
1520                 }
1521
1522                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1523                 foreach (qw(bridge tag firewall)) {
1524                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1525                 }
1526                 $conf->{$opt} = print_lxc_network($oldnet);
1527                 write_config($vmid, $conf);
1528         }
1529     } else {
1530         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1531     }
1532
1533     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1534 }
1535
1536 sub hotplug_net {
1537     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1538
1539     my $veth = "veth${vmid}i${netid}";
1540     my $vethpeer = $veth . "p";
1541     my $eth = $newnet->{name};
1542
1543     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1544     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1545
1546     # attach peer in container
1547     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1548     PVE::Tools::run_command($cmd);
1549
1550     # link up peer in container
1551     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1552     PVE::Tools::run_command($cmd);
1553
1554     my $done = { type => 'veth' };
1555     foreach (qw(bridge tag firewall hwaddr name)) {
1556         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1557     }
1558     $conf->{$opt} = print_lxc_network($done);
1559
1560     write_config($vmid, $conf);
1561 }
1562
1563 sub update_ipconfig {
1564     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1565
1566     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1567
1568     my $optdata = parse_lxc_network($conf->{$opt});
1569     my $deleted = [];
1570     my $added = [];
1571     my $nscmd = sub {
1572         my $cmdargs = shift;
1573         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1574     };
1575     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1576
1577     my $change_ip_config = sub {
1578         my ($ipversion) = @_;
1579
1580         my $family_opt = "-$ipversion";
1581         my $suffix = $ipversion == 4 ? '' : $ipversion;
1582         my $gw= "gw$suffix";
1583         my $ip= "ip$suffix";
1584
1585         my $newip = $newnet->{$ip};
1586         my $newgw = $newnet->{$gw};
1587         my $oldip = $optdata->{$ip};
1588
1589         my $change_ip = &$safe_string_ne($oldip, $newip);
1590         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1591
1592         return if !$change_ip && !$change_gw;
1593
1594         # step 1: add new IP, if this fails we cancel
1595         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1596         if ($change_ip && $is_real_ip) {
1597             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1598             if (my $err = $@) {
1599                 warn $err;
1600                 return;
1601             }
1602         }
1603
1604         # step 2: replace gateway
1605         #   If this fails we delete the added IP and cancel.
1606         #   If it succeeds we save the config and delete the old IP, ignoring
1607         #   errors. The config is then saved.
1608         # Note: 'ip route replace' can add
1609         if ($change_gw) {
1610             if ($newgw) {
1611                 eval {
1612                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1613                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1614                     }
1615                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1616                 };
1617                 if (my $err = $@) {
1618                     warn $err;
1619                     # the route was not replaced, the old IP is still available
1620                     # rollback (delete new IP) and cancel
1621                     if ($change_ip) {
1622                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1623                         warn $@ if $@; # no need to die here
1624                     }
1625                     return;
1626                 }
1627             } else {
1628                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1629                 # if the route was not deleted, the guest might have deleted it manually
1630                 # warn and continue
1631                 warn $@ if $@;
1632             }
1633         }
1634
1635         # from this point on we save the configuration
1636         # step 3: delete old IP ignoring errors
1637         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1638             # We need to enable promote_secondaries, otherwise our newly added
1639             # address will be removed along with the old one.
1640             my $promote = 0;
1641             eval {
1642                 if ($ipversion == 4) {
1643                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1644                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1645                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1646                 }
1647                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1648             };
1649             warn $@ if $@; # no need to die here
1650
1651             if ($ipversion == 4) {
1652                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1653             }
1654         }
1655
1656         foreach my $property ($ip, $gw) {
1657             if ($newnet->{$property}) {
1658                 $optdata->{$property} = $newnet->{$property};
1659             } else {
1660                 delete $optdata->{$property};
1661             }
1662         }
1663         $conf->{$opt} = print_lxc_network($optdata);
1664         write_config($vmid, $conf);
1665         $lxc_setup->setup_network($conf);
1666     };
1667
1668     &$change_ip_config(4);
1669     &$change_ip_config(6);
1670
1671 }
1672
1673 # Internal snapshots
1674
1675 # NOTE: Snapshot create/delete involves several non-atomic
1676 # action, and can take a long time.
1677 # So we try to avoid locking the file and use 'lock' variable
1678 # inside the config file instead.
1679
1680 my $snapshot_copy_config = sub {
1681     my ($source, $dest) = @_;
1682
1683     foreach my $k (keys %$source) {
1684         next if $k eq 'snapshots';
1685         next if $k eq 'snapstate';
1686         next if $k eq 'snaptime';
1687         next if $k eq 'vmstate';
1688         next if $k eq 'lock';
1689         next if $k eq 'digest';
1690         next if $k eq 'description';
1691
1692         $dest->{$k} = $source->{$k};
1693     }
1694 };
1695
1696 my $snapshot_prepare = sub {
1697     my ($vmid, $snapname, $comment) = @_;
1698
1699     my $snap;
1700
1701     my $updatefn =  sub {
1702
1703         my $conf = load_config($vmid);
1704
1705         die "you can't take a snapshot if it's a template\n"
1706             if is_template($conf);
1707
1708         check_lock($conf);
1709
1710         $conf->{lock} = 'snapshot';
1711
1712         die "snapshot name '$snapname' already used\n"
1713             if defined($conf->{snapshots}->{$snapname});
1714
1715         my $storecfg = PVE::Storage::config();
1716         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1717
1718         $snap = $conf->{snapshots}->{$snapname} = {};
1719
1720         &$snapshot_copy_config($conf, $snap);
1721
1722         $snap->{'snapstate'} = "prepare";
1723         $snap->{'snaptime'} = time();
1724         $snap->{'description'} = $comment if $comment;
1725         $conf->{snapshots}->{$snapname} = $snap;
1726
1727         write_config($vmid, $conf);
1728     };
1729
1730     lock_container($vmid, 10, $updatefn);
1731
1732     return $snap;
1733 };
1734
1735 my $snapshot_commit = sub {
1736     my ($vmid, $snapname) = @_;
1737
1738     my $updatefn = sub {
1739
1740         my $conf = load_config($vmid);
1741
1742         die "missing snapshot lock\n"
1743             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1744
1745         die "snapshot '$snapname' does not exist\n"
1746             if !defined($conf->{snapshots}->{$snapname});
1747
1748         die "wrong snapshot state\n"
1749             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1750                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1751
1752         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1753         delete $conf->{lock};
1754         $conf->{parent} = $snapname;
1755
1756         write_config($vmid, $conf);
1757     };
1758
1759     lock_container($vmid, 10 ,$updatefn);
1760 };
1761
1762 sub has_feature {
1763     my ($feature, $conf, $storecfg, $snapname) = @_;
1764
1765     my $err;
1766
1767     foreach_mountpoint($conf, sub {
1768         my ($ms, $mountpoint) = @_;
1769
1770         return if $err; # skip further test
1771
1772         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1773
1774         # TODO: implement support for mountpoints
1775         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1776             if $ms ne 'rootfs';
1777     });
1778
1779     return $err ? 0 : 1;
1780 }
1781
1782 sub snapshot_create {
1783     my ($vmid, $snapname, $comment) = @_;
1784
1785     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1786
1787     my $conf = load_config($vmid);
1788
1789     my $running = check_running($vmid);
1790     eval {
1791         if ($running) {
1792             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1793             PVE::Tools::run_command(['/bin/sync']);
1794         };
1795
1796         my $storecfg = PVE::Storage::config();
1797         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1798         my $volid = $rootinfo->{volume};
1799
1800         if ($running) {
1801             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1802         };
1803
1804         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1805         &$snapshot_commit($vmid, $snapname);
1806     };
1807     if(my $err = $@) {
1808         snapshot_delete($vmid, $snapname, 1);
1809         die "$err\n";
1810     }
1811 }
1812
1813 sub snapshot_delete {
1814     my ($vmid, $snapname, $force) = @_;
1815
1816     my $snap;
1817
1818     my $conf;
1819
1820     my $updatefn =  sub {
1821
1822         $conf = load_config($vmid);
1823
1824         die "you can't delete a snapshot if vm is a template\n"
1825             if is_template($conf);
1826
1827         $snap = $conf->{snapshots}->{$snapname};
1828
1829         check_lock($conf);
1830
1831         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1832
1833         $snap->{snapstate} = 'delete';
1834
1835         write_config($vmid, $conf);
1836     };
1837
1838     lock_container($vmid, 10, $updatefn);
1839
1840     my $storecfg = PVE::Storage::config();
1841
1842     my $del_snap =  sub {
1843
1844         check_lock($conf);
1845
1846         if ($conf->{parent} eq $snapname) {
1847             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1848                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1849             } else {
1850                 delete $conf->{parent};
1851             }
1852         }
1853
1854         delete $conf->{snapshots}->{$snapname};
1855
1856         write_config($vmid, $conf);
1857     };
1858
1859     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1860     my $rootinfo = parse_ct_mountpoint($rootfs);
1861     my $volid = $rootinfo->{volume};
1862
1863     eval {
1864         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1865     };
1866     my $err = $@;
1867
1868     if(!$err || ($err && $force)) {
1869         lock_container($vmid, 10, $del_snap);
1870         if ($err) {
1871             die "Can't delete snapshot: $vmid $snapname $err\n";
1872         }
1873     }
1874 }
1875
1876 sub snapshot_rollback {
1877     my ($vmid, $snapname) = @_;
1878
1879     my $storecfg = PVE::Storage::config();
1880
1881     my $conf = load_config($vmid);
1882
1883     die "you can't rollback if vm is a template\n" if is_template($conf);
1884
1885     my $snap = $conf->{snapshots}->{$snapname};
1886
1887     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1888
1889     my $rootfs = $snap->{rootfs};
1890     my $rootinfo = parse_ct_mountpoint($rootfs);
1891     my $volid = $rootinfo->{volume};
1892
1893     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1894
1895     my $updatefn = sub {
1896
1897         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1898             if $snap->{snapstate};
1899
1900         check_lock($conf);
1901
1902         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1903
1904         die "unable to rollback vm $vmid: vm is running\n"
1905             if check_running($vmid);
1906
1907         $conf->{lock} = 'rollback';
1908
1909         my $forcemachine;
1910
1911         # copy snapshot config to current config
1912
1913         my $tmp_conf = $conf;
1914         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1915         $conf->{snapshots} = $tmp_conf->{snapshots};
1916         delete $conf->{snaptime};
1917         delete $conf->{snapname};
1918         $conf->{parent} = $snapname;
1919
1920         write_config($vmid, $conf);
1921     };
1922
1923     my $unlockfn = sub {
1924         delete $conf->{lock};
1925         write_config($vmid, $conf);
1926     };
1927
1928     lock_container($vmid, 10, $updatefn);
1929
1930     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1931
1932     lock_container($vmid, 5, $unlockfn);
1933 }
1934
1935 sub template_create {
1936     my ($vmid, $conf) = @_;
1937
1938     my $storecfg = PVE::Storage::config();
1939
1940     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1941     my $volid = $rootinfo->{volume};
1942
1943     die "Template feature is not available for '$volid'\n"
1944         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1945
1946     PVE::Storage::activate_volumes($storecfg, [$volid]);
1947
1948     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1949     $rootinfo->{volume} = $template_volid;
1950     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1951
1952     write_config($vmid, $conf);
1953 }
1954
1955 sub is_template {
1956     my ($conf) = @_;
1957
1958     return 1 if defined $conf->{template} && $conf->{template} == 1;
1959 }
1960
1961 sub mountpoint_names {
1962     my ($reverse) = @_;
1963
1964     my @names = ('rootfs');
1965
1966     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1967         push @names, "mp$i";
1968     }
1969
1970     return $reverse ? reverse @names : @names;
1971 }
1972
1973 # The container might have *different* symlinks than the host. realpath/abs_path
1974 # use the actual filesystem to resolve links.
1975 sub sanitize_mountpoint {
1976     my ($mp) = @_;
1977     $mp = '/' . $mp; # we always start with a slash
1978     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1979     $mp =~ s@/\./@@g; # collapse /./
1980     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1981     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1982     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1983     return $mp;
1984 }
1985
1986 sub foreach_mountpoint_full {
1987     my ($conf, $reverse, $func) = @_;
1988
1989     foreach my $key (mountpoint_names($reverse)) {
1990         my $value = $conf->{$key};
1991         next if !defined($value);
1992         my $mountpoint = parse_ct_mountpoint($value, 1);
1993         next if !defined($mountpoint);
1994
1995         # just to be sure: rootfs is /
1996         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1997         $mountpoint->{mp} = sanitize_mountpoint($path);
1998
1999         $path = $mountpoint->{volume};
2000         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
2001
2002         &$func($key, $mountpoint);
2003     }
2004 }
2005
2006 sub foreach_mountpoint {
2007     my ($conf, $func) = @_;
2008
2009     foreach_mountpoint_full($conf, 0, $func);
2010 }
2011
2012 sub foreach_mountpoint_reverse {
2013     my ($conf, $func) = @_;
2014
2015     foreach_mountpoint_full($conf, 1, $func);
2016 }
2017
2018 sub check_ct_modify_config_perm {
2019     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2020
2021     return 1 if $authuser ne 'root@pam';
2022
2023     foreach my $opt (@$key_list) {
2024
2025         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2026             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2027         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2028             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2029         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2030             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2031         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2032                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2033             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2034         } else {
2035             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2036         }
2037     }
2038
2039     return 1;
2040 }
2041
2042 sub umount_all {
2043     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2044
2045     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2046     my $volid_list = get_vm_volumes($conf);
2047
2048     foreach_mountpoint_reverse($conf, sub {
2049         my ($ms, $mountpoint) = @_;
2050
2051         my $volid = $mountpoint->{volume};
2052         my $mount = $mountpoint->{mp};
2053
2054         return if !$volid || !$mount;
2055
2056         my $mount_path = "$rootdir/$mount";
2057         $mount_path =~ s!/+!/!g;
2058
2059         return if !PVE::ProcFSTools::is_mounted($mount_path);
2060
2061         eval {
2062             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2063         };
2064         if (my $err = $@) {
2065             if ($noerr) {
2066                 warn $err;
2067             } else {
2068                 die $err;
2069             }
2070         }
2071     });
2072 }
2073
2074 sub mount_all {
2075     my ($vmid, $storage_cfg, $conf) = @_;
2076
2077     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2078     File::Path::make_path($rootdir);
2079
2080     my $volid_list = get_vm_volumes($conf);
2081     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2082
2083     eval {
2084         foreach_mountpoint($conf, sub {
2085             my ($ms, $mountpoint) = @_;
2086
2087             my $volid = $mountpoint->{volume};
2088             my $mount = $mountpoint->{mp};
2089
2090             return if !$volid || !$mount;
2091
2092             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2093             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2094                 PVE::Storage::parse_volname($storage_cfg, $volid);
2095
2096             die "unable to mount base volume - internal error" if $isBase;
2097
2098             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2099         });
2100     };
2101     if (my $err = $@) {
2102         warn "mounting container failed - $err";
2103         umount_all($vmid, $storage_cfg, $conf, 1);
2104     }
2105
2106     return $rootdir;
2107 }
2108
2109
2110 sub mountpoint_mount_path {
2111     my ($mountpoint, $storage_cfg, $snapname) = @_;
2112
2113     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2114 }
2115
2116 my $check_mount_path = sub {
2117     my ($path) = @_;
2118     $path = File::Spec->canonpath($path);
2119     my $real = Cwd::realpath($path);
2120     if ($real ne $path) {
2121         die "mount path modified by symlink: $path != $real";
2122     }
2123 };
2124
2125 sub query_loopdev {
2126     my ($path) = @_;
2127     my $found;
2128     my $parser = sub {
2129         my $line = shift;
2130         if ($line =~ m@^(/dev/loop\d+):@) {
2131             $found = $1;
2132         }
2133     };
2134     my $cmd = ['losetup', '--associated', $path];
2135     PVE::Tools::run_command($cmd, outfunc => $parser);
2136     return $found;
2137 }
2138
2139 # use $rootdir = undef to just return the corresponding mount path
2140 sub mountpoint_mount {
2141     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2142
2143     my $volid = $mountpoint->{volume};
2144     my $mount = $mountpoint->{mp};
2145     my $type = $mountpoint->{type};
2146
2147     return if !$volid || !$mount;
2148
2149     my $mount_path;
2150
2151     if (defined($rootdir)) {
2152         $rootdir =~ s!/+$!!;
2153         $mount_path = "$rootdir/$mount";
2154         $mount_path =~ s!/+!/!g;
2155         &$check_mount_path($mount_path);
2156         File::Path::mkpath($mount_path);
2157     }
2158
2159     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2160
2161     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2162
2163     if ($storage) {
2164
2165         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2166         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2167
2168         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2169             PVE::Storage::parse_volname($storage_cfg, $volid);
2170
2171         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2172
2173         if ($format eq 'subvol') {
2174             if ($mount_path) {
2175                 if ($snapname) {
2176                     if ($scfg->{type} ne 'zfspool') {
2177                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2178                     }
2179                 }
2180                 PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2181             }
2182             return wantarray ? ($path, 0) : $path;
2183         } elsif ($format eq 'raw' || $format eq 'iso') {
2184             my $use_loopdev = 0;
2185             my @extra_opts;
2186             if ($scfg->{path}) {
2187                 push @extra_opts, '-o', 'loop';
2188                 $use_loopdev = 1;
2189             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2190                 # do nothing
2191             } else {
2192                 die "unsupported storage type '$scfg->{type}'\n";
2193             }
2194             if ($mount_path) {
2195                 if ($format eq 'iso') {
2196                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2197                 } elsif ($isBase || defined($snapname)) {
2198                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2199                 } else {
2200                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2201                 }
2202             }
2203             return wantarray ? ($path, $use_loopdev) : $path;
2204         } else {
2205             die "unsupported image format '$format'\n";
2206         }
2207     } elsif ($type eq 'device') {
2208         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2209         return wantarray ? ($volid, 0) : $volid;
2210     } elsif ($type eq 'bind' && -d $volid) {
2211         &$check_mount_path($volid);
2212         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2213         return wantarray ? ($volid, 0) : $volid;
2214     }
2215
2216     die "unsupported storage";
2217 }
2218
2219 sub get_vm_volumes {
2220     my ($conf, $excludes) = @_;
2221
2222     my $vollist = [];
2223
2224     foreach_mountpoint($conf, sub {
2225         my ($ms, $mountpoint) = @_;
2226
2227         return if $excludes && $ms eq $excludes;
2228
2229         my $volid = $mountpoint->{volume};
2230
2231         return if !$volid || $mountpoint->{type} ne 'volume';
2232
2233         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2234         return if !$sid;
2235
2236         push @$vollist, $volid;
2237     });
2238
2239     return $vollist;
2240 }
2241
2242 sub mkfs {
2243     my ($dev, $rootuid, $rootgid) = @_;
2244
2245     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2246                              '-E', "root_owner=$rootuid:$rootgid",
2247                              $dev]);
2248 }
2249
2250 sub format_disk {
2251     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2252
2253     if ($volid =~ m!^/dev/.+!) {
2254         mkfs($volid);
2255         return;
2256     }
2257
2258     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2259
2260     die "cannot format volume '$volid' with no storage\n" if !$storage;
2261
2262     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2263
2264     my $path = PVE::Storage::path($storage_cfg, $volid);
2265
2266     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2267         PVE::Storage::parse_volname($storage_cfg, $volid);
2268
2269     die "cannot format volume '$volid' (format == $format)\n"
2270         if $format ne 'raw';
2271
2272     mkfs($path, $rootuid, $rootgid);
2273 }
2274
2275 sub destroy_disks {
2276     my ($storecfg, $vollist) = @_;
2277
2278     foreach my $volid (@$vollist) {
2279         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2280         warn $@ if $@;
2281     }
2282 }
2283
2284 sub create_disks {
2285     my ($storecfg, $vmid, $settings, $conf) = @_;
2286
2287     my $vollist = [];
2288
2289     eval {
2290         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2291         my $chown_vollist = [];
2292
2293         foreach_mountpoint($settings, sub {
2294             my ($ms, $mountpoint) = @_;
2295
2296             my $volid = $mountpoint->{volume};
2297             my $mp = $mountpoint->{mp};
2298
2299             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2300
2301             return if !$storage;
2302
2303             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2304                 my ($storeid, $size_gb) = ($1, $2);
2305
2306                 my $size_kb = int(${size_gb}*1024) * 1024;
2307
2308                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2309                 # fixme: use better naming ct-$vmid-disk-X.raw?
2310
2311                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2312                     if ($size_kb > 0) {
2313                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2314                                                            undef, $size_kb);
2315                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2316                     } else {
2317                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2318                                                            undef, 0);
2319                         push @$chown_vollist, $volid;
2320                     }
2321                 } elsif ($scfg->{type} eq 'zfspool') {
2322
2323                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2324                                                        undef, $size_kb);
2325                     push @$chown_vollist, $volid;
2326                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2327
2328                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2329                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2330
2331                 } elsif ($scfg->{type} eq 'rbd') {
2332
2333                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2334                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2335                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2336                 } else {
2337                     die "unable to create containers on storage type '$scfg->{type}'\n";
2338                 }
2339                 push @$vollist, $volid;
2340                 $mountpoint->{volume} = $volid;
2341                 $mountpoint->{size} = $size_kb * 1024;
2342                 $conf->{$ms} = print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
2343             } else {
2344                 # use specified/existing volid
2345             }
2346         });
2347
2348         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2349         foreach my $volid (@$chown_vollist) {
2350             my $path = PVE::Storage::path($storecfg, $volid, undef);
2351             chown($rootuid, $rootgid, $path);
2352         }
2353         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2354     };
2355     # free allocated images on error
2356     if (my $err = $@) {
2357         destroy_disks($storecfg, $vollist);
2358         die $err;
2359     }
2360     return $vollist;
2361 }
2362
2363 # bash completion helper
2364
2365 sub complete_os_templates {
2366     my ($cmdname, $pname, $cvalue) = @_;
2367
2368     my $cfg = PVE::Storage::config();
2369
2370     my $storeid;
2371
2372     if ($cvalue =~ m/^([^:]+):/) {
2373         $storeid = $1;
2374     }
2375
2376     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2377     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2378
2379     my $res = [];
2380     foreach my $id (keys %$data) {
2381         foreach my $item (@{$data->{$id}}) {
2382             push @$res, $item->{volid} if defined($item->{volid});
2383         }
2384     }
2385
2386     return $res;
2387 }
2388
2389 my $complete_ctid_full = sub {
2390     my ($running) = @_;
2391
2392     my $idlist = vmstatus();
2393
2394     my $active_hash = list_active_containers();
2395
2396     my $res = [];
2397
2398     foreach my $id (keys %$idlist) {
2399         my $d = $idlist->{$id};
2400         if (defined($running)) {
2401             next if $d->{template};
2402             next if $running && !$active_hash->{$id};
2403             next if !$running && $active_hash->{$id};
2404         }
2405         push @$res, $id;
2406
2407     }
2408     return $res;
2409 };
2410
2411 sub complete_ctid {
2412     return &$complete_ctid_full();
2413 }
2414
2415 sub complete_ctid_stopped {
2416     return &$complete_ctid_full(0);
2417 }
2418
2419 sub complete_ctid_running {
2420     return &$complete_ctid_full(1);
2421 }
2422
2423 sub parse_id_maps {
2424     my ($conf) = @_;
2425
2426     my $id_map = [];
2427     my $rootuid = 0;
2428     my $rootgid = 0;
2429
2430     my $lxc = $conf->{lxc};
2431     foreach my $entry (@$lxc) {
2432         my ($key, $value) = @$entry;
2433         next if $key ne 'lxc.id_map';
2434         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2435             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2436             push @$id_map, [$type, $ct, $host, $length];
2437             if ($ct == 0) {
2438                 $rootuid = $host if $type eq 'u';
2439                 $rootgid = $host if $type eq 'g';
2440             }
2441         } else {
2442             die "failed to parse id_map: $value\n";
2443         }
2444     }
2445
2446     if (!@$id_map && $conf->{unprivileged}) {
2447         # Should we read them from /etc/subuid?
2448         $id_map = [ ['u', '0', '100000', '65536'],
2449                     ['g', '0', '100000', '65536'] ];
2450         $rootuid = $rootgid = 100000;
2451     }
2452
2453     return ($id_map, $rootuid, $rootgid);
2454 }
2455
2456 sub userns_command {
2457     my ($id_map) = @_;
2458     if (@$id_map) {
2459         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2460     }
2461     return [];
2462 }
2463
2464 1;