src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format_description => 'volume',
  42         description => 'Volume, device or directory to mount into the container.',
  43     },
  44     backup => {
  45         type => 'boolean',
  46         format_description => '[1|0]',
  47         description => 'Whether to include the mountpoint in backups.',
  48         optional => 1,
  49     },
  50     size => {
  51         type => 'string',
  52         format => 'disk-size',
  53         format_description => 'DiskSize',
  54         description => 'Volume size (read only value).',
  55         optional => 1,
  56     },
  57 };
  58
  59 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  60     type => 'string', format => $rootfs_desc,
  61     description => "Use volume as container root.",
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 my $confdesc = {
  72     lock => {
  73         optional => 1,
  74         type => 'string',
  75         description => "Lock/unlock the VM.",
  76         enum => [qw(migrate backup snapshot rollback)],
  77     },
  78     onboot => {
  79         optional => 1,
  80         type => 'boolean',
  81         description => "Specifies whether a VM will be started during system bootup.",
  82         default => 0,
  83     },
  84     startup => get_standard_option('pve-startup-order'),
  85     template => {
  86         optional => 1,
  87         type => 'boolean',
  88         description => "Enable/disable Template.",
  89         default => 0,
  90     },
  91     arch => {
  92         optional => 1,
  93         type => 'string',
  94         enum => ['amd64', 'i386'],
  95         description => "OS architecture type.",
  96         default => 'amd64',
  97     },
  98     ostype => {
  99         optional => 1,
 100         type => 'string',
 101         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
 102         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 103     },
 104     console => {
 105         optional => 1,
 106         type => 'boolean',
 107         description => "Attach a console device (/dev/console) to the container.",
 108         default => 1,
 109     },
 110     tty => {
 111         optional => 1,
 112         type => 'integer',
 113         description => "Specify the number of tty available to the container",
 114         minimum => 0,
 115         maximum => 6,
 116         default => 2,
 117     },
 118     cpulimit => {
 119         optional => 1,
 120         type => 'number',
 121         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 122         minimum => 0,
 123         maximum => 128,
 124         default => 0,
 125     },
 126     cpuunits => {
 127         optional => 1,
 128         type => 'integer',
 129         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 130         minimum => 0,
 131         maximum => 500000,
 132         default => 1024,
 133     },
 134     memory => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Amount of RAM for the VM in MB.",
 138         minimum => 16,
 139         default => 512,
 140     },
 141     swap => {
 142         optional => 1,
 143         type => 'integer',
 144         description => "Amount of SWAP for the VM in MB.",
 145         minimum => 0,
 146         default => 512,
 147     },
 148     hostname => {
 149         optional => 1,
 150         description => "Set a host name for the container.",
 151         type => 'string', format => 'dns-name',
 152         maxLength => 255,
 153     },
 154     description => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Container description. Only used on the configuration web interface.",
 158     },
 159     searchdomain => {
 160         optional => 1,
 161         type => 'string', format => 'dns-name-list',
 162         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 163     },
 164     nameserver => {
 165         optional => 1,
 166         type => 'string', format => 'address-list',
 167         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 168     },
 169     rootfs => get_standard_option('pve-ct-rootfs'),
 170     parent => {
 171         optional => 1,
 172         type => 'string', format => 'pve-configid',
 173         maxLength => 40,
 174         description => "Parent snapshot name. This is used internally, and should not be modified.",
 175     },
 176     snaptime => {
 177         optional => 1,
 178         description => "Timestamp for snapshots.",
 179         type => 'integer',
 180         minimum => 0,
 181     },
 182     cmode => {
 183         optional => 1,
 184         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 185         type => 'string',
 186         enum => ['shell', 'console', 'tty'],
 187         default => 'tty',
 188     },
 189     protection => {
 190         optional => 1,
 191         type => 'boolean',
 192         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 193         default => 0,
 194     },
 195     unprivileged => {
 196         optional => 1,
 197         type => 'boolean',
 198         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 199         default => 0,
 200     },
 201 };
 202
 203 my $valid_lxc_conf_keys = {
 204     'lxc.include' => 1,
 205     'lxc.arch' => 1,
 206     'lxc.utsname' => 1,
 207     'lxc.haltsignal' => 1,
 208     'lxc.rebootsignal' => 1,
 209     'lxc.stopsignal' => 1,
 210     'lxc.init_cmd' => 1,
 211     'lxc.network.type' => 1,
 212     'lxc.network.flags' => 1,
 213     'lxc.network.link' => 1,
 214     'lxc.network.mtu' => 1,
 215     'lxc.network.name' => 1,
 216     'lxc.network.hwaddr' => 1,
 217     'lxc.network.ipv4' => 1,
 218     'lxc.network.ipv4.gateway' => 1,
 219     'lxc.network.ipv6' => 1,
 220     'lxc.network.ipv6.gateway' => 1,
 221     'lxc.network.script.up' => 1,
 222     'lxc.network.script.down' => 1,
 223     'lxc.pts' => 1,
 224     'lxc.console.logfile' => 1,
 225     'lxc.console' => 1,
 226     'lxc.tty' => 1,
 227     'lxc.devttydir' => 1,
 228     'lxc.hook.autodev' => 1,
 229     'lxc.autodev' => 1,
 230     'lxc.kmsg' => 1,
 231     'lxc.mount' => 1,
 232     'lxc.mount.entry' => 1,
 233     'lxc.mount.auto' => 1,
 234     'lxc.rootfs' => 1,
 235     'lxc.rootfs.mount' => 1,
 236     'lxc.rootfs.options' => 1,
 237     # lxc.cgroup.*
 238     'lxc.cap.drop' => 1,
 239     'lxc.cap.keep' => 1,
 240     'lxc.aa_profile' => 1,
 241     'lxc.aa_allow_incomplete' => 1,
 242     'lxc.se_context' => 1,
 243     'lxc.seccomp' => 1,
 244     'lxc.id_map' => 1,
 245     'lxc.hook.pre-start' => 1,
 246     'lxc.hook.pre-mount' => 1,
 247     'lxc.hook.mount' => 1,
 248     'lxc.hook.start' => 1,
 249     'lxc.hook.stop' => 1,
 250     'lxc.hook.post-stop' => 1,
 251     'lxc.hook.clone' => 1,
 252     'lxc.hook.destroy' => 1,
 253     'lxc.loglevel' => 1,
 254     'lxc.logfile' => 1,
 255     'lxc.start.auto' => 1,
 256     'lxc.start.delay' => 1,
 257     'lxc.start.order' => 1,
 258     'lxc.group' => 1,
 259     'lxc.environment' => 1,
 260     'lxc.' => 1,
 261     'lxc.' => 1,
 262     'lxc.' => 1,
 263     'lxc.' => 1,
 264 };
 265
 266 my $netconf_desc = {
 267     type => {
 268         type => 'string',
 269         optional => 1,
 270         description => "Network interface type.",
 271         enum => [qw(veth)],
 272     },
 273     name => {
 274         type => 'string',
 275         format_description => 'String',
 276         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 277         pattern => '[-_.\w\d]+',
 278     },
 279     bridge => {
 280         type => 'string',
 281         format_description => 'vmbr<Number>',
 282         description => 'Bridge to attach the network device to.',
 283         pattern => '[-_.\w\d]+',
 284         optional => 1,
 285     },
 286     hwaddr => {
 287         type => 'string',
 288         format_description => 'MAC',
 289         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 290         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 291         optional => 1,
 292     },
 293     mtu => {
 294         type => 'integer',
 295         format_description => 'Number',
 296         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 297         minimum => 64, # minimum ethernet frame is 64 bytes
 298         optional => 1,
 299     },
 300     ip => {
 301         type => 'string',
 302         format => 'pve-ipv4-config',
 303         format_description => 'IPv4Format/CIDR',
 304         description => 'IPv4 address in CIDR format.',
 305         optional => 1,
 306     },
 307     gw => {
 308         type => 'string',
 309         format => 'ipv4',
 310         format_description => 'GatewayIPv4',
 311         description => 'Default gateway for IPv4 traffic.',
 312         optional => 1,
 313     },
 314     ip6 => {
 315         type => 'string',
 316         format => 'pve-ipv6-config',
 317         format_description => 'IPv6Format/CIDR',
 318         description => 'IPv6 address in CIDR format.',
 319         optional => 1,
 320     },
 321     gw6 => {
 322         type => 'string',
 323         format => 'ipv6',
 324         format_description => 'GatewayIPv6',
 325         description => 'Default gateway for IPv6 traffic.',
 326         optional => 1,
 327     },
 328     firewall => {
 329         type => 'boolean',
 330         format_description => '[1|0]',
 331         description => "Controls whether this interface's firewall rules should be used.",
 332         optional => 1,
 333     },
 334     tag => {
 335         type => 'integer',
 336         format_description => 'VlanNo',
 337         minimum => '2',
 338         maximum => '4094',
 339         description => "VLAN tag foro this interface.",
 340         optional => 1,
 341     },
 342 };
 343 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 344
 345 my $MAX_LXC_NETWORKS = 10;
 346 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 347     $confdesc->{"net$i"} = {
 348         optional => 1,
 349         type => 'string', format => $netconf_desc,
 350         description => "Specifies network interfaces for the container.",
 351     };
 352 }
 353
 354 my $mp_desc = {
 355     %$rootfs_desc,
 356     mp => {
 357         type => 'string',
 358         format_description => 'Path',
 359         description => 'Path to the mountpoint as seen from inside the container.',
 360         optional => 1,
 361     },
 362 };
 363 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 364
 365 my $unuseddesc = {
 366     optional => 1,
 367     type => 'string', format => 'pve-volume-id',
 368     description => "Reference to unused volumes.",
 369 };
 370
 371 my $MAX_MOUNT_POINTS = 10;
 372 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 373     $confdesc->{"mp$i"} = {
 374         optional => 1,
 375         type => 'string', format => $mp_desc,
 376         description => "Use volume as container mount point (experimental feature).",
 377         optional => 1,
 378     };
 379 }
 380
 381 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 382 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 383     $confdesc->{"unused$i"} = $unuseddesc;
 384 }
 385
 386 sub write_pct_config {
 387     my ($filename, $conf) = @_;
 388
 389     delete $conf->{snapstate}; # just to be sure
 390
 391     my $generate_raw_config = sub {
 392         my ($conf) = @_;
 393
 394         my $raw = '';
 395
 396         # add description as comment to top of file
 397         my $descr = $conf->{description} || '';
 398         foreach my $cl (split(/\n/, $descr)) {
 399             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 400         }
 401
 402         foreach my $key (sort keys %$conf) {
 403             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 404                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 405             my $value = $conf->{$key};
 406             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 407             $raw .= "$key: $value\n";
 408         }
 409
 410         if (my $lxcconf = $conf->{lxc}) {
 411             foreach my $entry (@$lxcconf) {
 412                 my ($k, $v) = @$entry;
 413                 $raw .= "$k: $v\n";
 414             }
 415         }
 416
 417         return $raw;
 418     };
 419
 420     my $raw = &$generate_raw_config($conf);
 421
 422     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 423         $raw .= "\n[$snapname]\n";
 424         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 425     }
 426
 427     return $raw;
 428 }
 429
 430 sub check_type {
 431     my ($key, $value) = @_;
 432
 433     die "unknown setting '$key'\n" if !$confdesc->{$key};
 434
 435     my $type = $confdesc->{$key}->{type};
 436
 437     if (!defined($value)) {
 438         die "got undefined value\n";
 439     }
 440
 441     if ($value =~ m/[\n\r]/) {
 442         die "property contains a line feed\n";
 443     }
 444
 445     if ($type eq 'boolean') {
 446         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 447         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 448         die "type check ('boolean') failed - got '$value'\n";
 449     } elsif ($type eq 'integer') {
 450         return int($1) if $value =~ m/^(\d+)$/;
 451         die "type check ('integer') failed - got '$value'\n";
 452     } elsif ($type eq 'number') {
 453         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 454         die "type check ('number') failed - got '$value'\n";
 455     } elsif ($type eq 'string') {
 456         if (my $fmt = $confdesc->{$key}->{format}) {
 457             PVE::JSONSchema::check_format($fmt, $value);
 458             return $value;
 459         }
 460         return $value;
 461     } else {
 462         die "internal error"
 463     }
 464 }
 465
 466 sub parse_pct_config {
 467     my ($filename, $raw) = @_;
 468
 469     return undef if !defined($raw);
 470
 471     my $res = {
 472         digest => Digest::SHA::sha1_hex($raw),
 473         snapshots => {},
 474     };
 475
 476     $filename =~ m|/lxc/(\d+).conf$|
 477         || die "got strange filename '$filename'";
 478
 479     my $vmid = $1;
 480
 481     my $conf = $res;
 482     my $descr = '';
 483     my $section = '';
 484
 485     my @lines = split(/\n/, $raw);
 486     foreach my $line (@lines) {
 487         next if $line =~ m/^\s*$/;
 488
 489         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 490             $section = $1;
 491             $conf->{description} = $descr if $descr;
 492             $descr = '';
 493             $conf = $res->{snapshots}->{$section} = {};
 494             next;
 495         }
 496
 497         if ($line =~ m/^\#(.*)\s*$/) {
 498             $descr .= PVE::Tools::decode_text($1) . "\n";
 499             next;
 500         }
 501
 502         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 503             my $key = $1;
 504             my $value = $3;
 505             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 506                 push @{$conf->{lxc}}, [$key, $value];
 507             } else {
 508                 warn "vm $vmid - unable to parse config: $line\n";
 509             }
 510         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 511             $descr .= PVE::Tools::decode_text($2);
 512         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 513             $conf->{snapstate} = $1;
 514         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 515             my $key = $1;
 516             my $value = $2;
 517             eval { $value = check_type($key, $value); };
 518             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 519             $conf->{$key} = $value;
 520         } else {
 521             warn "vm $vmid - unable to parse config: $line\n";
 522         }
 523     }
 524
 525     $conf->{description} = $descr if $descr;
 526
 527     delete $res->{snapstate}; # just to be sure
 528
 529     return $res;
 530 }
 531
 532 sub config_list {
 533     my $vmlist = PVE::Cluster::get_vmlist();
 534     my $res = {};
 535     return $res if !$vmlist || !$vmlist->{ids};
 536     my $ids = $vmlist->{ids};
 537
 538     foreach my $vmid (keys %$ids) {
 539         next if !$vmid; # skip CT0
 540         my $d = $ids->{$vmid};
 541         next if !$d->{node} || $d->{node} ne $nodename;
 542         next if !$d->{type} || $d->{type} ne 'lxc';
 543         $res->{$vmid}->{type} = 'lxc';
 544     }
 545     return $res;
 546 }
 547
 548 sub cfs_config_path {
 549     my ($vmid, $node) = @_;
 550
 551     $node = $nodename if !$node;
 552     return "nodes/$node/lxc/$vmid.conf";
 553 }
 554
 555 sub config_file {
 556     my ($vmid, $node) = @_;
 557
 558     my $cfspath = cfs_config_path($vmid, $node);
 559     return "/etc/pve/$cfspath";
 560 }
 561
 562 sub load_config {
 563     my ($vmid, $node) = @_;
 564
 565     $node = $nodename if !$node;
 566     my $cfspath = cfs_config_path($vmid, $node);
 567
 568     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 569     die "container $vmid does not exists\n" if !defined($conf);
 570
 571     return $conf;
 572 }
 573
 574 sub create_config {
 575     my ($vmid, $conf) = @_;
 576
 577     my $dir = "/etc/pve/nodes/$nodename/lxc";
 578     mkdir $dir;
 579
 580     write_config($vmid, $conf);
 581 }
 582
 583 sub destroy_config {
 584     my ($vmid) = @_;
 585
 586     unlink config_file($vmid, $nodename);
 587 }
 588
 589 sub write_config {
 590     my ($vmid, $conf) = @_;
 591
 592     my $cfspath = cfs_config_path($vmid);
 593
 594     PVE::Cluster::cfs_write_file($cfspath, $conf);
 595 }
 596
 597 # flock: we use one file handle per process, so lock file
 598 # can be called multiple times and succeeds for the same process.
 599
 600 my $lock_handles =  {};
 601 my $lockdir = "/run/lock/lxc";
 602
 603 sub lock_filename {
 604     my ($vmid) = @_;
 605
 606     return "$lockdir/pve-config-${vmid}.lock";
 607 }
 608
 609 sub lock_aquire {
 610     my ($vmid, $timeout) = @_;
 611
 612     $timeout = 10 if !$timeout;
 613     my $mode = LOCK_EX;
 614
 615     my $filename = lock_filename($vmid);
 616
 617     mkdir $lockdir if !-d $lockdir;
 618
 619     my $lock_func = sub {
 620         if (!$lock_handles->{$$}->{$filename}) {
 621             my $fh = new IO::File(">>$filename") ||
 622                 die "can't open file - $!\n";
 623             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 624         }
 625
 626         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 627             print STDERR "trying to aquire lock...";
 628             my $success;
 629             while(1) {
 630                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 631                 # try again on EINTR (see bug #273)
 632                 if ($success || ($! != EINTR)) {
 633                     last;
 634                 }
 635             }
 636             if (!$success) {
 637                 print STDERR " failed\n";
 638                 die "can't aquire lock - $!\n";
 639             }
 640
 641             print STDERR " OK\n";
 642         }
 643
 644         $lock_handles->{$$}->{$filename}->{refcount}++;
 645     };
 646
 647     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 648     my $err = $@;
 649     if ($err) {
 650         die "can't lock file '$filename' - $err";
 651     }
 652 }
 653
 654 sub lock_release {
 655     my ($vmid) = @_;
 656
 657     my $filename = lock_filename($vmid);
 658
 659     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 660         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 661         if ($refcount <= 0) {
 662             $lock_handles->{$$}->{$filename} = undef;
 663             close ($fh);
 664         }
 665     }
 666 }
 667
 668 sub lock_container {
 669     my ($vmid, $timeout, $code, @param) = @_;
 670
 671     my $res;
 672
 673     lock_aquire($vmid, $timeout);
 674     eval { $res = &$code(@param) };
 675     my $err = $@;
 676     lock_release($vmid);
 677
 678     die $err if $err;
 679
 680     return $res;
 681 }
 682
 683 sub option_exists {
 684     my ($name) = @_;
 685
 686     return defined($confdesc->{$name});
 687 }
 688
 689 # add JSON properties for create and set function
 690 sub json_config_properties {
 691     my $prop = shift;
 692
 693     foreach my $opt (keys %$confdesc) {
 694         next if $opt eq 'parent' || $opt eq 'snaptime';
 695         next if $prop->{$opt};
 696         $prop->{$opt} = $confdesc->{$opt};
 697     }
 698
 699     return $prop;
 700 }
 701
 702 sub json_config_properties_no_rootfs {
 703     my $prop = shift;
 704
 705     foreach my $opt (keys %$confdesc) {
 706         next if $prop->{$opt};
 707         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 708         $prop->{$opt} = $confdesc->{$opt};
 709     }
 710
 711     return $prop;
 712 }
 713
 714 # container status helpers
 715
 716 sub list_active_containers {
 717
 718     my $filename = "/proc/net/unix";
 719
 720     # similar test is used by lcxcontainers.c: list_active_containers
 721     my $res = {};
 722
 723     my $fh = IO::File->new ($filename, "r");
 724     return $res if !$fh;
 725
 726     while (defined(my $line = <$fh>)) {
 727         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 728             my $path = $1;
 729             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 730                 $res->{$1} = 1;
 731             }
 732         }
 733     }
 734
 735     close($fh);
 736
 737     return $res;
 738 }
 739
 740 # warning: this is slow
 741 sub check_running {
 742     my ($vmid) = @_;
 743
 744     my $active_hash = list_active_containers();
 745
 746     return 1 if defined($active_hash->{$vmid});
 747
 748     return undef;
 749 }
 750
 751 sub get_container_disk_usage {
 752     my ($vmid, $pid) = @_;
 753
 754     return PVE::Tools::df("/proc/$pid/root/", 1);
 755 }
 756
 757 my $last_proc_vmid_stat;
 758
 759 my $parse_cpuacct_stat = sub {
 760     my ($vmid) = @_;
 761
 762     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 763
 764     my $stat = {};
 765
 766     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 767
 768         $stat->{utime} = $1;
 769         $stat->{stime} = $2;
 770
 771     }
 772
 773     return $stat;
 774 };
 775
 776 sub vmstatus {
 777     my ($opt_vmid) = @_;
 778
 779     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 780
 781     my $active_hash = list_active_containers();
 782
 783     my $cpucount = $cpuinfo->{cpus} || 1;
 784
 785     my $cdtime = gettimeofday;
 786
 787     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 788
 789     foreach my $vmid (keys %$list) {
 790         my $d = $list->{$vmid};
 791
 792         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 793         warn $@ if $@; # ignore errors (consider them stopped)
 794
 795         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 796
 797         my $cfspath = cfs_config_path($vmid);
 798         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 799
 800         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 801         $d->{name} =~ s/[\s]//g;
 802
 803         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 804
 805         if ($d->{pid}) {
 806             my $res = get_container_disk_usage($vmid, $d->{pid});
 807             $d->{disk} = $res->{used};
 808             $d->{maxdisk} = $res->{total};
 809         } else {
 810             $d->{disk} = 0;
 811             # use 4GB by default ??
 812             if (my $rootfs = $conf->{rootfs}) {
 813                 my $rootinfo = parse_ct_mountpoint($rootfs);
 814                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 815             } else {
 816                 $d->{maxdisk} = 4*1024*1024*1024;
 817             }
 818         }
 819
 820         $d->{mem} = 0;
 821         $d->{swap} = 0;
 822         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 823         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 824
 825         $d->{uptime} = 0;
 826         $d->{cpu} = 0;
 827
 828         $d->{netout} = 0;
 829         $d->{netin} = 0;
 830
 831         $d->{diskread} = 0;
 832         $d->{diskwrite} = 0;
 833
 834         $d->{template} = is_template($conf);
 835     }
 836
 837     foreach my $vmid (keys %$list) {
 838         my $d = $list->{$vmid};
 839         my $pid = $d->{pid};
 840
 841         next if !$pid; # skip stopped CTs
 842
 843         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 844         $d->{uptime} = time - $ctime; # the method lxcfs uses
 845
 846         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 847         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 848
 849         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 850         my @bytes = split(/\n/, $blkio_bytes);
 851         foreach my $byte (@bytes) {
 852             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 853                 $d->{diskread} = $2 if $key eq 'Read';
 854                 $d->{diskwrite} = $2 if $key eq 'Write';
 855             }
 856         }
 857
 858         my $pstat = &$parse_cpuacct_stat($vmid);
 859
 860         my $used = $pstat->{utime} + $pstat->{stime};
 861
 862         my $old = $last_proc_vmid_stat->{$vmid};
 863         if (!$old) {
 864             $last_proc_vmid_stat->{$vmid} = {
 865                 time => $cdtime,
 866                 used => $used,
 867                 cpu => 0,
 868             };
 869             next;
 870         }
 871
 872         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 873
 874         if ($dtime > 1000) {
 875             my $dutime = $used -  $old->{used};
 876
 877             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 878             $last_proc_vmid_stat->{$vmid} = {
 879                 time => $cdtime,
 880                 used => $used,
 881                 cpu => $d->{cpu},
 882             };
 883         } else {
 884             $d->{cpu} = $old->{cpu};
 885         }
 886     }
 887
 888     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 889
 890     foreach my $dev (keys %$netdev) {
 891         next if $dev !~ m/^veth([1-9]\d*)i/;
 892         my $vmid = $1;
 893         my $d = $list->{$vmid};
 894
 895         next if !$d;
 896
 897         $d->{netout} += $netdev->{$dev}->{receive};
 898         $d->{netin} += $netdev->{$dev}->{transmit};
 899
 900     }
 901
 902     return $list;
 903 }
 904
 905 sub parse_ct_mountpoint {
 906     my ($data, $noerr) = @_;
 907
 908     $data //= '';
 909
 910     my $res;
 911     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 912     if ($@) {
 913         return undef if $noerr;
 914         die $@;
 915     }
 916
 917     if (defined(my $size = $res->{size})) {
 918         $size = PVE::JSONSchema::parse_size($size);
 919         if (!defined($size)) {
 920             return undef if $noerr;
 921             die "invalid size: $size\n";
 922         }
 923         $res->{size} = $size;
 924     }
 925
 926     return $res;
 927 }
 928
 929 sub print_ct_mountpoint {
 930     my ($info, $nomp) = @_;
 931     my $skip = $nomp ? ['mp'] : [];
 932     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 933 }
 934
 935 sub print_lxc_network {
 936     my $net = shift;
 937     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 938 }
 939
 940 sub parse_lxc_network {
 941     my ($data) = @_;
 942
 943     my $res = {};
 944
 945     return $res if !$data;
 946
 947     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 948
 949     $res->{type} = 'veth';
 950     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 951
 952     return $res;
 953 }
 954
 955 sub read_cgroup_value {
 956     my ($group, $vmid, $name, $full) = @_;
 957
 958     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 959
 960     return PVE::Tools::file_get_contents($path) if $full;
 961
 962     return PVE::Tools::file_read_firstline($path);
 963 }
 964
 965 sub write_cgroup_value {
 966    my ($group, $vmid, $name, $value) = @_;
 967
 968    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 969    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 970
 971 }
 972
 973 sub find_lxc_console_pids {
 974
 975     my $res = {};
 976
 977     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 978         my ($pid) = @_;
 979
 980         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 981         return if !$cmdline;
 982
 983         my @args = split(/\0/, $cmdline);
 984
 985         # serach for lxc-console -n <vmid>
 986         return if scalar(@args) != 3;
 987         return if $args[1] ne '-n';
 988         return if $args[2] !~ m/^\d+$/;
 989         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 990
 991         my $vmid = $args[2];
 992
 993         push @{$res->{$vmid}}, $pid;
 994     });
 995
 996     return $res;
 997 }
 998
 999 sub find_lxc_pid {
1000     my ($vmid) = @_;
1001
1002     my $pid = undef;
1003     my $parser = sub {
1004         my $line = shift;
1005         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1006     };
1007     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1008
1009     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1010
1011     return $pid;
1012 }
1013
1014 # Note: we cannot use Net:IP, because that only allows strict
1015 # CIDR networks
1016 sub parse_ipv4_cidr {
1017     my ($cidr, $noerr) = @_;
1018
1019     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1020         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1021     }
1022
1023     return undef if $noerr;
1024
1025     die "unable to parse ipv4 address/mask\n";
1026 }
1027
1028 sub check_lock {
1029     my ($conf) = @_;
1030
1031     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1032 }
1033
1034 sub check_protection {
1035     my ($vm_conf, $err_msg) = @_;
1036
1037     if ($vm_conf->{protection}) {
1038         die "$err_msg - protection mode enabled\n";
1039     }
1040 }
1041
1042 sub update_lxc_config {
1043     my ($storage_cfg, $vmid, $conf) = @_;
1044
1045     my $dir = "/var/lib/lxc/$vmid";
1046
1047     if ($conf->{template}) {
1048
1049         unlink "$dir/config";
1050
1051         return;
1052     }
1053
1054     my $raw = '';
1055
1056     die "missing 'arch' - internal error" if !$conf->{arch};
1057     $raw .= "lxc.arch = $conf->{arch}\n";
1058
1059     my $unprivileged = $conf->{unprivileged};
1060     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1061
1062     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1063     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1064         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1065         if ($unprivileged || $custom_idmap) {
1066             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1067         }
1068     } else {
1069         die "implement me";
1070     }
1071
1072     $raw .= "lxc.start.unshare = 1\n";
1073
1074     # Should we read them from /etc/subuid?
1075     if ($unprivileged && !$custom_idmap) {
1076         $raw .= "lxc.id_map = u 0 100000 65536\n";
1077         $raw .= "lxc.id_map = g 0 100000 65536\n";
1078     }
1079
1080     if (!has_dev_console($conf)) {
1081         $raw .= "lxc.console = none\n";
1082         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1083     }
1084
1085     my $ttycount = get_tty_count($conf);
1086     $raw .= "lxc.tty = $ttycount\n";
1087
1088     # some init scripts expects a linux terminal (turnkey).
1089     $raw .= "lxc.environment = TERM=linux\n";
1090
1091     my $utsname = $conf->{hostname} || "CT$vmid";
1092     $raw .= "lxc.utsname = $utsname\n";
1093
1094     my $memory = $conf->{memory} || 512;
1095     my $swap = $conf->{swap} // 0;
1096
1097     my $lxcmem = int($memory*1024*1024);
1098     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1099     $raw .= "lxc.cgroup.memory.kmem.limit_in_bytes = $lxcmem\n";
1100
1101     my $lxcswap = int(($memory + $swap)*1024*1024);
1102     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1103
1104     if (my $cpulimit = $conf->{cpulimit}) {
1105         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1106         my $value = int(100000*$cpulimit);
1107         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1108     }
1109
1110     my $shares = $conf->{cpuunits} || 1024;
1111     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1112
1113     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1114     $mountpoint->{mp} = '/';
1115
1116     $raw .= "lxc.rootfs = $dir/rootfs\n";
1117     $raw .= "lxc.hook.stop = /usr/lib/x86_64-linux-gnu/lxc/hooks/unmount-namespace\n";
1118
1119     my $netcount = 0;
1120     foreach my $k (keys %$conf) {
1121         next if $k !~ m/^net(\d+)$/;
1122         my $ind = $1;
1123         my $d = parse_lxc_network($conf->{$k});
1124         $netcount++;
1125         $raw .= "lxc.network.type = veth\n";
1126         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1127         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1128         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1129         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1130     }
1131
1132     if (my $lxcconf = $conf->{lxc}) {
1133         foreach my $entry (@$lxcconf) {
1134             my ($k, $v) = @$entry;
1135             $netcount++ if $k eq 'lxc.network.type';
1136             $raw .= "$k = $v\n";
1137         }
1138     }
1139
1140     $raw .= "lxc.network.type = empty\n" if !$netcount;
1141
1142     File::Path::mkpath("$dir/rootfs");
1143
1144     PVE::Tools::file_set_contents("$dir/config", $raw);
1145 }
1146
1147 # verify and cleanup nameserver list (replace \0 with ' ')
1148 sub verify_nameserver_list {
1149     my ($nameserver_list) = @_;
1150
1151     my @list = ();
1152     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1153         PVE::JSONSchema::pve_verify_ip($server);
1154         push @list, $server;
1155     }
1156
1157     return join(' ', @list);
1158 }
1159
1160 sub verify_searchdomain_list {
1161     my ($searchdomain_list) = @_;
1162
1163     my @list = ();
1164     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1165         # todo: should we add checks for valid dns domains?
1166         push @list, $server;
1167     }
1168
1169     return join(' ', @list);
1170 }
1171
1172 sub add_unused_volume {
1173     my ($config, $volid) = @_;
1174
1175     # skip bind mounts and block devices
1176     return if $volid =~ m|^/|;
1177
1178     my $key;
1179     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1180         my $test = "unused$ind";
1181         if (my $vid = $config->{$test}) {
1182             return if $vid eq $volid; # do not add duplicates
1183         } else {
1184             $key = $test;
1185         }
1186     }
1187
1188     die "To many unused volume - please delete them first.\n" if !$key;
1189
1190     $config->{$key} = $volid;
1191
1192     return $key;
1193 }
1194
1195 sub update_pct_config {
1196     my ($vmid, $conf, $running, $param, $delete) = @_;
1197
1198     my @nohotplug;
1199
1200     my $new_disks = 0;
1201     my @deleted_volumes;
1202
1203     my $rootdir;
1204     if ($running) {
1205         my $pid = find_lxc_pid($vmid);
1206         $rootdir = "/proc/$pid/root";
1207     }
1208
1209     my $hotplug_error = sub {
1210         if ($running) {
1211             push @nohotplug, @_;
1212             return 1;
1213         } else {
1214             return 0;
1215         }
1216     };
1217
1218     if (defined($delete)) {
1219         foreach my $opt (@$delete) {
1220             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1221                 die "unable to delete required option '$opt'\n";
1222             } elsif ($opt eq 'swap') {
1223                 delete $conf->{$opt};
1224                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1225             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1226                 delete $conf->{$opt};
1227             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1228                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1229                 next if $hotplug_error->($opt);
1230                 delete $conf->{$opt};
1231             } elsif ($opt =~ m/^net(\d)$/) {
1232                 delete $conf->{$opt};
1233                 next if !$running;
1234                 my $netid = $1;
1235                 PVE::Network::veth_delete("veth${vmid}i$netid");
1236             } elsif ($opt eq 'protection') {
1237                 delete $conf->{$opt};
1238             } elsif ($opt =~ m/^unused(\d+)$/) {
1239                 next if $hotplug_error->($opt);
1240                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1241                 push @deleted_volumes, $conf->{$opt};
1242                 delete $conf->{$opt};
1243             } elsif ($opt =~ m/^mp(\d+)$/) {
1244                 next if $hotplug_error->($opt);
1245                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1246                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1247                 add_unused_volume($conf, $mountpoint->{volume});
1248                 delete $conf->{$opt};
1249             } elsif ($opt eq 'unprivileged') {
1250                 die "unable to delete read-only option: '$opt'\n";
1251             } else {
1252                 die "implement me"
1253             }
1254             write_config($vmid, $conf) if $running;
1255         }
1256     }
1257
1258     # There's no separate swap size to configure, there's memory and "total"
1259     # memory (iow. memory+swap). This means we have to change them together.
1260     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1261     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1262     if (defined($wanted_memory) || defined($wanted_swap)) {
1263
1264         $wanted_memory //= ($conf->{memory} || 512);
1265         $wanted_swap //=  ($conf->{swap} || 0);
1266
1267         my $total = $wanted_memory + $wanted_swap;
1268         if ($running) {
1269             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1270             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1271         }
1272         $conf->{memory} = $wanted_memory;
1273         $conf->{swap} = $wanted_swap;
1274
1275         write_config($vmid, $conf) if $running;
1276     }
1277
1278     foreach my $opt (keys %$param) {
1279         my $value = $param->{$opt};
1280         if ($opt eq 'hostname') {
1281             $conf->{$opt} = $value;
1282         } elsif ($opt eq 'onboot') {
1283             $conf->{$opt} = $value ? 1 : 0;
1284         } elsif ($opt eq 'startup') {
1285             $conf->{$opt} = $value;
1286         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1287             next if $hotplug_error->($opt);
1288             $conf->{$opt} = $value;
1289         } elsif ($opt eq 'nameserver') {
1290             next if $hotplug_error->($opt);
1291             my $list = verify_nameserver_list($value);
1292             $conf->{$opt} = $list;
1293         } elsif ($opt eq 'searchdomain') {
1294             next if $hotplug_error->($opt);
1295             my $list = verify_searchdomain_list($value);
1296             $conf->{$opt} = $list;
1297         } elsif ($opt eq 'cpulimit') {
1298             next if $hotplug_error->($opt); # FIXME: hotplug
1299             $conf->{$opt} = $value;
1300         } elsif ($opt eq 'cpuunits') {
1301             $conf->{$opt} = $value;
1302             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1303         } elsif ($opt eq 'description') {
1304             $conf->{$opt} = PVE::Tools::encode_text($value);
1305         } elsif ($opt =~ m/^net(\d+)$/) {
1306             my $netid = $1;
1307             my $net = parse_lxc_network($value);
1308             if (!$running) {
1309                 $conf->{$opt} = print_lxc_network($net);
1310             } else {
1311                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1312             }
1313         } elsif ($opt eq 'protection') {
1314             $conf->{$opt} = $value ? 1 : 0;
1315         } elsif ($opt =~ m/^mp(\d+)$/) {
1316             next if $hotplug_error->($opt);
1317             check_protection($conf, "can't update CT $vmid drive '$opt'");
1318             $conf->{$opt} = $value;
1319             $new_disks = 1;
1320         } elsif ($opt eq 'rootfs') {
1321             check_protection($conf, "can't update CT $vmid drive '$opt'");
1322             die "implement me: $opt";
1323         } elsif ($opt eq 'unprivileged') {
1324             die "unable to modify read-only option: '$opt'\n";
1325         } else {
1326             die "implement me: $opt";
1327         }
1328         write_config($vmid, $conf) if $running;
1329     }
1330
1331     if (@deleted_volumes) {
1332         my $storage_cfg = PVE::Storage::config();
1333         foreach my $volume (@deleted_volumes) {
1334             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1335         }
1336     }
1337
1338     if ($new_disks) {
1339         my $storage_cfg = PVE::Storage::config();
1340         create_disks($storage_cfg, $vmid, $conf, $conf);
1341     }
1342
1343     # This should be the last thing we do here
1344     if ($running && scalar(@nohotplug)) {
1345         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1346     }
1347 }
1348
1349 sub has_dev_console {
1350     my ($conf) = @_;
1351
1352     return !(defined($conf->{console}) && !$conf->{console});
1353 }
1354
1355 sub get_tty_count {
1356     my ($conf) = @_;
1357
1358     return $conf->{tty} // $confdesc->{tty}->{default};
1359 }
1360
1361 sub get_cmode {
1362     my ($conf) = @_;
1363
1364     return $conf->{cmode} // $confdesc->{cmode}->{default};
1365 }
1366
1367 sub get_console_command {
1368     my ($vmid, $conf) = @_;
1369
1370     my $cmode = get_cmode($conf);
1371
1372     if ($cmode eq 'console') {
1373         return ['lxc-console', '-n',  $vmid, '-t', 0];
1374     } elsif ($cmode eq 'tty') {
1375         return ['lxc-console', '-n',  $vmid];
1376     } elsif ($cmode eq 'shell') {
1377         return ['lxc-attach', '--clear-env', '-n', $vmid];
1378     } else {
1379         die "internal error";
1380     }
1381 }
1382
1383 sub get_primary_ips {
1384     my ($conf) = @_;
1385
1386     # return data from net0
1387
1388     return undef if !defined($conf->{net0});
1389     my $net = parse_lxc_network($conf->{net0});
1390
1391     my $ipv4 = $net->{ip};
1392     if ($ipv4) {
1393         if ($ipv4 =~ /^(dhcp|manual)$/) {
1394             $ipv4 = undef
1395         } else {
1396             $ipv4 =~ s!/\d+$!!;
1397         }
1398     }
1399     my $ipv6 = $net->{ip6};
1400     if ($ipv6) {
1401         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1402             $ipv6 = undef;
1403         } else {
1404             $ipv6 =~ s!/\d+$!!;
1405         }
1406     }
1407
1408     return ($ipv4, $ipv6);
1409 }
1410
1411 sub delete_mountpoint_volume {
1412     my ($storage_cfg, $vmid, $volume) = @_;
1413
1414     # skip bind mounts and block devices
1415     if ($volume =~ m|^/|) {
1416             return;
1417     }
1418
1419     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1420     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1421 }
1422
1423 sub destroy_lxc_container {
1424     my ($storage_cfg, $vmid, $conf) = @_;
1425
1426     foreach_mountpoint($conf, sub {
1427         my ($ms, $mountpoint) = @_;
1428         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1429     });
1430
1431     rmdir "/var/lib/lxc/$vmid/rootfs";
1432     unlink "/var/lib/lxc/$vmid/config";
1433     rmdir "/var/lib/lxc/$vmid";
1434     destroy_config($vmid);
1435
1436     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1437     #PVE::Tools::run_command($cmd);
1438 }
1439
1440 sub vm_stop_cleanup {
1441     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1442
1443     eval {
1444         if (!$keepActive) {
1445
1446             my $vollist = get_vm_volumes($conf);
1447             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1448         }
1449     };
1450     warn $@ if $@; # avoid errors - just warn
1451 }
1452
1453 my $safe_num_ne = sub {
1454     my ($a, $b) = @_;
1455
1456     return 0 if !defined($a) && !defined($b);
1457     return 1 if !defined($a);
1458     return 1 if !defined($b);
1459
1460     return $a != $b;
1461 };
1462
1463 my $safe_string_ne = sub {
1464     my ($a, $b) = @_;
1465
1466     return 0 if !defined($a) && !defined($b);
1467     return 1 if !defined($a);
1468     return 1 if !defined($b);
1469
1470     return $a ne $b;
1471 };
1472
1473 sub update_net {
1474     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1475
1476     if ($newnet->{type} ne 'veth') {
1477         # for when there are physical interfaces
1478         die "cannot update interface of type $newnet->{type}";
1479     }
1480
1481     my $veth = "veth${vmid}i${netid}";
1482     my $eth = $newnet->{name};
1483
1484     if (my $oldnetcfg = $conf->{$opt}) {
1485         my $oldnet = parse_lxc_network($oldnetcfg);
1486
1487         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1488             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1489
1490             PVE::Network::veth_delete($veth);
1491             delete $conf->{$opt};
1492             write_config($vmid, $conf);
1493
1494             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1495
1496         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1497                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1498                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1499
1500                 if ($oldnet->{bridge}) {
1501                     PVE::Network::tap_unplug($veth);
1502                     foreach (qw(bridge tag firewall)) {
1503                         delete $oldnet->{$_};
1504                     }
1505                     $conf->{$opt} = print_lxc_network($oldnet);
1506                     write_config($vmid, $conf);
1507                 }
1508
1509                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1510                 foreach (qw(bridge tag firewall)) {
1511                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1512                 }
1513                 $conf->{$opt} = print_lxc_network($oldnet);
1514                 write_config($vmid, $conf);
1515         }
1516     } else {
1517         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1518     }
1519
1520     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1521 }
1522
1523 sub hotplug_net {
1524     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1525
1526     my $veth = "veth${vmid}i${netid}";
1527     my $vethpeer = $veth . "p";
1528     my $eth = $newnet->{name};
1529
1530     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1531     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1532
1533     # attach peer in container
1534     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1535     PVE::Tools::run_command($cmd);
1536
1537     # link up peer in container
1538     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1539     PVE::Tools::run_command($cmd);
1540
1541     my $done = { type => 'veth' };
1542     foreach (qw(bridge tag firewall hwaddr name)) {
1543         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1544     }
1545     $conf->{$opt} = print_lxc_network($done);
1546
1547     write_config($vmid, $conf);
1548 }
1549
1550 sub update_ipconfig {
1551     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1552
1553     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1554
1555     my $optdata = parse_lxc_network($conf->{$opt});
1556     my $deleted = [];
1557     my $added = [];
1558     my $nscmd = sub {
1559         my $cmdargs = shift;
1560         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1561     };
1562     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1563
1564     my $change_ip_config = sub {
1565         my ($ipversion) = @_;
1566
1567         my $family_opt = "-$ipversion";
1568         my $suffix = $ipversion == 4 ? '' : $ipversion;
1569         my $gw= "gw$suffix";
1570         my $ip= "ip$suffix";
1571
1572         my $newip = $newnet->{$ip};
1573         my $newgw = $newnet->{$gw};
1574         my $oldip = $optdata->{$ip};
1575
1576         my $change_ip = &$safe_string_ne($oldip, $newip);
1577         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1578
1579         return if !$change_ip && !$change_gw;
1580
1581         # step 1: add new IP, if this fails we cancel
1582         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1583         if ($change_ip && $is_real_ip) {
1584             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1585             if (my $err = $@) {
1586                 warn $err;
1587                 return;
1588             }
1589         }
1590
1591         # step 2: replace gateway
1592         #   If this fails we delete the added IP and cancel.
1593         #   If it succeeds we save the config and delete the old IP, ignoring
1594         #   errors. The config is then saved.
1595         # Note: 'ip route replace' can add
1596         if ($change_gw) {
1597             if ($newgw) {
1598                 eval {
1599                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1600                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1601                     }
1602                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1603                 };
1604                 if (my $err = $@) {
1605                     warn $err;
1606                     # the route was not replaced, the old IP is still available
1607                     # rollback (delete new IP) and cancel
1608                     if ($change_ip) {
1609                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1610                         warn $@ if $@; # no need to die here
1611                     }
1612                     return;
1613                 }
1614             } else {
1615                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1616                 # if the route was not deleted, the guest might have deleted it manually
1617                 # warn and continue
1618                 warn $@ if $@;
1619             }
1620         }
1621
1622         # from this point on we save the configuration
1623         # step 3: delete old IP ignoring errors
1624         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1625             # We need to enable promote_secondaries, otherwise our newly added
1626             # address will be removed along with the old one.
1627             my $promote = 0;
1628             eval {
1629                 if ($ipversion == 4) {
1630                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1631                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1632                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1633                 }
1634                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1635             };
1636             warn $@ if $@; # no need to die here
1637
1638             if ($ipversion == 4) {
1639                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1640             }
1641         }
1642
1643         foreach my $property ($ip, $gw) {
1644             if ($newnet->{$property}) {
1645                 $optdata->{$property} = $newnet->{$property};
1646             } else {
1647                 delete $optdata->{$property};
1648             }
1649         }
1650         $conf->{$opt} = print_lxc_network($optdata);
1651         write_config($vmid, $conf);
1652         $lxc_setup->setup_network($conf);
1653     };
1654
1655     &$change_ip_config(4);
1656     &$change_ip_config(6);
1657
1658 }
1659
1660 # Internal snapshots
1661
1662 # NOTE: Snapshot create/delete involves several non-atomic
1663 # action, and can take a long time.
1664 # So we try to avoid locking the file and use 'lock' variable
1665 # inside the config file instead.
1666
1667 my $snapshot_copy_config = sub {
1668     my ($source, $dest) = @_;
1669
1670     foreach my $k (keys %$source) {
1671         next if $k eq 'snapshots';
1672         next if $k eq 'snapstate';
1673         next if $k eq 'snaptime';
1674         next if $k eq 'vmstate';
1675         next if $k eq 'lock';
1676         next if $k eq 'digest';
1677         next if $k eq 'description';
1678
1679         $dest->{$k} = $source->{$k};
1680     }
1681 };
1682
1683 my $snapshot_prepare = sub {
1684     my ($vmid, $snapname, $comment) = @_;
1685
1686     my $snap;
1687
1688     my $updatefn =  sub {
1689
1690         my $conf = load_config($vmid);
1691
1692         die "you can't take a snapshot if it's a template\n"
1693             if is_template($conf);
1694
1695         check_lock($conf);
1696
1697         $conf->{lock} = 'snapshot';
1698
1699         die "snapshot name '$snapname' already used\n"
1700             if defined($conf->{snapshots}->{$snapname});
1701
1702         my $storecfg = PVE::Storage::config();
1703         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1704
1705         $snap = $conf->{snapshots}->{$snapname} = {};
1706
1707         &$snapshot_copy_config($conf, $snap);
1708
1709         $snap->{'snapstate'} = "prepare";
1710         $snap->{'snaptime'} = time();
1711         $snap->{'description'} = $comment if $comment;
1712         $conf->{snapshots}->{$snapname} = $snap;
1713
1714         write_config($vmid, $conf);
1715     };
1716
1717     lock_container($vmid, 10, $updatefn);
1718
1719     return $snap;
1720 };
1721
1722 my $snapshot_commit = sub {
1723     my ($vmid, $snapname) = @_;
1724
1725     my $updatefn = sub {
1726
1727         my $conf = load_config($vmid);
1728
1729         die "missing snapshot lock\n"
1730             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1731
1732         die "snapshot '$snapname' does not exist\n"
1733             if !defined($conf->{snapshots}->{$snapname});
1734
1735         die "wrong snapshot state\n"
1736             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1737                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1738
1739         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1740         delete $conf->{lock};
1741         $conf->{parent} = $snapname;
1742
1743         write_config($vmid, $conf);
1744     };
1745
1746     lock_container($vmid, 10 ,$updatefn);
1747 };
1748
1749 sub has_feature {
1750     my ($feature, $conf, $storecfg, $snapname) = @_;
1751
1752     my $err;
1753
1754     foreach_mountpoint($conf, sub {
1755         my ($ms, $mountpoint) = @_;
1756
1757         return if $err; # skip further test
1758
1759         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1760
1761         # TODO: implement support for mountpoints
1762         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1763             if $ms ne 'rootfs';
1764     });
1765
1766     return $err ? 0 : 1;
1767 }
1768
1769 sub snapshot_create {
1770     my ($vmid, $snapname, $comment) = @_;
1771
1772     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1773
1774     my $conf = load_config($vmid);
1775
1776     my $running = check_running($vmid);
1777     eval {
1778         if ($running) {
1779             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1780             PVE::Tools::run_command(['/bin/sync']);
1781         };
1782
1783         my $storecfg = PVE::Storage::config();
1784         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1785         my $volid = $rootinfo->{volume};
1786
1787         if ($running) {
1788             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1789         };
1790
1791         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1792         &$snapshot_commit($vmid, $snapname);
1793     };
1794     if(my $err = $@) {
1795         snapshot_delete($vmid, $snapname, 1);
1796         die "$err\n";
1797     }
1798 }
1799
1800 sub snapshot_delete {
1801     my ($vmid, $snapname, $force) = @_;
1802
1803     my $snap;
1804
1805     my $conf;
1806
1807     my $updatefn =  sub {
1808
1809         $conf = load_config($vmid);
1810
1811         die "you can't delete a snapshot if vm is a template\n"
1812             if is_template($conf);
1813
1814         $snap = $conf->{snapshots}->{$snapname};
1815
1816         check_lock($conf);
1817
1818         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1819
1820         $snap->{snapstate} = 'delete';
1821
1822         write_config($vmid, $conf);
1823     };
1824
1825     lock_container($vmid, 10, $updatefn);
1826
1827     my $storecfg = PVE::Storage::config();
1828
1829     my $del_snap =  sub {
1830
1831         check_lock($conf);
1832
1833         if ($conf->{parent} eq $snapname) {
1834             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1835                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1836             } else {
1837                 delete $conf->{parent};
1838             }
1839         }
1840
1841         delete $conf->{snapshots}->{$snapname};
1842
1843         write_config($vmid, $conf);
1844     };
1845
1846     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1847     my $rootinfo = parse_ct_mountpoint($rootfs);
1848     my $volid = $rootinfo->{volume};
1849
1850     eval {
1851         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1852     };
1853     my $err = $@;
1854
1855     if(!$err || ($err && $force)) {
1856         lock_container($vmid, 10, $del_snap);
1857         if ($err) {
1858             die "Can't delete snapshot: $vmid $snapname $err\n";
1859         }
1860     }
1861 }
1862
1863 sub snapshot_rollback {
1864     my ($vmid, $snapname) = @_;
1865
1866     my $storecfg = PVE::Storage::config();
1867
1868     my $conf = load_config($vmid);
1869
1870     die "you can't rollback if vm is a template\n" if is_template($conf);
1871
1872     my $snap = $conf->{snapshots}->{$snapname};
1873
1874     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1875
1876     my $rootfs = $snap->{rootfs};
1877     my $rootinfo = parse_ct_mountpoint($rootfs);
1878     my $volid = $rootinfo->{volume};
1879
1880     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1881
1882     my $updatefn = sub {
1883
1884         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1885             if $snap->{snapstate};
1886
1887         check_lock($conf);
1888
1889         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1890
1891         die "unable to rollback vm $vmid: vm is running\n"
1892             if check_running($vmid);
1893
1894         $conf->{lock} = 'rollback';
1895
1896         my $forcemachine;
1897
1898         # copy snapshot config to current config
1899
1900         my $tmp_conf = $conf;
1901         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1902         $conf->{snapshots} = $tmp_conf->{snapshots};
1903         delete $conf->{snaptime};
1904         delete $conf->{snapname};
1905         $conf->{parent} = $snapname;
1906
1907         write_config($vmid, $conf);
1908     };
1909
1910     my $unlockfn = sub {
1911         delete $conf->{lock};
1912         write_config($vmid, $conf);
1913     };
1914
1915     lock_container($vmid, 10, $updatefn);
1916
1917     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1918
1919     lock_container($vmid, 5, $unlockfn);
1920 }
1921
1922 sub template_create {
1923     my ($vmid, $conf) = @_;
1924
1925     my $storecfg = PVE::Storage::config();
1926
1927     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1928     my $volid = $rootinfo->{volume};
1929
1930     die "Template feature is not available for '$volid'\n"
1931         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1932
1933     PVE::Storage::activate_volumes($storecfg, [$volid]);
1934
1935     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1936     $rootinfo->{volume} = $template_volid;
1937     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1938
1939     write_config($vmid, $conf);
1940 }
1941
1942 sub is_template {
1943     my ($conf) = @_;
1944
1945     return 1 if defined $conf->{template} && $conf->{template} == 1;
1946 }
1947
1948 sub mountpoint_names {
1949     my ($reverse) = @_;
1950
1951     my @names = ('rootfs');
1952
1953     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1954         push @names, "mp$i";
1955     }
1956
1957     return $reverse ? reverse @names : @names;
1958 }
1959
1960 # The container might have *different* symlinks than the host. realpath/abs_path
1961 # use the actual filesystem to resolve links.
1962 sub sanitize_mountpoint {
1963     my ($mp) = @_;
1964     $mp = '/' . $mp; # we always start with a slash
1965     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1966     $mp =~ s@/\./@@g; # collapse /./
1967     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1968     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1969     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1970     return $mp;
1971 }
1972
1973 sub foreach_mountpoint_full {
1974     my ($conf, $reverse, $func) = @_;
1975
1976     foreach my $key (mountpoint_names($reverse)) {
1977         my $value = $conf->{$key};
1978         next if !defined($value);
1979         my $mountpoint = parse_ct_mountpoint($value, 1);
1980         next if !defined($mountpoint);
1981
1982         # just to be sure: rootfs is /
1983         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1984         $mountpoint->{mp} = sanitize_mountpoint($path);
1985
1986         $path = $mountpoint->{volume};
1987         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1988
1989         &$func($key, $mountpoint);
1990     }
1991 }
1992
1993 sub foreach_mountpoint {
1994     my ($conf, $func) = @_;
1995
1996     foreach_mountpoint_full($conf, 0, $func);
1997 }
1998
1999 sub foreach_mountpoint_reverse {
2000     my ($conf, $func) = @_;
2001
2002     foreach_mountpoint_full($conf, 1, $func);
2003 }
2004
2005 sub check_ct_modify_config_perm {
2006     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2007
2008     return 1 if $authuser ne 'root@pam';
2009
2010     foreach my $opt (@$key_list) {
2011
2012         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2013             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2014         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2015             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2016         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2017             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2018         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2019                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2020             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2021         } else {
2022             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2023         }
2024     }
2025
2026     return 1;
2027 }
2028
2029 sub umount_all {
2030     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2031
2032     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2033     my $volid_list = get_vm_volumes($conf);
2034
2035     foreach_mountpoint_reverse($conf, sub {
2036         my ($ms, $mountpoint) = @_;
2037
2038         my $volid = $mountpoint->{volume};
2039         my $mount = $mountpoint->{mp};
2040
2041         return if !$volid || !$mount;
2042
2043         my $mount_path = "$rootdir/$mount";
2044         $mount_path =~ s!/+!/!g;
2045
2046         return if !PVE::ProcFSTools::is_mounted($mount_path);
2047
2048         eval {
2049             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2050         };
2051         if (my $err = $@) {
2052             if ($noerr) {
2053                 warn $err;
2054             } else {
2055                 die $err;
2056             }
2057         }
2058     });
2059 }
2060
2061 sub mount_all {
2062     my ($vmid, $storage_cfg, $conf) = @_;
2063
2064     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2065     File::Path::make_path($rootdir);
2066
2067     my $volid_list = get_vm_volumes($conf);
2068     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2069
2070     eval {
2071         foreach_mountpoint($conf, sub {
2072             my ($ms, $mountpoint) = @_;
2073
2074             my $volid = $mountpoint->{volume};
2075             my $mount = $mountpoint->{mp};
2076
2077             return if !$volid || !$mount;
2078
2079             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2080             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2081                 PVE::Storage::parse_volname($storage_cfg, $volid);
2082
2083             die "unable to mount base volume - internal error" if $isBase;
2084
2085             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2086         });
2087     };
2088     if (my $err = $@) {
2089         warn "mounting container failed - $err";
2090         umount_all($vmid, $storage_cfg, $conf, 1);
2091     }
2092
2093     return $rootdir;
2094 }
2095
2096
2097 sub mountpoint_mount_path {
2098     my ($mountpoint, $storage_cfg, $snapname) = @_;
2099
2100     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2101 }
2102
2103 my $check_mount_path = sub {
2104     my ($path) = @_;
2105     $path = File::Spec->canonpath($path);
2106     my $real = Cwd::realpath($path);
2107     if ($real ne $path) {
2108         die "mount path modified by symlink: $path != $real";
2109     }
2110 };
2111
2112 sub query_loopdev {
2113     my ($path) = @_;
2114     my $found;
2115     my $parser = sub {
2116         my $line = shift;
2117         if ($line =~ m@^(/dev/loop\d+):@) {
2118             $found = $1;
2119         }
2120     };
2121     my $cmd = ['losetup', '--associated', $path];
2122     PVE::Tools::run_command($cmd, outfunc => $parser);
2123     return $found;
2124 }
2125
2126 # use $rootdir = undef to just return the corresponding mount path
2127 sub mountpoint_mount {
2128     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2129
2130     my $volid = $mountpoint->{volume};
2131     my $mount = $mountpoint->{mp};
2132
2133     return if !$volid || !$mount;
2134
2135     my $mount_path;
2136
2137     if (defined($rootdir)) {
2138         $rootdir =~ s!/+$!!;
2139         $mount_path = "$rootdir/$mount";
2140         $mount_path =~ s!/+!/!g;
2141         &$check_mount_path($mount_path);
2142         File::Path::mkpath($mount_path);
2143     }
2144
2145     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2146
2147     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2148
2149     if ($storage) {
2150
2151         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2152         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2153
2154         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2155             PVE::Storage::parse_volname($storage_cfg, $volid);
2156
2157         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2158
2159         if ($format eq 'subvol') {
2160             if ($mount_path) {
2161                 if ($snapname) {
2162                     if ($scfg->{type} ne 'zfspool') {
2163                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2164                     }
2165                 }
2166                 PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2167             }
2168             return wantarray ? ($path, 0) : $path;
2169         } elsif ($format eq 'raw' || $format eq 'iso') {
2170             my $use_loopdev = 0;
2171             my @extra_opts;
2172             if ($scfg->{path}) {
2173                 push @extra_opts, '-o', 'loop';
2174                 $use_loopdev = 1;
2175             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2176                 # do nothing
2177             } else {
2178                 die "unsupported storage type '$scfg->{type}'\n";
2179             }
2180             if ($mount_path) {
2181                 if ($format eq 'iso') {
2182                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2183                 } elsif ($isBase || defined($snapname)) {
2184                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2185                 } else {
2186                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2187                 }
2188             }
2189             return wantarray ? ($path, $use_loopdev) : $path;
2190         } else {
2191             die "unsupported image format '$format'\n";
2192         }
2193     } elsif ($volid =~ m|^/dev/.+|) {
2194         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2195         return wantarray ? ($volid, 0) : $volid;
2196     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2197         &$check_mount_path($volid);
2198         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2199         return wantarray ? ($volid, 0) : $volid;
2200     }
2201
2202     die "unsupported storage";
2203 }
2204
2205 sub get_vm_volumes {
2206     my ($conf, $excludes) = @_;
2207
2208     my $vollist = [];
2209
2210     foreach_mountpoint($conf, sub {
2211         my ($ms, $mountpoint) = @_;
2212
2213         return if $excludes && $ms eq $excludes;
2214
2215         my $volid = $mountpoint->{volume};
2216
2217         return if !$volid || $volid =~ m|^/|;
2218
2219         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2220         return if !$sid;
2221
2222         push @$vollist, $volid;
2223     });
2224
2225     return $vollist;
2226 }
2227
2228 sub mkfs {
2229     my ($dev, $rootuid, $rootgid) = @_;
2230
2231     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp',
2232                              '-E', "root_owner=$rootuid:$rootgid",
2233                              $dev]);
2234 }
2235
2236 sub format_disk {
2237     my ($storage_cfg, $volid, $rootuid, $rootgid) = @_;
2238
2239     if ($volid =~ m!^/dev/.+!) {
2240         mkfs($volid);
2241         return;
2242     }
2243
2244     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2245
2246     die "cannot format volume '$volid' with no storage\n" if !$storage;
2247
2248     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2249
2250     my $path = PVE::Storage::path($storage_cfg, $volid);
2251
2252     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2253         PVE::Storage::parse_volname($storage_cfg, $volid);
2254
2255     die "cannot format volume '$volid' (format == $format)\n"
2256         if $format ne 'raw';
2257
2258     mkfs($path, $rootuid, $rootgid);
2259 }
2260
2261 sub destroy_disks {
2262     my ($storecfg, $vollist) = @_;
2263
2264     foreach my $volid (@$vollist) {
2265         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2266         warn $@ if $@;
2267     }
2268 }
2269
2270 sub create_disks {
2271     my ($storecfg, $vmid, $settings, $conf) = @_;
2272
2273     my $vollist = [];
2274
2275     eval {
2276         my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
2277         my $chown_vollist = [];
2278
2279         foreach_mountpoint($settings, sub {
2280             my ($ms, $mountpoint) = @_;
2281
2282             my $volid = $mountpoint->{volume};
2283             my $mp = $mountpoint->{mp};
2284
2285             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2286
2287             return if !$storage;
2288
2289             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2290                 my ($storeid, $size_gb) = ($1, $2);
2291
2292                 my $size_kb = int(${size_gb}*1024) * 1024;
2293
2294                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2295                 # fixme: use better naming ct-$vmid-disk-X.raw?
2296
2297                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2298                     if ($size_kb > 0) {
2299                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2300                                                            undef, $size_kb);
2301                         format_disk($storecfg, $volid, $rootuid, $rootgid);
2302                     } else {
2303                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2304                                                            undef, 0);
2305                         push @$chown_vollist, $volid;
2306                     }
2307                 } elsif ($scfg->{type} eq 'zfspool') {
2308
2309                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2310                                                        undef, $size_kb);
2311                     push @$chown_vollist, $volid;
2312                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2313
2314                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2315                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2316
2317                 } elsif ($scfg->{type} eq 'rbd') {
2318
2319                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2320                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2321                     format_disk($storecfg, $volid, $rootuid, $rootgid);
2322                 } else {
2323                     die "unable to create containers on storage type '$scfg->{type}'\n";
2324                 }
2325                 push @$vollist, $volid;
2326                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2327                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2328             } else {
2329                 # use specified/existing volid
2330             }
2331         });
2332
2333         PVE::Storage::activate_volumes($storecfg, $chown_vollist, undef);
2334         foreach my $volid (@$chown_vollist) {
2335             my $path = PVE::Storage::path($storecfg, $volid, undef);
2336             chown($rootuid, $rootgid, $path);
2337         }
2338         PVE::Storage::deactivate_volumes($storecfg, $chown_vollist, undef);
2339     };
2340     # free allocated images on error
2341     if (my $err = $@) {
2342         destroy_disks($storecfg, $vollist);
2343         die $err;
2344     }
2345     return $vollist;
2346 }
2347
2348 # bash completion helper
2349
2350 sub complete_os_templates {
2351     my ($cmdname, $pname, $cvalue) = @_;
2352
2353     my $cfg = PVE::Storage::config();
2354
2355     my $storeid;
2356
2357     if ($cvalue =~ m/^([^:]+):/) {
2358         $storeid = $1;
2359     }
2360
2361     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2362     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2363
2364     my $res = [];
2365     foreach my $id (keys %$data) {
2366         foreach my $item (@{$data->{$id}}) {
2367             push @$res, $item->{volid} if defined($item->{volid});
2368         }
2369     }
2370
2371     return $res;
2372 }
2373
2374 my $complete_ctid_full = sub {
2375     my ($running) = @_;
2376
2377     my $idlist = vmstatus();
2378
2379     my $active_hash = list_active_containers();
2380
2381     my $res = [];
2382
2383     foreach my $id (keys %$idlist) {
2384         my $d = $idlist->{$id};
2385         if (defined($running)) {
2386             next if $d->{template};
2387             next if $running && !$active_hash->{$id};
2388             next if !$running && $active_hash->{$id};
2389         }
2390         push @$res, $id;
2391
2392     }
2393     return $res;
2394 };
2395
2396 sub complete_ctid {
2397     return &$complete_ctid_full();
2398 }
2399
2400 sub complete_ctid_stopped {
2401     return &$complete_ctid_full(0);
2402 }
2403
2404 sub complete_ctid_running {
2405     return &$complete_ctid_full(1);
2406 }
2407
2408 sub parse_id_maps {
2409     my ($conf) = @_;
2410
2411     my $id_map = [];
2412     my $rootuid = 0;
2413     my $rootgid = 0;
2414
2415     my $lxc = $conf->{lxc};
2416     foreach my $entry (@$lxc) {
2417         my ($key, $value) = @$entry;
2418         next if $key ne 'lxc.id_map';
2419         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2420             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2421             push @$id_map, [$type, $ct, $host, $length];
2422             if ($ct == 0) {
2423                 $rootuid = $host if $type eq 'u';
2424                 $rootgid = $host if $type eq 'g';
2425             }
2426         } else {
2427             die "failed to parse id_map: $value\n";
2428         }
2429     }
2430
2431     if (!@$id_map && $conf->{unprivileged}) {
2432         # Should we read them from /etc/subuid?
2433         $id_map = [ ['u', '0', '100000', '65536'],
2434                     ['g', '0', '100000', '65536'] ];
2435         $rootuid = $rootgid = 100000;
2436     }
2437
2438     return ($id_map, $rootuid, $rootgid);
2439 }
2440
2441 sub userns_command {
2442     my ($id_map) = @_;
2443     if (@$id_map) {
2444         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2445     }
2446     return [];
2447 }
2448
2449 1;