src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 our $COMMON_TAR_FLAGS = [ '--totals', '--sparse', '--numeric-owner', '--acls',
  30                           '--xattrs',
  31                           '--xattrs-include=user.*',
  32                           '--xattrs-include=security.capability',
  33                           '--warning=no-xattr-write' ];
  34
  35 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  36
  37 my $rootfs_desc = {
  38     volume => {
  39         type => 'string',
  40         default_key => 1,
  41         format_description => 'volume',
  42         description => 'Volume, device or directory to mount into the container.',
  43     },
  44     backup => {
  45         type => 'boolean',
  46         format_description => '[1|0]',
  47         description => 'Whether to include the mountpoint in backups.',
  48         optional => 1,
  49     },
  50     size => {
  51         type => 'string',
  52         format => 'disk-size',
  53         format_description => 'DiskSize',
  54         description => 'Volume size (read only value).',
  55         optional => 1,
  56     },
  57 };
  58
  59 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  60     type => 'string', format => $rootfs_desc,
  61     description => "Use volume as container root.",
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 my $confdesc = {
  72     lock => {
  73         optional => 1,
  74         type => 'string',
  75         description => "Lock/unlock the VM.",
  76         enum => [qw(migrate backup snapshot rollback)],
  77     },
  78     onboot => {
  79         optional => 1,
  80         type => 'boolean',
  81         description => "Specifies whether a VM will be started during system bootup.",
  82         default => 0,
  83     },
  84     startup => get_standard_option('pve-startup-order'),
  85     template => {
  86         optional => 1,
  87         type => 'boolean',
  88         description => "Enable/disable Template.",
  89         default => 0,
  90     },
  91     arch => {
  92         optional => 1,
  93         type => 'string',
  94         enum => ['amd64', 'i386'],
  95         description => "OS architecture type.",
  96         default => 'amd64',
  97     },
  98     ostype => {
  99         optional => 1,
 100         type => 'string',
 101         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
 102         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
 103     },
 104     console => {
 105         optional => 1,
 106         type => 'boolean',
 107         description => "Attach a console device (/dev/console) to the container.",
 108         default => 1,
 109     },
 110     tty => {
 111         optional => 1,
 112         type => 'integer',
 113         description => "Specify the number of tty available to the container",
 114         minimum => 0,
 115         maximum => 6,
 116         default => 2,
 117     },
 118     cpulimit => {
 119         optional => 1,
 120         type => 'number',
 121         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 122         minimum => 0,
 123         maximum => 128,
 124         default => 0,
 125     },
 126     cpuunits => {
 127         optional => 1,
 128         type => 'integer',
 129         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 130         minimum => 0,
 131         maximum => 500000,
 132         default => 1024,
 133     },
 134     memory => {
 135         optional => 1,
 136         type => 'integer',
 137         description => "Amount of RAM for the VM in MB.",
 138         minimum => 16,
 139         default => 512,
 140     },
 141     swap => {
 142         optional => 1,
 143         type => 'integer',
 144         description => "Amount of SWAP for the VM in MB.",
 145         minimum => 0,
 146         default => 512,
 147     },
 148     hostname => {
 149         optional => 1,
 150         description => "Set a host name for the container.",
 151         type => 'string', format => 'dns-name',
 152         maxLength => 255,
 153     },
 154     description => {
 155         optional => 1,
 156         type => 'string',
 157         description => "Container description. Only used on the configuration web interface.",
 158     },
 159     searchdomain => {
 160         optional => 1,
 161         type => 'string', format => 'dns-name-list',
 162         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 163     },
 164     nameserver => {
 165         optional => 1,
 166         type => 'string', format => 'address-list',
 167         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 168     },
 169     rootfs => get_standard_option('pve-ct-rootfs'),
 170     parent => {
 171         optional => 1,
 172         type => 'string', format => 'pve-configid',
 173         maxLength => 40,
 174         description => "Parent snapshot name. This is used internally, and should not be modified.",
 175     },
 176     snaptime => {
 177         optional => 1,
 178         description => "Timestamp for snapshots.",
 179         type => 'integer',
 180         minimum => 0,
 181     },
 182     cmode => {
 183         optional => 1,
 184         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 185         type => 'string',
 186         enum => ['shell', 'console', 'tty'],
 187         default => 'tty',
 188     },
 189     protection => {
 190         optional => 1,
 191         type => 'boolean',
 192         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 193         default => 0,
 194     },
 195     unprivileged => {
 196         optional => 1,
 197         type => 'boolean',
 198         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 199         default => 0,
 200     },
 201 };
 202
 203 my $valid_lxc_conf_keys = {
 204     'lxc.include' => 1,
 205     'lxc.arch' => 1,
 206     'lxc.utsname' => 1,
 207     'lxc.haltsignal' => 1,
 208     'lxc.rebootsignal' => 1,
 209     'lxc.stopsignal' => 1,
 210     'lxc.init_cmd' => 1,
 211     'lxc.network.type' => 1,
 212     'lxc.network.flags' => 1,
 213     'lxc.network.link' => 1,
 214     'lxc.network.mtu' => 1,
 215     'lxc.network.name' => 1,
 216     'lxc.network.hwaddr' => 1,
 217     'lxc.network.ipv4' => 1,
 218     'lxc.network.ipv4.gateway' => 1,
 219     'lxc.network.ipv6' => 1,
 220     'lxc.network.ipv6.gateway' => 1,
 221     'lxc.network.script.up' => 1,
 222     'lxc.network.script.down' => 1,
 223     'lxc.pts' => 1,
 224     'lxc.console.logfile' => 1,
 225     'lxc.console' => 1,
 226     'lxc.tty' => 1,
 227     'lxc.devttydir' => 1,
 228     'lxc.hook.autodev' => 1,
 229     'lxc.autodev' => 1,
 230     'lxc.kmsg' => 1,
 231     'lxc.mount' => 1,
 232     'lxc.mount.entry' => 1,
 233     'lxc.mount.auto' => 1,
 234     'lxc.rootfs' => 1,
 235     'lxc.rootfs.mount' => 1,
 236     'lxc.rootfs.options' => 1,
 237     # lxc.cgroup.*
 238     'lxc.cap.drop' => 1,
 239     'lxc.cap.keep' => 1,
 240     'lxc.aa_profile' => 1,
 241     'lxc.aa_allow_incomplete' => 1,
 242     'lxc.se_context' => 1,
 243     'lxc.seccomp' => 1,
 244     'lxc.id_map' => 1,
 245     'lxc.hook.pre-start' => 1,
 246     'lxc.hook.pre-mount' => 1,
 247     'lxc.hook.mount' => 1,
 248     'lxc.hook.start' => 1,
 249     'lxc.hook.stop' => 1,
 250     'lxc.hook.post-stop' => 1,
 251     'lxc.hook.clone' => 1,
 252     'lxc.hook.destroy' => 1,
 253     'lxc.loglevel' => 1,
 254     'lxc.logfile' => 1,
 255     'lxc.start.auto' => 1,
 256     'lxc.start.delay' => 1,
 257     'lxc.start.order' => 1,
 258     'lxc.group' => 1,
 259     'lxc.environment' => 1,
 260     'lxc.' => 1,
 261     'lxc.' => 1,
 262     'lxc.' => 1,
 263     'lxc.' => 1,
 264 };
 265
 266 my $netconf_desc = {
 267     type => {
 268         type => 'string',
 269         optional => 1,
 270         description => "Network interface type.",
 271         enum => [qw(veth)],
 272     },
 273     name => {
 274         type => 'string',
 275         format_description => 'String',
 276         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 277         pattern => '[-_.\w\d]+',
 278     },
 279     bridge => {
 280         type => 'string',
 281         format_description => 'vmbr<Number>',
 282         description => 'Bridge to attach the network device to.',
 283         pattern => '[-_.\w\d]+',
 284         optional => 1,
 285     },
 286     hwaddr => {
 287         type => 'string',
 288         format_description => 'MAC',
 289         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 290         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 291         optional => 1,
 292     },
 293     mtu => {
 294         type => 'integer',
 295         format_description => 'Number',
 296         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 297         minimum => 64, # minimum ethernet frame is 64 bytes
 298         optional => 1,
 299     },
 300     ip => {
 301         type => 'string',
 302         format => 'pve-ipv4-config',
 303         format_description => 'IPv4Format/CIDR',
 304         description => 'IPv4 address in CIDR format.',
 305         optional => 1,
 306     },
 307     gw => {
 308         type => 'string',
 309         format => 'ipv4',
 310         format_description => 'GatewayIPv4',
 311         description => 'Default gateway for IPv4 traffic.',
 312         optional => 1,
 313     },
 314     ip6 => {
 315         type => 'string',
 316         format => 'pve-ipv6-config',
 317         format_description => 'IPv6Format/CIDR',
 318         description => 'IPv6 address in CIDR format.',
 319         optional => 1,
 320     },
 321     gw6 => {
 322         type => 'string',
 323         format => 'ipv6',
 324         format_description => 'GatewayIPv6',
 325         description => 'Default gateway for IPv6 traffic.',
 326         optional => 1,
 327     },
 328     firewall => {
 329         type => 'boolean',
 330         format_description => '[1|0]',
 331         description => "Controls whether this interface's firewall rules should be used.",
 332         optional => 1,
 333     },
 334     tag => {
 335         type => 'integer',
 336         format_description => 'VlanNo',
 337         minimum => '2',
 338         maximum => '4094',
 339         description => "VLAN tag foro this interface.",
 340         optional => 1,
 341     },
 342 };
 343 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 344
 345 my $MAX_LXC_NETWORKS = 10;
 346 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 347     $confdesc->{"net$i"} = {
 348         optional => 1,
 349         type => 'string', format => $netconf_desc,
 350         description => "Specifies network interfaces for the container.",
 351     };
 352 }
 353
 354 my $mp_desc = {
 355     %$rootfs_desc,
 356     mp => {
 357         type => 'string',
 358         format_description => 'Path',
 359         description => 'Path to the mountpoint as seen from inside the container.',
 360         optional => 1,
 361     },
 362 };
 363 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 364
 365 my $unuseddesc = {
 366     optional => 1,
 367     type => 'string', format => 'pve-volume-id',
 368     description => "Reference to unused volumes.",
 369 };
 370
 371 my $MAX_MOUNT_POINTS = 10;
 372 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 373     $confdesc->{"mp$i"} = {
 374         optional => 1,
 375         type => 'string', format => $mp_desc,
 376         description => "Use volume as container mount point (experimental feature).",
 377         optional => 1,
 378     };
 379 }
 380
 381 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 382 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 383     $confdesc->{"unused$i"} = $unuseddesc;
 384 }
 385
 386 sub write_pct_config {
 387     my ($filename, $conf) = @_;
 388
 389     delete $conf->{snapstate}; # just to be sure
 390
 391     my $generate_raw_config = sub {
 392         my ($conf) = @_;
 393
 394         my $raw = '';
 395
 396         # add description as comment to top of file
 397         my $descr = $conf->{description} || '';
 398         foreach my $cl (split(/\n/, $descr)) {
 399             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 400         }
 401
 402         foreach my $key (sort keys %$conf) {
 403             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 404                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 405             my $value = $conf->{$key};
 406             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 407             $raw .= "$key: $value\n";
 408         }
 409
 410         if (my $lxcconf = $conf->{lxc}) {
 411             foreach my $entry (@$lxcconf) {
 412                 my ($k, $v) = @$entry;
 413                 $raw .= "$k: $v\n";
 414             }
 415         }
 416
 417         return $raw;
 418     };
 419
 420     my $raw = &$generate_raw_config($conf);
 421
 422     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 423         $raw .= "\n[$snapname]\n";
 424         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 425     }
 426
 427     return $raw;
 428 }
 429
 430 sub check_type {
 431     my ($key, $value) = @_;
 432
 433     die "unknown setting '$key'\n" if !$confdesc->{$key};
 434
 435     my $type = $confdesc->{$key}->{type};
 436
 437     if (!defined($value)) {
 438         die "got undefined value\n";
 439     }
 440
 441     if ($value =~ m/[\n\r]/) {
 442         die "property contains a line feed\n";
 443     }
 444
 445     if ($type eq 'boolean') {
 446         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 447         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 448         die "type check ('boolean') failed - got '$value'\n";
 449     } elsif ($type eq 'integer') {
 450         return int($1) if $value =~ m/^(\d+)$/;
 451         die "type check ('integer') failed - got '$value'\n";
 452     } elsif ($type eq 'number') {
 453         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 454         die "type check ('number') failed - got '$value'\n";
 455     } elsif ($type eq 'string') {
 456         if (my $fmt = $confdesc->{$key}->{format}) {
 457             PVE::JSONSchema::check_format($fmt, $value);
 458             return $value;
 459         }
 460         return $value;
 461     } else {
 462         die "internal error"
 463     }
 464 }
 465
 466 sub parse_pct_config {
 467     my ($filename, $raw) = @_;
 468
 469     return undef if !defined($raw);
 470
 471     my $res = {
 472         digest => Digest::SHA::sha1_hex($raw),
 473         snapshots => {},
 474     };
 475
 476     $filename =~ m|/lxc/(\d+).conf$|
 477         || die "got strange filename '$filename'";
 478
 479     my $vmid = $1;
 480
 481     my $conf = $res;
 482     my $descr = '';
 483     my $section = '';
 484
 485     my @lines = split(/\n/, $raw);
 486     foreach my $line (@lines) {
 487         next if $line =~ m/^\s*$/;
 488
 489         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 490             $section = $1;
 491             $conf->{description} = $descr if $descr;
 492             $descr = '';
 493             $conf = $res->{snapshots}->{$section} = {};
 494             next;
 495         }
 496
 497         if ($line =~ m/^\#(.*)\s*$/) {
 498             $descr .= PVE::Tools::decode_text($1) . "\n";
 499             next;
 500         }
 501
 502         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 503             my $key = $1;
 504             my $value = $3;
 505             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 506                 push @{$conf->{lxc}}, [$key, $value];
 507             } else {
 508                 warn "vm $vmid - unable to parse config: $line\n";
 509             }
 510         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 511             $descr .= PVE::Tools::decode_text($2);
 512         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 513             $conf->{snapstate} = $1;
 514         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 515             my $key = $1;
 516             my $value = $2;
 517             eval { $value = check_type($key, $value); };
 518             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 519             $conf->{$key} = $value;
 520         } else {
 521             warn "vm $vmid - unable to parse config: $line\n";
 522         }
 523     }
 524
 525     $conf->{description} = $descr if $descr;
 526
 527     delete $res->{snapstate}; # just to be sure
 528
 529     return $res;
 530 }
 531
 532 sub config_list {
 533     my $vmlist = PVE::Cluster::get_vmlist();
 534     my $res = {};
 535     return $res if !$vmlist || !$vmlist->{ids};
 536     my $ids = $vmlist->{ids};
 537
 538     foreach my $vmid (keys %$ids) {
 539         next if !$vmid; # skip CT0
 540         my $d = $ids->{$vmid};
 541         next if !$d->{node} || $d->{node} ne $nodename;
 542         next if !$d->{type} || $d->{type} ne 'lxc';
 543         $res->{$vmid}->{type} = 'lxc';
 544     }
 545     return $res;
 546 }
 547
 548 sub cfs_config_path {
 549     my ($vmid, $node) = @_;
 550
 551     $node = $nodename if !$node;
 552     return "nodes/$node/lxc/$vmid.conf";
 553 }
 554
 555 sub config_file {
 556     my ($vmid, $node) = @_;
 557
 558     my $cfspath = cfs_config_path($vmid, $node);
 559     return "/etc/pve/$cfspath";
 560 }
 561
 562 sub load_config {
 563     my ($vmid, $node) = @_;
 564
 565     $node = $nodename if !$node;
 566     my $cfspath = cfs_config_path($vmid, $node);
 567
 568     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 569     die "container $vmid does not exists\n" if !defined($conf);
 570
 571     return $conf;
 572 }
 573
 574 sub create_config {
 575     my ($vmid, $conf) = @_;
 576
 577     my $dir = "/etc/pve/nodes/$nodename/lxc";
 578     mkdir $dir;
 579
 580     write_config($vmid, $conf);
 581 }
 582
 583 sub destroy_config {
 584     my ($vmid) = @_;
 585
 586     unlink config_file($vmid, $nodename);
 587 }
 588
 589 sub write_config {
 590     my ($vmid, $conf) = @_;
 591
 592     my $cfspath = cfs_config_path($vmid);
 593
 594     PVE::Cluster::cfs_write_file($cfspath, $conf);
 595 }
 596
 597 # flock: we use one file handle per process, so lock file
 598 # can be called multiple times and succeeds for the same process.
 599
 600 my $lock_handles =  {};
 601 my $lockdir = "/run/lock/lxc";
 602
 603 sub lock_filename {
 604     my ($vmid) = @_;
 605
 606     return "$lockdir/pve-config-${vmid}.lock";
 607 }
 608
 609 sub lock_aquire {
 610     my ($vmid, $timeout) = @_;
 611
 612     $timeout = 10 if !$timeout;
 613     my $mode = LOCK_EX;
 614
 615     my $filename = lock_filename($vmid);
 616
 617     mkdir $lockdir if !-d $lockdir;
 618
 619     my $lock_func = sub {
 620         if (!$lock_handles->{$$}->{$filename}) {
 621             my $fh = new IO::File(">>$filename") ||
 622                 die "can't open file - $!\n";
 623             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 624         }
 625
 626         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 627             print STDERR "trying to aquire lock...";
 628             my $success;
 629             while(1) {
 630                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 631                 # try again on EINTR (see bug #273)
 632                 if ($success || ($! != EINTR)) {
 633                     last;
 634                 }
 635             }
 636             if (!$success) {
 637                 print STDERR " failed\n";
 638                 die "can't aquire lock - $!\n";
 639             }
 640
 641             print STDERR " OK\n";
 642         }
 643
 644         $lock_handles->{$$}->{$filename}->{refcount}++;
 645     };
 646
 647     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 648     my $err = $@;
 649     if ($err) {
 650         die "can't lock file '$filename' - $err";
 651     }
 652 }
 653
 654 sub lock_release {
 655     my ($vmid) = @_;
 656
 657     my $filename = lock_filename($vmid);
 658
 659     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 660         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 661         if ($refcount <= 0) {
 662             $lock_handles->{$$}->{$filename} = undef;
 663             close ($fh);
 664         }
 665     }
 666 }
 667
 668 sub lock_container {
 669     my ($vmid, $timeout, $code, @param) = @_;
 670
 671     my $res;
 672
 673     lock_aquire($vmid, $timeout);
 674     eval { $res = &$code(@param) };
 675     my $err = $@;
 676     lock_release($vmid);
 677
 678     die $err if $err;
 679
 680     return $res;
 681 }
 682
 683 sub option_exists {
 684     my ($name) = @_;
 685
 686     return defined($confdesc->{$name});
 687 }
 688
 689 # add JSON properties for create and set function
 690 sub json_config_properties {
 691     my $prop = shift;
 692
 693     foreach my $opt (keys %$confdesc) {
 694         next if $opt eq 'parent' || $opt eq 'snaptime';
 695         next if $prop->{$opt};
 696         $prop->{$opt} = $confdesc->{$opt};
 697     }
 698
 699     return $prop;
 700 }
 701
 702 sub json_config_properties_no_rootfs {
 703     my $prop = shift;
 704
 705     foreach my $opt (keys %$confdesc) {
 706         next if $prop->{$opt};
 707         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 708         $prop->{$opt} = $confdesc->{$opt};
 709     }
 710
 711     return $prop;
 712 }
 713
 714 # container status helpers
 715
 716 sub list_active_containers {
 717
 718     my $filename = "/proc/net/unix";
 719
 720     # similar test is used by lcxcontainers.c: list_active_containers
 721     my $res = {};
 722
 723     my $fh = IO::File->new ($filename, "r");
 724     return $res if !$fh;
 725
 726     while (defined(my $line = <$fh>)) {
 727         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 728             my $path = $1;
 729             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 730                 $res->{$1} = 1;
 731             }
 732         }
 733     }
 734
 735     close($fh);
 736
 737     return $res;
 738 }
 739
 740 # warning: this is slow
 741 sub check_running {
 742     my ($vmid) = @_;
 743
 744     my $active_hash = list_active_containers();
 745
 746     return 1 if defined($active_hash->{$vmid});
 747
 748     return undef;
 749 }
 750
 751 sub get_container_disk_usage {
 752     my ($vmid, $pid) = @_;
 753
 754     return PVE::Tools::df("/proc/$pid/root/", 1);
 755 }
 756
 757 my $last_proc_vmid_stat;
 758
 759 my $parse_cpuacct_stat = sub {
 760     my ($vmid) = @_;
 761
 762     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 763
 764     my $stat = {};
 765
 766     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 767
 768         $stat->{utime} = $1;
 769         $stat->{stime} = $2;
 770
 771     }
 772
 773     return $stat;
 774 };
 775
 776 sub vmstatus {
 777     my ($opt_vmid) = @_;
 778
 779     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 780
 781     my $active_hash = list_active_containers();
 782
 783     my $cpucount = $cpuinfo->{cpus} || 1;
 784
 785     my $cdtime = gettimeofday;
 786
 787     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 788
 789     foreach my $vmid (keys %$list) {
 790         my $d = $list->{$vmid};
 791
 792         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 793         warn $@ if $@; # ignore errors (consider them stopped)
 794
 795         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 796
 797         my $cfspath = cfs_config_path($vmid);
 798         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 799
 800         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 801         $d->{name} =~ s/[\s]//g;
 802
 803         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 804
 805         if ($d->{pid}) {
 806             my $res = get_container_disk_usage($vmid, $d->{pid});
 807             $d->{disk} = $res->{used};
 808             $d->{maxdisk} = $res->{total};
 809         } else {
 810             $d->{disk} = 0;
 811             # use 4GB by default ??
 812             if (my $rootfs = $conf->{rootfs}) {
 813                 my $rootinfo = parse_ct_mountpoint($rootfs);
 814                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 815             } else {
 816                 $d->{maxdisk} = 4*1024*1024*1024;
 817             }
 818         }
 819
 820         $d->{mem} = 0;
 821         $d->{swap} = 0;
 822         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 823         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 824
 825         $d->{uptime} = 0;
 826         $d->{cpu} = 0;
 827
 828         $d->{netout} = 0;
 829         $d->{netin} = 0;
 830
 831         $d->{diskread} = 0;
 832         $d->{diskwrite} = 0;
 833
 834         $d->{template} = is_template($conf);
 835     }
 836
 837     foreach my $vmid (keys %$list) {
 838         my $d = $list->{$vmid};
 839         my $pid = $d->{pid};
 840
 841         next if !$pid; # skip stopped CTs
 842
 843         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 844         $d->{uptime} = time - $ctime; # the method lxcfs uses
 845
 846         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 847         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 848
 849         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 850         my @bytes = split(/\n/, $blkio_bytes);
 851         foreach my $byte (@bytes) {
 852             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 853                 $d->{diskread} = $2 if $key eq 'Read';
 854                 $d->{diskwrite} = $2 if $key eq 'Write';
 855             }
 856         }
 857
 858         my $pstat = &$parse_cpuacct_stat($vmid);
 859
 860         my $used = $pstat->{utime} + $pstat->{stime};
 861
 862         my $old = $last_proc_vmid_stat->{$vmid};
 863         if (!$old) {
 864             $last_proc_vmid_stat->{$vmid} = {
 865                 time => $cdtime,
 866                 used => $used,
 867                 cpu => 0,
 868             };
 869             next;
 870         }
 871
 872         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 873
 874         if ($dtime > 1000) {
 875             my $dutime = $used -  $old->{used};
 876
 877             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 878             $last_proc_vmid_stat->{$vmid} = {
 879                 time => $cdtime,
 880                 used => $used,
 881                 cpu => $d->{cpu},
 882             };
 883         } else {
 884             $d->{cpu} = $old->{cpu};
 885         }
 886     }
 887
 888     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 889
 890     foreach my $dev (keys %$netdev) {
 891         next if $dev !~ m/^veth([1-9]\d*)i/;
 892         my $vmid = $1;
 893         my $d = $list->{$vmid};
 894
 895         next if !$d;
 896
 897         $d->{netout} += $netdev->{$dev}->{receive};
 898         $d->{netin} += $netdev->{$dev}->{transmit};
 899
 900     }
 901
 902     return $list;
 903 }
 904
 905 sub parse_ct_mountpoint {
 906     my ($data, $noerr) = @_;
 907
 908     $data //= '';
 909
 910     my $res;
 911     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 912     if ($@) {
 913         return undef if $noerr;
 914         die $@;
 915     }
 916
 917     if (defined(my $size = $res->{size})) {
 918         $size = PVE::JSONSchema::parse_size($size);
 919         if (!defined($size)) {
 920             return undef if $noerr;
 921             die "invalid size: $size\n";
 922         }
 923         $res->{size} = $size;
 924     }
 925
 926     return $res;
 927 }
 928
 929 sub print_ct_mountpoint {
 930     my ($info, $nomp) = @_;
 931     my $skip = $nomp ? ['mp'] : [];
 932     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 933 }
 934
 935 sub print_lxc_network {
 936     my $net = shift;
 937     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 938 }
 939
 940 sub parse_lxc_network {
 941     my ($data) = @_;
 942
 943     my $res = {};
 944
 945     return $res if !$data;
 946
 947     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 948
 949     $res->{type} = 'veth';
 950     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 951
 952     return $res;
 953 }
 954
 955 sub read_cgroup_value {
 956     my ($group, $vmid, $name, $full) = @_;
 957
 958     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 959
 960     return PVE::Tools::file_get_contents($path) if $full;
 961
 962     return PVE::Tools::file_read_firstline($path);
 963 }
 964
 965 sub write_cgroup_value {
 966    my ($group, $vmid, $name, $value) = @_;
 967
 968    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 969    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 970
 971 }
 972
 973 sub find_lxc_console_pids {
 974
 975     my $res = {};
 976
 977     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 978         my ($pid) = @_;
 979
 980         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 981         return if !$cmdline;
 982
 983         my @args = split(/\0/, $cmdline);
 984
 985         # serach for lxc-console -n <vmid>
 986         return if scalar(@args) != 3;
 987         return if $args[1] ne '-n';
 988         return if $args[2] !~ m/^\d+$/;
 989         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 990
 991         my $vmid = $args[2];
 992
 993         push @{$res->{$vmid}}, $pid;
 994     });
 995
 996     return $res;
 997 }
 998
 999 sub find_lxc_pid {
1000     my ($vmid) = @_;
1001
1002     my $pid = undef;
1003     my $parser = sub {
1004         my $line = shift;
1005         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
1006     };
1007     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
1008
1009     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
1010
1011     return $pid;
1012 }
1013
1014 # Note: we cannot use Net:IP, because that only allows strict
1015 # CIDR networks
1016 sub parse_ipv4_cidr {
1017     my ($cidr, $noerr) = @_;
1018
1019     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1020         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1021     }
1022
1023     return undef if $noerr;
1024
1025     die "unable to parse ipv4 address/mask\n";
1026 }
1027
1028 sub check_lock {
1029     my ($conf) = @_;
1030
1031     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1032 }
1033
1034 sub check_protection {
1035     my ($vm_conf, $err_msg) = @_;
1036
1037     if ($vm_conf->{protection}) {
1038         die "$err_msg - protection mode enabled\n";
1039     }
1040 }
1041
1042 sub update_lxc_config {
1043     my ($storage_cfg, $vmid, $conf) = @_;
1044
1045     my $dir = "/var/lib/lxc/$vmid";
1046
1047     if ($conf->{template}) {
1048
1049         unlink "$dir/config";
1050
1051         return;
1052     }
1053
1054     my $raw = '';
1055
1056     die "missing 'arch' - internal error" if !$conf->{arch};
1057     $raw .= "lxc.arch = $conf->{arch}\n";
1058
1059     my $unprivileged = $conf->{unprivileged};
1060     my $custom_idmap = grep { $_->[0] eq 'lxc.id_map' } @{$conf->{lxc}};
1061
1062     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1063     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1064         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1065         if ($unprivileged || $custom_idmap) {
1066             $raw .= "lxc.include = /usr/share/lxc/config/$ostype.userns.conf\n"
1067         }
1068     } else {
1069         die "implement me";
1070     }
1071
1072     # Should we read them from /etc/subuid?
1073     if ($unprivileged && !$custom_idmap) {
1074         $raw .= "lxc.id_map = u 0 100000 65536\n";
1075         $raw .= "lxc.id_map = g 0 100000 65536\n";
1076     }
1077
1078     if (!has_dev_console($conf)) {
1079         $raw .= "lxc.console = none\n";
1080         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1081     }
1082
1083     my $ttycount = get_tty_count($conf);
1084     $raw .= "lxc.tty = $ttycount\n";
1085
1086     # some init scripts expects a linux terminal (turnkey).
1087     $raw .= "lxc.environment = TERM=linux\n";
1088
1089     my $utsname = $conf->{hostname} || "CT$vmid";
1090     $raw .= "lxc.utsname = $utsname\n";
1091
1092     my $memory = $conf->{memory} || 512;
1093     my $swap = $conf->{swap} // 0;
1094
1095     my $lxcmem = int($memory*1024*1024);
1096     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1097
1098     my $lxcswap = int(($memory + $swap)*1024*1024);
1099     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1100
1101     if (my $cpulimit = $conf->{cpulimit}) {
1102         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1103         my $value = int(100000*$cpulimit);
1104         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1105     }
1106
1107     my $shares = $conf->{cpuunits} || 1024;
1108     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1109
1110     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1111     $mountpoint->{mp} = '/';
1112
1113     $raw .= "lxc.rootfs = $dir/rootfs\n";
1114
1115     my $netcount = 0;
1116     foreach my $k (keys %$conf) {
1117         next if $k !~ m/^net(\d+)$/;
1118         my $ind = $1;
1119         my $d = parse_lxc_network($conf->{$k});
1120         $netcount++;
1121         $raw .= "lxc.network.type = veth\n";
1122         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1123         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1124         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1125         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1126     }
1127
1128     if (my $lxcconf = $conf->{lxc}) {
1129         foreach my $entry (@$lxcconf) {
1130             my ($k, $v) = @$entry;
1131             $netcount++ if $k eq 'lxc.network.type';
1132             $raw .= "$k = $v\n";
1133         }
1134     }
1135
1136     $raw .= "lxc.network.type = empty\n" if !$netcount;
1137
1138     File::Path::mkpath("$dir/rootfs");
1139
1140     PVE::Tools::file_set_contents("$dir/config", $raw);
1141 }
1142
1143 # verify and cleanup nameserver list (replace \0 with ' ')
1144 sub verify_nameserver_list {
1145     my ($nameserver_list) = @_;
1146
1147     my @list = ();
1148     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1149         PVE::JSONSchema::pve_verify_ip($server);
1150         push @list, $server;
1151     }
1152
1153     return join(' ', @list);
1154 }
1155
1156 sub verify_searchdomain_list {
1157     my ($searchdomain_list) = @_;
1158
1159     my @list = ();
1160     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1161         # todo: should we add checks for valid dns domains?
1162         push @list, $server;
1163     }
1164
1165     return join(' ', @list);
1166 }
1167
1168 sub add_unused_volume {
1169     my ($config, $volid) = @_;
1170
1171     # skip bind mounts and block devices
1172     return if $volid =~ m|^/|;
1173
1174     my $key;
1175     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1176         my $test = "unused$ind";
1177         if (my $vid = $config->{$test}) {
1178             return if $vid eq $volid; # do not add duplicates
1179         } else {
1180             $key = $test;
1181         }
1182     }
1183
1184     die "To many unused volume - please delete them first.\n" if !$key;
1185
1186     $config->{$key} = $volid;
1187
1188     return $key;
1189 }
1190
1191 sub update_pct_config {
1192     my ($vmid, $conf, $running, $param, $delete) = @_;
1193
1194     my @nohotplug;
1195
1196     my $new_disks = 0;
1197     my @deleted_volumes;
1198
1199     my $rootdir;
1200     if ($running) {
1201         my $pid = find_lxc_pid($vmid);
1202         $rootdir = "/proc/$pid/root";
1203     }
1204
1205     my $hotplug_error = sub {
1206         if ($running) {
1207             push @nohotplug, @_;
1208             return 1;
1209         } else {
1210             return 0;
1211         }
1212     };
1213
1214     if (defined($delete)) {
1215         foreach my $opt (@$delete) {
1216             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1217                 die "unable to delete required option '$opt'\n";
1218             } elsif ($opt eq 'swap') {
1219                 delete $conf->{$opt};
1220                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1221             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1222                 delete $conf->{$opt};
1223             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1224                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1225                 next if $hotplug_error->($opt);
1226                 delete $conf->{$opt};
1227             } elsif ($opt =~ m/^net(\d)$/) {
1228                 delete $conf->{$opt};
1229                 next if !$running;
1230                 my $netid = $1;
1231                 PVE::Network::veth_delete("veth${vmid}i$netid");
1232             } elsif ($opt eq 'protection') {
1233                 delete $conf->{$opt};
1234             } elsif ($opt =~ m/^unused(\d+)$/) {
1235                 next if $hotplug_error->($opt);
1236                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1237                 push @deleted_volumes, $conf->{$opt};
1238                 delete $conf->{$opt};
1239             } elsif ($opt =~ m/^mp(\d+)$/) {
1240                 next if $hotplug_error->($opt);
1241                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1242                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1243                 add_unused_volume($conf, $mountpoint->{volume});
1244                 delete $conf->{$opt};
1245             } elsif ($opt eq 'unprivileged') {
1246                 die "unable to delete read-only option: '$opt'\n";
1247             } else {
1248                 die "implement me"
1249             }
1250             write_config($vmid, $conf) if $running;
1251         }
1252     }
1253
1254     # There's no separate swap size to configure, there's memory and "total"
1255     # memory (iow. memory+swap). This means we have to change them together.
1256     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1257     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1258     if (defined($wanted_memory) || defined($wanted_swap)) {
1259
1260         $wanted_memory //= ($conf->{memory} || 512);
1261         $wanted_swap //=  ($conf->{swap} || 0);
1262
1263         my $total = $wanted_memory + $wanted_swap;
1264         if ($running) {
1265             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1266             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1267         }
1268         $conf->{memory} = $wanted_memory;
1269         $conf->{swap} = $wanted_swap;
1270
1271         write_config($vmid, $conf) if $running;
1272     }
1273
1274     foreach my $opt (keys %$param) {
1275         my $value = $param->{$opt};
1276         if ($opt eq 'hostname') {
1277             $conf->{$opt} = $value;
1278         } elsif ($opt eq 'onboot') {
1279             $conf->{$opt} = $value ? 1 : 0;
1280         } elsif ($opt eq 'startup') {
1281             $conf->{$opt} = $value;
1282         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1283             next if $hotplug_error->($opt);
1284             $conf->{$opt} = $value;
1285         } elsif ($opt eq 'nameserver') {
1286             next if $hotplug_error->($opt);
1287             my $list = verify_nameserver_list($value);
1288             $conf->{$opt} = $list;
1289         } elsif ($opt eq 'searchdomain') {
1290             next if $hotplug_error->($opt);
1291             my $list = verify_searchdomain_list($value);
1292             $conf->{$opt} = $list;
1293         } elsif ($opt eq 'cpulimit') {
1294             next if $hotplug_error->($opt); # FIXME: hotplug
1295             $conf->{$opt} = $value;
1296         } elsif ($opt eq 'cpuunits') {
1297             $conf->{$opt} = $value;
1298             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1299         } elsif ($opt eq 'description') {
1300             $conf->{$opt} = PVE::Tools::encode_text($value);
1301         } elsif ($opt =~ m/^net(\d+)$/) {
1302             my $netid = $1;
1303             my $net = parse_lxc_network($value);
1304             if (!$running) {
1305                 $conf->{$opt} = print_lxc_network($net);
1306             } else {
1307                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1308             }
1309         } elsif ($opt eq 'protection') {
1310             $conf->{$opt} = $value ? 1 : 0;
1311         } elsif ($opt =~ m/^mp(\d+)$/) {
1312             next if $hotplug_error->($opt);
1313             check_protection($conf, "can't update CT $vmid drive '$opt'");
1314             $conf->{$opt} = $value;
1315             $new_disks = 1;
1316         } elsif ($opt eq 'rootfs') {
1317             check_protection($conf, "can't update CT $vmid drive '$opt'");
1318             die "implement me: $opt";
1319         } elsif ($opt eq 'unprivileged') {
1320             die "unable to modify read-only option: '$opt'\n";
1321         } else {
1322             die "implement me: $opt";
1323         }
1324         write_config($vmid, $conf) if $running;
1325     }
1326
1327     if (@deleted_volumes) {
1328         my $storage_cfg = PVE::Storage::config();
1329         foreach my $volume (@deleted_volumes) {
1330             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1331         }
1332     }
1333
1334     if ($new_disks) {
1335         my $storage_cfg = PVE::Storage::config();
1336         create_disks($storage_cfg, $vmid, $conf, $conf);
1337     }
1338
1339     # This should be the last thing we do here
1340     if ($running && scalar(@nohotplug)) {
1341         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1342     }
1343 }
1344
1345 sub has_dev_console {
1346     my ($conf) = @_;
1347
1348     return !(defined($conf->{console}) && !$conf->{console});
1349 }
1350
1351 sub get_tty_count {
1352     my ($conf) = @_;
1353
1354     return $conf->{tty} // $confdesc->{tty}->{default};
1355 }
1356
1357 sub get_cmode {
1358     my ($conf) = @_;
1359
1360     return $conf->{cmode} // $confdesc->{cmode}->{default};
1361 }
1362
1363 sub get_console_command {
1364     my ($vmid, $conf) = @_;
1365
1366     my $cmode = get_cmode($conf);
1367
1368     if ($cmode eq 'console') {
1369         return ['lxc-console', '-n',  $vmid, '-t', 0];
1370     } elsif ($cmode eq 'tty') {
1371         return ['lxc-console', '-n',  $vmid];
1372     } elsif ($cmode eq 'shell') {
1373         return ['lxc-attach', '--clear-env', '-n', $vmid];
1374     } else {
1375         die "internal error";
1376     }
1377 }
1378
1379 sub get_primary_ips {
1380     my ($conf) = @_;
1381
1382     # return data from net0
1383
1384     return undef if !defined($conf->{net0});
1385     my $net = parse_lxc_network($conf->{net0});
1386
1387     my $ipv4 = $net->{ip};
1388     if ($ipv4) {
1389         if ($ipv4 =~ /^(dhcp|manual)$/) {
1390             $ipv4 = undef
1391         } else {
1392             $ipv4 =~ s!/\d+$!!;
1393         }
1394     }
1395     my $ipv6 = $net->{ip6};
1396     if ($ipv6) {
1397         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1398             $ipv6 = undef;
1399         } else {
1400             $ipv6 =~ s!/\d+$!!;
1401         }
1402     }
1403
1404     return ($ipv4, $ipv6);
1405 }
1406
1407 sub delete_mountpoint_volume {
1408     my ($storage_cfg, $vmid, $volume) = @_;
1409
1410     # skip bind mounts and block devices
1411     if ($volume =~ m|^/|) {
1412             return;
1413     }
1414
1415     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1416     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1417 }
1418
1419 sub destroy_lxc_container {
1420     my ($storage_cfg, $vmid, $conf) = @_;
1421
1422     foreach_mountpoint($conf, sub {
1423         my ($ms, $mountpoint) = @_;
1424         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1425     });
1426
1427     rmdir "/var/lib/lxc/$vmid/rootfs";
1428     unlink "/var/lib/lxc/$vmid/config";
1429     rmdir "/var/lib/lxc/$vmid";
1430     destroy_config($vmid);
1431
1432     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1433     #PVE::Tools::run_command($cmd);
1434 }
1435
1436 sub vm_stop_cleanup {
1437     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1438
1439     eval {
1440         if (!$keepActive) {
1441
1442             my $vollist = get_vm_volumes($conf);
1443             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1444         }
1445     };
1446     warn $@ if $@; # avoid errors - just warn
1447 }
1448
1449 my $safe_num_ne = sub {
1450     my ($a, $b) = @_;
1451
1452     return 0 if !defined($a) && !defined($b);
1453     return 1 if !defined($a);
1454     return 1 if !defined($b);
1455
1456     return $a != $b;
1457 };
1458
1459 my $safe_string_ne = sub {
1460     my ($a, $b) = @_;
1461
1462     return 0 if !defined($a) && !defined($b);
1463     return 1 if !defined($a);
1464     return 1 if !defined($b);
1465
1466     return $a ne $b;
1467 };
1468
1469 sub update_net {
1470     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1471
1472     if ($newnet->{type} ne 'veth') {
1473         # for when there are physical interfaces
1474         die "cannot update interface of type $newnet->{type}";
1475     }
1476
1477     my $veth = "veth${vmid}i${netid}";
1478     my $eth = $newnet->{name};
1479
1480     if (my $oldnetcfg = $conf->{$opt}) {
1481         my $oldnet = parse_lxc_network($oldnetcfg);
1482
1483         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1484             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1485
1486             PVE::Network::veth_delete($veth);
1487             delete $conf->{$opt};
1488             write_config($vmid, $conf);
1489
1490             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1491
1492         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1493                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1494                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1495
1496                 if ($oldnet->{bridge}) {
1497                     PVE::Network::tap_unplug($veth);
1498                     foreach (qw(bridge tag firewall)) {
1499                         delete $oldnet->{$_};
1500                     }
1501                     $conf->{$opt} = print_lxc_network($oldnet);
1502                     write_config($vmid, $conf);
1503                 }
1504
1505                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1506                 foreach (qw(bridge tag firewall)) {
1507                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1508                 }
1509                 $conf->{$opt} = print_lxc_network($oldnet);
1510                 write_config($vmid, $conf);
1511         }
1512     } else {
1513         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1514     }
1515
1516     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1517 }
1518
1519 sub hotplug_net {
1520     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1521
1522     my $veth = "veth${vmid}i${netid}";
1523     my $vethpeer = $veth . "p";
1524     my $eth = $newnet->{name};
1525
1526     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1527     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1528
1529     # attach peer in container
1530     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1531     PVE::Tools::run_command($cmd);
1532
1533     # link up peer in container
1534     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1535     PVE::Tools::run_command($cmd);
1536
1537     my $done = { type => 'veth' };
1538     foreach (qw(bridge tag firewall hwaddr name)) {
1539         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1540     }
1541     $conf->{$opt} = print_lxc_network($done);
1542
1543     write_config($vmid, $conf);
1544 }
1545
1546 sub update_ipconfig {
1547     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1548
1549     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1550
1551     my $optdata = parse_lxc_network($conf->{$opt});
1552     my $deleted = [];
1553     my $added = [];
1554     my $nscmd = sub {
1555         my $cmdargs = shift;
1556         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1557     };
1558     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1559
1560     my $change_ip_config = sub {
1561         my ($ipversion) = @_;
1562
1563         my $family_opt = "-$ipversion";
1564         my $suffix = $ipversion == 4 ? '' : $ipversion;
1565         my $gw= "gw$suffix";
1566         my $ip= "ip$suffix";
1567
1568         my $newip = $newnet->{$ip};
1569         my $newgw = $newnet->{$gw};
1570         my $oldip = $optdata->{$ip};
1571
1572         my $change_ip = &$safe_string_ne($oldip, $newip);
1573         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1574
1575         return if !$change_ip && !$change_gw;
1576
1577         # step 1: add new IP, if this fails we cancel
1578         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1579         if ($change_ip && $is_real_ip) {
1580             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1581             if (my $err = $@) {
1582                 warn $err;
1583                 return;
1584             }
1585         }
1586
1587         # step 2: replace gateway
1588         #   If this fails we delete the added IP and cancel.
1589         #   If it succeeds we save the config and delete the old IP, ignoring
1590         #   errors. The config is then saved.
1591         # Note: 'ip route replace' can add
1592         if ($change_gw) {
1593             if ($newgw) {
1594                 eval {
1595                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1596                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1597                     }
1598                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1599                 };
1600                 if (my $err = $@) {
1601                     warn $err;
1602                     # the route was not replaced, the old IP is still available
1603                     # rollback (delete new IP) and cancel
1604                     if ($change_ip) {
1605                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1606                         warn $@ if $@; # no need to die here
1607                     }
1608                     return;
1609                 }
1610             } else {
1611                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1612                 # if the route was not deleted, the guest might have deleted it manually
1613                 # warn and continue
1614                 warn $@ if $@;
1615             }
1616         }
1617
1618         # from this point on we save the configuration
1619         # step 3: delete old IP ignoring errors
1620         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1621             # We need to enable promote_secondaries, otherwise our newly added
1622             # address will be removed along with the old one.
1623             my $promote = 0;
1624             eval {
1625                 if ($ipversion == 4) {
1626                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1627                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1628                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1629                 }
1630                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1631             };
1632             warn $@ if $@; # no need to die here
1633
1634             if ($ipversion == 4) {
1635                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1636             }
1637         }
1638
1639         foreach my $property ($ip, $gw) {
1640             if ($newnet->{$property}) {
1641                 $optdata->{$property} = $newnet->{$property};
1642             } else {
1643                 delete $optdata->{$property};
1644             }
1645         }
1646         $conf->{$opt} = print_lxc_network($optdata);
1647         write_config($vmid, $conf);
1648         $lxc_setup->setup_network($conf);
1649     };
1650
1651     &$change_ip_config(4);
1652     &$change_ip_config(6);
1653
1654 }
1655
1656 # Internal snapshots
1657
1658 # NOTE: Snapshot create/delete involves several non-atomic
1659 # action, and can take a long time.
1660 # So we try to avoid locking the file and use 'lock' variable
1661 # inside the config file instead.
1662
1663 my $snapshot_copy_config = sub {
1664     my ($source, $dest) = @_;
1665
1666     foreach my $k (keys %$source) {
1667         next if $k eq 'snapshots';
1668         next if $k eq 'snapstate';
1669         next if $k eq 'snaptime';
1670         next if $k eq 'vmstate';
1671         next if $k eq 'lock';
1672         next if $k eq 'digest';
1673         next if $k eq 'description';
1674
1675         $dest->{$k} = $source->{$k};
1676     }
1677 };
1678
1679 my $snapshot_prepare = sub {
1680     my ($vmid, $snapname, $comment) = @_;
1681
1682     my $snap;
1683
1684     my $updatefn =  sub {
1685
1686         my $conf = load_config($vmid);
1687
1688         die "you can't take a snapshot if it's a template\n"
1689             if is_template($conf);
1690
1691         check_lock($conf);
1692
1693         $conf->{lock} = 'snapshot';
1694
1695         die "snapshot name '$snapname' already used\n"
1696             if defined($conf->{snapshots}->{$snapname});
1697
1698         my $storecfg = PVE::Storage::config();
1699         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1700
1701         $snap = $conf->{snapshots}->{$snapname} = {};
1702
1703         &$snapshot_copy_config($conf, $snap);
1704
1705         $snap->{'snapstate'} = "prepare";
1706         $snap->{'snaptime'} = time();
1707         $snap->{'description'} = $comment if $comment;
1708         $conf->{snapshots}->{$snapname} = $snap;
1709
1710         write_config($vmid, $conf);
1711     };
1712
1713     lock_container($vmid, 10, $updatefn);
1714
1715     return $snap;
1716 };
1717
1718 my $snapshot_commit = sub {
1719     my ($vmid, $snapname) = @_;
1720
1721     my $updatefn = sub {
1722
1723         my $conf = load_config($vmid);
1724
1725         die "missing snapshot lock\n"
1726             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1727
1728         die "snapshot '$snapname' does not exist\n"
1729             if !defined($conf->{snapshots}->{$snapname});
1730
1731         die "wrong snapshot state\n"
1732             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1733                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1734
1735         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1736         delete $conf->{lock};
1737         $conf->{parent} = $snapname;
1738
1739         write_config($vmid, $conf);
1740     };
1741
1742     lock_container($vmid, 10 ,$updatefn);
1743 };
1744
1745 sub has_feature {
1746     my ($feature, $conf, $storecfg, $snapname) = @_;
1747
1748     my $err;
1749
1750     foreach_mountpoint($conf, sub {
1751         my ($ms, $mountpoint) = @_;
1752
1753         return if $err; # skip further test
1754
1755         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1756
1757         # TODO: implement support for mountpoints
1758         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1759             if $ms ne 'rootfs';
1760     });
1761
1762     return $err ? 0 : 1;
1763 }
1764
1765 sub snapshot_create {
1766     my ($vmid, $snapname, $comment) = @_;
1767
1768     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1769
1770     my $conf = load_config($vmid);
1771
1772     my $running = check_running($vmid);
1773     eval {
1774         if ($running) {
1775             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1776             PVE::Tools::run_command(['/bin/sync']);
1777         };
1778
1779         my $storecfg = PVE::Storage::config();
1780         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1781         my $volid = $rootinfo->{volume};
1782
1783         if ($running) {
1784             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1785         };
1786
1787         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1788         &$snapshot_commit($vmid, $snapname);
1789     };
1790     if(my $err = $@) {
1791         snapshot_delete($vmid, $snapname, 1);
1792         die "$err\n";
1793     }
1794 }
1795
1796 sub snapshot_delete {
1797     my ($vmid, $snapname, $force) = @_;
1798
1799     my $snap;
1800
1801     my $conf;
1802
1803     my $updatefn =  sub {
1804
1805         $conf = load_config($vmid);
1806
1807         die "you can't delete a snapshot if vm is a template\n"
1808             if is_template($conf);
1809
1810         $snap = $conf->{snapshots}->{$snapname};
1811
1812         check_lock($conf);
1813
1814         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1815
1816         $snap->{snapstate} = 'delete';
1817
1818         write_config($vmid, $conf);
1819     };
1820
1821     lock_container($vmid, 10, $updatefn);
1822
1823     my $storecfg = PVE::Storage::config();
1824
1825     my $del_snap =  sub {
1826
1827         check_lock($conf);
1828
1829         if ($conf->{parent} eq $snapname) {
1830             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1831                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1832             } else {
1833                 delete $conf->{parent};
1834             }
1835         }
1836
1837         delete $conf->{snapshots}->{$snapname};
1838
1839         write_config($vmid, $conf);
1840     };
1841
1842     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1843     my $rootinfo = parse_ct_mountpoint($rootfs);
1844     my $volid = $rootinfo->{volume};
1845
1846     eval {
1847         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1848     };
1849     my $err = $@;
1850
1851     if(!$err || ($err && $force)) {
1852         lock_container($vmid, 10, $del_snap);
1853         if ($err) {
1854             die "Can't delete snapshot: $vmid $snapname $err\n";
1855         }
1856     }
1857 }
1858
1859 sub snapshot_rollback {
1860     my ($vmid, $snapname) = @_;
1861
1862     my $storecfg = PVE::Storage::config();
1863
1864     my $conf = load_config($vmid);
1865
1866     die "you can't rollback if vm is a template\n" if is_template($conf);
1867
1868     my $snap = $conf->{snapshots}->{$snapname};
1869
1870     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1871
1872     my $rootfs = $snap->{rootfs};
1873     my $rootinfo = parse_ct_mountpoint($rootfs);
1874     my $volid = $rootinfo->{volume};
1875
1876     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1877
1878     my $updatefn = sub {
1879
1880         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1881             if $snap->{snapstate};
1882
1883         check_lock($conf);
1884
1885         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1886
1887         die "unable to rollback vm $vmid: vm is running\n"
1888             if check_running($vmid);
1889
1890         $conf->{lock} = 'rollback';
1891
1892         my $forcemachine;
1893
1894         # copy snapshot config to current config
1895
1896         my $tmp_conf = $conf;
1897         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1898         $conf->{snapshots} = $tmp_conf->{snapshots};
1899         delete $conf->{snaptime};
1900         delete $conf->{snapname};
1901         $conf->{parent} = $snapname;
1902
1903         write_config($vmid, $conf);
1904     };
1905
1906     my $unlockfn = sub {
1907         delete $conf->{lock};
1908         write_config($vmid, $conf);
1909     };
1910
1911     lock_container($vmid, 10, $updatefn);
1912
1913     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1914
1915     lock_container($vmid, 5, $unlockfn);
1916 }
1917
1918 sub template_create {
1919     my ($vmid, $conf) = @_;
1920
1921     my $storecfg = PVE::Storage::config();
1922
1923     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1924     my $volid = $rootinfo->{volume};
1925
1926     die "Template feature is not available for '$volid'\n"
1927         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1928
1929     PVE::Storage::activate_volumes($storecfg, [$volid]);
1930
1931     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1932     $rootinfo->{volume} = $template_volid;
1933     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1934
1935     write_config($vmid, $conf);
1936 }
1937
1938 sub is_template {
1939     my ($conf) = @_;
1940
1941     return 1 if defined $conf->{template} && $conf->{template} == 1;
1942 }
1943
1944 sub mountpoint_names {
1945     my ($reverse) = @_;
1946
1947     my @names = ('rootfs');
1948
1949     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1950         push @names, "mp$i";
1951     }
1952
1953     return $reverse ? reverse @names : @names;
1954 }
1955
1956 # The container might have *different* symlinks than the host. realpath/abs_path
1957 # use the actual filesystem to resolve links.
1958 sub sanitize_mountpoint {
1959     my ($mp) = @_;
1960     $mp = '/' . $mp; # we always start with a slash
1961     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1962     $mp =~ s@/\./@@g; # collapse /./
1963     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1964     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1965     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1966     return $mp;
1967 }
1968
1969 sub foreach_mountpoint_full {
1970     my ($conf, $reverse, $func) = @_;
1971
1972     foreach my $key (mountpoint_names($reverse)) {
1973         my $value = $conf->{$key};
1974         next if !defined($value);
1975         my $mountpoint = parse_ct_mountpoint($value, 1);
1976         next if !defined($mountpoint);
1977
1978         # just to be sure: rootfs is /
1979         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1980         $mountpoint->{mp} = sanitize_mountpoint($path);
1981
1982         $path = $mountpoint->{volume};
1983         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1984
1985         &$func($key, $mountpoint);
1986     }
1987 }
1988
1989 sub foreach_mountpoint {
1990     my ($conf, $func) = @_;
1991
1992     foreach_mountpoint_full($conf, 0, $func);
1993 }
1994
1995 sub foreach_mountpoint_reverse {
1996     my ($conf, $func) = @_;
1997
1998     foreach_mountpoint_full($conf, 1, $func);
1999 }
2000
2001 sub check_ct_modify_config_perm {
2002     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
2003
2004     return 1 if $authuser ne 'root@pam';
2005
2006     foreach my $opt (@$key_list) {
2007
2008         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
2009             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
2010         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
2011             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
2012         } elsif ($opt eq 'memory' || $opt eq 'swap') {
2013             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
2014         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
2015                  $opt eq 'searchdomain' || $opt eq 'hostname') {
2016             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
2017         } else {
2018             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
2019         }
2020     }
2021
2022     return 1;
2023 }
2024
2025 sub umount_all {
2026     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2027
2028     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2029     my $volid_list = get_vm_volumes($conf);
2030
2031     foreach_mountpoint_reverse($conf, sub {
2032         my ($ms, $mountpoint) = @_;
2033
2034         my $volid = $mountpoint->{volume};
2035         my $mount = $mountpoint->{mp};
2036
2037         return if !$volid || !$mount;
2038
2039         my $mount_path = "$rootdir/$mount";
2040         $mount_path =~ s!/+!/!g;
2041
2042         return if !PVE::ProcFSTools::is_mounted($mount_path);
2043
2044         eval {
2045             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2046         };
2047         if (my $err = $@) {
2048             if ($noerr) {
2049                 warn $err;
2050             } else {
2051                 die $err;
2052             }
2053         }
2054     });
2055 }
2056
2057 sub mount_all {
2058     my ($vmid, $storage_cfg, $conf) = @_;
2059
2060     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2061     File::Path::make_path($rootdir);
2062
2063     my $volid_list = get_vm_volumes($conf);
2064     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2065
2066     eval {
2067         foreach_mountpoint($conf, sub {
2068             my ($ms, $mountpoint) = @_;
2069
2070             my $volid = $mountpoint->{volume};
2071             my $mount = $mountpoint->{mp};
2072
2073             return if !$volid || !$mount;
2074
2075             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2076             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2077                 PVE::Storage::parse_volname($storage_cfg, $volid);
2078
2079             die "unable to mount base volume - internal error" if $isBase;
2080
2081             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2082         });
2083     };
2084     if (my $err = $@) {
2085         warn "mounting container failed - $err";
2086         umount_all($vmid, $storage_cfg, $conf, 1);
2087     }
2088
2089     return $rootdir;
2090 }
2091
2092
2093 sub mountpoint_mount_path {
2094     my ($mountpoint, $storage_cfg, $snapname) = @_;
2095
2096     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2097 }
2098
2099 my $check_mount_path = sub {
2100     my ($path) = @_;
2101     $path = File::Spec->canonpath($path);
2102     my $real = Cwd::realpath($path);
2103     if ($real ne $path) {
2104         die "mount path modified by symlink: $path != $real";
2105     }
2106 };
2107
2108 # use $rootdir = undef to just return the corresponding mount path
2109 sub mountpoint_mount {
2110     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2111
2112     my $volid = $mountpoint->{volume};
2113     my $mount = $mountpoint->{mp};
2114
2115     return if !$volid || !$mount;
2116
2117     my $mount_path;
2118
2119     if (defined($rootdir)) {
2120         $rootdir =~ s!/+$!!;
2121         $mount_path = "$rootdir/$mount";
2122         $mount_path =~ s!/+!/!g;
2123         &$check_mount_path($mount_path);
2124         File::Path::mkpath($mount_path);
2125     }
2126
2127     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2128
2129     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2130
2131     if ($storage) {
2132
2133         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2134         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2135
2136         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2137             PVE::Storage::parse_volname($storage_cfg, $volid);
2138
2139         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2140
2141         if ($format eq 'subvol') {
2142             if ($mount_path) {
2143                 if ($snapname) {
2144                     if ($scfg->{type} eq 'zfspool') {
2145                         my $path_arg = $path;
2146                         $path_arg =~ s!^/+!!;
2147                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2148                     } else {
2149                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2150                     }
2151                 } else {
2152                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2153                 }
2154             }
2155             return wantarray ? ($path, 0) : $path;
2156         } elsif ($format eq 'raw' || $format eq 'iso') {
2157             my $use_loopdev = 0;
2158             my @extra_opts;
2159             if ($scfg->{path}) {
2160                 push @extra_opts, '-o', 'loop';
2161                 $use_loopdev = 1;
2162             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2163                 # do nothing
2164             } else {
2165                 die "unsupported storage type '$scfg->{type}'\n";
2166             }
2167             if ($mount_path) {
2168                 if ($format eq 'iso') {
2169                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2170                 } elsif ($isBase || defined($snapname)) {
2171                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2172                 } else {
2173                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2174                 }
2175             }
2176             return wantarray ? ($path, $use_loopdev) : $path;
2177         } else {
2178             die "unsupported image format '$format'\n";
2179         }
2180     } elsif ($volid =~ m|^/dev/.+|) {
2181         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2182         return wantarray ? ($volid, 0) : $volid;
2183     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2184         &$check_mount_path($volid);
2185         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2186         return wantarray ? ($volid, 0) : $volid;
2187     }
2188
2189     die "unsupported storage";
2190 }
2191
2192 sub get_vm_volumes {
2193     my ($conf, $excludes) = @_;
2194
2195     my $vollist = [];
2196
2197     foreach_mountpoint($conf, sub {
2198         my ($ms, $mountpoint) = @_;
2199
2200         return if $excludes && $ms eq $excludes;
2201
2202         my $volid = $mountpoint->{volume};
2203
2204         return if !$volid || $volid =~ m|^/|;
2205
2206         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2207         return if !$sid;
2208
2209         push @$vollist, $volid;
2210     });
2211
2212     return $vollist;
2213 }
2214
2215 sub mkfs {
2216     my ($dev) = @_;
2217
2218     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2219 }
2220
2221 sub format_disk {
2222     my ($storage_cfg, $volid) = @_;
2223
2224     if ($volid =~ m!^/dev/.+!) {
2225         mkfs($volid);
2226         return;
2227     }
2228
2229     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2230
2231     die "cannot format volume '$volid' with no storage\n" if !$storage;
2232
2233     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2234
2235     my $path = PVE::Storage::path($storage_cfg, $volid);
2236
2237     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2238         PVE::Storage::parse_volname($storage_cfg, $volid);
2239
2240     die "cannot format volume '$volid' (format == $format)\n"
2241         if $format ne 'raw';
2242
2243     mkfs($path);
2244 }
2245
2246 sub destroy_disks {
2247     my ($storecfg, $vollist) = @_;
2248
2249     foreach my $volid (@$vollist) {
2250         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2251         warn $@ if $@;
2252     }
2253 }
2254
2255 sub create_disks {
2256     my ($storecfg, $vmid, $settings, $conf) = @_;
2257
2258     my $vollist = [];
2259
2260     eval {
2261         foreach_mountpoint($settings, sub {
2262             my ($ms, $mountpoint) = @_;
2263
2264             my $volid = $mountpoint->{volume};
2265             my $mp = $mountpoint->{mp};
2266
2267             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2268
2269             return if !$storage;
2270
2271             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2272                 my ($storeid, $size_gb) = ($1, $2);
2273
2274                 my $size_kb = int(${size_gb}*1024) * 1024;
2275
2276                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2277                 # fixme: use better naming ct-$vmid-disk-X.raw?
2278
2279                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2280                     if ($size_kb > 0) {
2281                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2282                                                            undef, $size_kb);
2283                         format_disk($storecfg, $volid);
2284                     } else {
2285                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2286                                                            undef, 0);
2287                     }
2288                 } elsif ($scfg->{type} eq 'zfspool') {
2289
2290                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2291                                                        undef, $size_kb);
2292                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2293
2294                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2295                     format_disk($storecfg, $volid);
2296
2297                 } elsif ($scfg->{type} eq 'rbd') {
2298
2299                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2300                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2301                     format_disk($storecfg, $volid);
2302                 } else {
2303                     die "unable to create containers on storage type '$scfg->{type}'\n";
2304                 }
2305                 push @$vollist, $volid;
2306                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2307                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2308             } else {
2309                 # use specified/existing volid
2310             }
2311         });
2312     };
2313     # free allocated images on error
2314     if (my $err = $@) {
2315         destroy_disks($storecfg, $vollist);
2316         die $err;
2317     }
2318     return $vollist;
2319 }
2320
2321 # bash completion helper
2322
2323 sub complete_os_templates {
2324     my ($cmdname, $pname, $cvalue) = @_;
2325
2326     my $cfg = PVE::Storage::config();
2327
2328     my $storeid;
2329
2330     if ($cvalue =~ m/^([^:]+):/) {
2331         $storeid = $1;
2332     }
2333
2334     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2335     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2336
2337     my $res = [];
2338     foreach my $id (keys %$data) {
2339         foreach my $item (@{$data->{$id}}) {
2340             push @$res, $item->{volid} if defined($item->{volid});
2341         }
2342     }
2343
2344     return $res;
2345 }
2346
2347 my $complete_ctid_full = sub {
2348     my ($running) = @_;
2349
2350     my $idlist = vmstatus();
2351
2352     my $active_hash = list_active_containers();
2353
2354     my $res = [];
2355
2356     foreach my $id (keys %$idlist) {
2357         my $d = $idlist->{$id};
2358         if (defined($running)) {
2359             next if $d->{template};
2360             next if $running && !$active_hash->{$id};
2361             next if !$running && $active_hash->{$id};
2362         }
2363         push @$res, $id;
2364
2365     }
2366     return $res;
2367 };
2368
2369 sub complete_ctid {
2370     return &$complete_ctid_full();
2371 }
2372
2373 sub complete_ctid_stopped {
2374     return &$complete_ctid_full(0);
2375 }
2376
2377 sub complete_ctid_running {
2378     return &$complete_ctid_full(1);
2379 }
2380
2381 sub parse_id_maps {
2382     my ($conf) = @_;
2383
2384     my $id_map = [];
2385     my $rootuid = 0;
2386     my $rootgid = 0;
2387
2388     my $lxc = $conf->{lxc};
2389     foreach my $entry (@$lxc) {
2390         my ($key, $value) = @$entry;
2391         next if $key ne 'lxc.id_map';
2392         if ($value =~ /^([ug])\s+(\d+)\s+(\d+)\s+(\d+)\s*$/) {
2393             my ($type, $ct, $host, $length) = ($1, $2, $3, $4);
2394             push @$id_map, [$type, $ct, $host, $length];
2395             if ($ct == 0) {
2396                 $rootuid = $host if $type eq 'u';
2397                 $rootgid = $host if $type eq 'g';
2398             }
2399         } else {
2400             die "failed to parse id_map: $value\n";
2401         }
2402     }
2403
2404     if (!@$id_map && $conf->{unprivileged}) {
2405         # Should we read them from /etc/subuid?
2406         $id_map = [ ['u', '0', '100000', '65536'],
2407                     ['g', '0', '100000', '65536'] ];
2408         $rootuid = $rootgid = 100000;
2409     }
2410
2411     return ($id_map, $rootuid, $rootgid);
2412 }
2413
2414 sub userns_command {
2415     my ($id_map) = @_;
2416     if (@$id_map) {
2417         return ['lxc-usernsexec', (map { ('-m', join(':', @$_)) } @$id_map), '--'];
2418     }
2419     return [];
2420 }
2421
2422 1;