src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use File::Spec;
   9 use Cwd qw();
  10 use Fcntl ':flock';
  11
  12 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  13 use PVE::Storage;
  14 use PVE::SafeSyslog;
  15 use PVE::INotify;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  18 use PVE::Network;
  19 use PVE::AccessControl;
  20 use PVE::ProcFSTools;
  21 use Time::HiRes qw (gettimeofday);
  22
  23 use Data::Dumper;
  24
  25 my $nodename = PVE::INotify::nodename();
  26
  27 my $cpuinfo= PVE::ProcFSTools::read_cpuinfo();
  28
  29 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  30
  31 my $rootfs_desc = {
  32     volume => {
  33         type => 'string',
  34         default_key => 1,
  35         format_description => 'volume',
  36         description => 'Volume, device or directory to mount into the container.',
  37     },
  38     backup => {
  39         type => 'boolean',
  40         format_description => '[1|0]',
  41         description => 'Whether to include the mountpoint in backups.',
  42         optional => 1,
  43     },
  44     size => {
  45         type => 'string',
  46         format => 'disk-size',
  47         format_description => 'DiskSize',
  48         description => 'Volume size (read only value).',
  49         optional => 1,
  50     },
  51 };
  52
  53 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  54     type => 'string', format => $rootfs_desc,
  55     description => "Use volume as container root.",
  56     optional => 1,
  57 });
  58
  59 PVE::JSONSchema::register_standard_option('pve-lxc-snapshot-name', {
  60     description => "The name of the snapshot.",
  61     type => 'string', format => 'pve-configid',
  62     maxLength => 40,
  63 });
  64
  65 my $confdesc = {
  66     lock => {
  67         optional => 1,
  68         type => 'string',
  69         description => "Lock/unlock the VM.",
  70         enum => [qw(migrate backup snapshot rollback)],
  71     },
  72     onboot => {
  73         optional => 1,
  74         type => 'boolean',
  75         description => "Specifies whether a VM will be started during system bootup.",
  76         default => 0,
  77     },
  78     startup => get_standard_option('pve-startup-order'),
  79     template => {
  80         optional => 1,
  81         type => 'boolean',
  82         description => "Enable/disable Template.",
  83         default => 0,
  84     },
  85     arch => {
  86         optional => 1,
  87         type => 'string',
  88         enum => ['amd64', 'i386'],
  89         description => "OS architecture type.",
  90         default => 'amd64',
  91     },
  92     ostype => {
  93         optional => 1,
  94         type => 'string',
  95         enum => ['debian', 'ubuntu', 'centos', 'archlinux'],
  96         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  97     },
  98     console => {
  99         optional => 1,
 100         type => 'boolean',
 101         description => "Attach a console device (/dev/console) to the container.",
 102         default => 1,
 103     },
 104     tty => {
 105         optional => 1,
 106         type => 'integer',
 107         description => "Specify the number of tty available to the container",
 108         minimum => 0,
 109         maximum => 6,
 110         default => 2,
 111     },
 112     cpulimit => {
 113         optional => 1,
 114         type => 'number',
 115         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 116         minimum => 0,
 117         maximum => 128,
 118         default => 0,
 119     },
 120     cpuunits => {
 121         optional => 1,
 122         type => 'integer',
 123         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 124         minimum => 0,
 125         maximum => 500000,
 126         default => 1024,
 127     },
 128     memory => {
 129         optional => 1,
 130         type => 'integer',
 131         description => "Amount of RAM for the VM in MB.",
 132         minimum => 16,
 133         default => 512,
 134     },
 135     swap => {
 136         optional => 1,
 137         type => 'integer',
 138         description => "Amount of SWAP for the VM in MB.",
 139         minimum => 0,
 140         default => 512,
 141     },
 142     hostname => {
 143         optional => 1,
 144         description => "Set a host name for the container.",
 145         type => 'string', format => 'dns-name',
 146         maxLength => 255,
 147     },
 148     description => {
 149         optional => 1,
 150         type => 'string',
 151         description => "Container description. Only used on the configuration web interface.",
 152     },
 153     searchdomain => {
 154         optional => 1,
 155         type => 'string', format => 'dns-name-list',
 156         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 157     },
 158     nameserver => {
 159         optional => 1,
 160         type => 'string', format => 'address-list',
 161         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 162     },
 163     rootfs => get_standard_option('pve-ct-rootfs'),
 164     parent => {
 165         optional => 1,
 166         type => 'string', format => 'pve-configid',
 167         maxLength => 40,
 168         description => "Parent snapshot name. This is used internally, and should not be modified.",
 169     },
 170     snaptime => {
 171         optional => 1,
 172         description => "Timestamp for snapshots.",
 173         type => 'integer',
 174         minimum => 0,
 175     },
 176     cmode => {
 177         optional => 1,
 178         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 179         type => 'string',
 180         enum => ['shell', 'console', 'tty'],
 181         default => 'tty',
 182     },
 183     protection => {
 184         optional => 1,
 185         type => 'boolean',
 186         description => "Sets the protection flag of the container. This will prevent the remove operation. This will prevent the CT or CT's disk remove/update operation.",
 187         default => 0,
 188     },
 189 };
 190
 191 my $valid_lxc_conf_keys = {
 192     'lxc.include' => 1,
 193     'lxc.arch' => 1,
 194     'lxc.utsname' => 1,
 195     'lxc.haltsignal' => 1,
 196     'lxc.rebootsignal' => 1,
 197     'lxc.stopsignal' => 1,
 198     'lxc.init_cmd' => 1,
 199     'lxc.network.type' => 1,
 200     'lxc.network.flags' => 1,
 201     'lxc.network.link' => 1,
 202     'lxc.network.mtu' => 1,
 203     'lxc.network.name' => 1,
 204     'lxc.network.hwaddr' => 1,
 205     'lxc.network.ipv4' => 1,
 206     'lxc.network.ipv4.gateway' => 1,
 207     'lxc.network.ipv6' => 1,
 208     'lxc.network.ipv6.gateway' => 1,
 209     'lxc.network.script.up' => 1,
 210     'lxc.network.script.down' => 1,
 211     'lxc.pts' => 1,
 212     'lxc.console.logfile' => 1,
 213     'lxc.console' => 1,
 214     'lxc.tty' => 1,
 215     'lxc.devttydir' => 1,
 216     'lxc.hook.autodev' => 1,
 217     'lxc.autodev' => 1,
 218     'lxc.kmsg' => 1,
 219     'lxc.mount' => 1,
 220     'lxc.mount.entry' => 1,
 221     'lxc.mount.auto' => 1,
 222     'lxc.rootfs' => 1,
 223     'lxc.rootfs.mount' => 1,
 224     'lxc.rootfs.options' => 1,
 225     # lxc.cgroup.*
 226     'lxc.cap.drop' => 1,
 227     'lxc.cap.keep' => 1,
 228     'lxc.aa_profile' => 1,
 229     'lxc.aa_allow_incomplete' => 1,
 230     'lxc.se_context' => 1,
 231     'lxc.seccomp' => 1,
 232     'lxc.id_map' => 1,
 233     'lxc.hook.pre-start' => 1,
 234     'lxc.hook.pre-mount' => 1,
 235     'lxc.hook.mount' => 1,
 236     'lxc.hook.start' => 1,
 237     'lxc.hook.stop' => 1,
 238     'lxc.hook.post-stop' => 1,
 239     'lxc.hook.clone' => 1,
 240     'lxc.hook.destroy' => 1,
 241     'lxc.loglevel' => 1,
 242     'lxc.logfile' => 1,
 243     'lxc.start.auto' => 1,
 244     'lxc.start.delay' => 1,
 245     'lxc.start.order' => 1,
 246     'lxc.group' => 1,
 247     'lxc.environment' => 1,
 248     'lxc.' => 1,
 249     'lxc.' => 1,
 250     'lxc.' => 1,
 251     'lxc.' => 1,
 252 };
 253
 254 my $netconf_desc = {
 255     type => {
 256         type => 'string',
 257         optional => 1,
 258         description => "Network interface type.",
 259         enum => [qw(veth)],
 260     },
 261     name => {
 262         type => 'string',
 263         format_description => 'String',
 264         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 265         pattern => '[-_.\w\d]+',
 266     },
 267     bridge => {
 268         type => 'string',
 269         format_description => 'vmbr<Number>',
 270         description => 'Bridge to attach the network device to.',
 271         pattern => '[-_.\w\d]+',
 272         optional => 1,
 273     },
 274     hwaddr => {
 275         type => 'string',
 276         format_description => 'MAC',
 277         description => 'Bridge to attach the network device to. (lxc.network.hwaddr)',
 278         pattern => qr/(?:[a-f0-9]{2}:){5}[a-f0-9]{2}/i,
 279         optional => 1,
 280     },
 281     mtu => {
 282         type => 'integer',
 283         format_description => 'Number',
 284         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 285         minimum => 64, # minimum ethernet frame is 64 bytes
 286         optional => 1,
 287     },
 288     ip => {
 289         type => 'string',
 290         format => 'pve-ipv4-config',
 291         format_description => 'IPv4Format/CIDR',
 292         description => 'IPv4 address in CIDR format.',
 293         optional => 1,
 294     },
 295     gw => {
 296         type => 'string',
 297         format => 'ipv4',
 298         format_description => 'GatewayIPv4',
 299         description => 'Default gateway for IPv4 traffic.',
 300         optional => 1,
 301     },
 302     ip6 => {
 303         type => 'string',
 304         format => 'pve-ipv6-config',
 305         format_description => 'IPv6Format/CIDR',
 306         description => 'IPv6 address in CIDR format.',
 307         optional => 1,
 308     },
 309     gw6 => {
 310         type => 'string',
 311         format => 'ipv6',
 312         format_description => 'GatewayIPv6',
 313         description => 'Default gateway for IPv6 traffic.',
 314         optional => 1,
 315     },
 316     firewall => {
 317         type => 'boolean',
 318         format_description => '[1|0]',
 319         description => "Controls whether this interface's firewall rules should be used.",
 320         optional => 1,
 321     },
 322     tag => {
 323         type => 'integer',
 324         format_description => 'VlanNo',
 325         minimum => '2',
 326         maximum => '4094',
 327         description => "VLAN tag foro this interface.",
 328         optional => 1,
 329     },
 330 };
 331 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 332
 333 my $MAX_LXC_NETWORKS = 10;
 334 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 335     $confdesc->{"net$i"} = {
 336         optional => 1,
 337         type => 'string', format => $netconf_desc,
 338         description => "Specifies network interfaces for the container.",
 339     };
 340 }
 341
 342 my $mp_desc = {
 343     %$rootfs_desc,
 344     mp => {
 345         type => 'string',
 346         format_description => 'Path',
 347         description => 'Path to the mountpoint as seen from inside the container.',
 348         optional => 1,
 349     },
 350 };
 351 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 352
 353 my $unuseddesc = {
 354     optional => 1,
 355     type => 'string', format => 'pve-volume-id',
 356     description => "Reference to unused volumes.",
 357 };
 358
 359 my $MAX_MOUNT_POINTS = 10;
 360 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 361     $confdesc->{"mp$i"} = {
 362         optional => 1,
 363         type => 'string', format => $mp_desc,
 364         description => "Use volume as container mount point (experimental feature).",
 365         optional => 1,
 366     };
 367 }
 368
 369 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
 370 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 371     $confdesc->{"unused$i"} = $unuseddesc;
 372 }
 373
 374 sub write_pct_config {
 375     my ($filename, $conf) = @_;
 376
 377     delete $conf->{snapstate}; # just to be sure
 378
 379     my $generate_raw_config = sub {
 380         my ($conf) = @_;
 381
 382         my $raw = '';
 383
 384         # add description as comment to top of file
 385         my $descr = $conf->{description} || '';
 386         foreach my $cl (split(/\n/, $descr)) {
 387             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 388         }
 389
 390         foreach my $key (sort keys %$conf) {
 391             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 392                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 393             my $value = $conf->{$key};
 394             die "detected invalid newline inside property '$key'\n" if $value =~ m/\n/;
 395             $raw .= "$key: $value\n";
 396         }
 397
 398         if (my $lxcconf = $conf->{lxc}) {
 399             foreach my $entry (@$lxcconf) {
 400                 my ($k, $v) = @$entry;
 401                 $raw .= "$k: $v\n";
 402             }
 403         }
 404
 405         return $raw;
 406     };
 407
 408     my $raw = &$generate_raw_config($conf);
 409
 410     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 411         $raw .= "\n[$snapname]\n";
 412         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 413     }
 414
 415     return $raw;
 416 }
 417
 418 sub check_type {
 419     my ($key, $value) = @_;
 420
 421     die "unknown setting '$key'\n" if !$confdesc->{$key};
 422
 423     my $type = $confdesc->{$key}->{type};
 424
 425     if (!defined($value)) {
 426         die "got undefined value\n";
 427     }
 428
 429     if ($value =~ m/[\n\r]/) {
 430         die "property contains a line feed\n";
 431     }
 432
 433     if ($type eq 'boolean') {
 434         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 435         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 436         die "type check ('boolean') failed - got '$value'\n";
 437     } elsif ($type eq 'integer') {
 438         return int($1) if $value =~ m/^(\d+)$/;
 439         die "type check ('integer') failed - got '$value'\n";
 440     } elsif ($type eq 'number') {
 441         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 442         die "type check ('number') failed - got '$value'\n";
 443     } elsif ($type eq 'string') {
 444         if (my $fmt = $confdesc->{$key}->{format}) {
 445             PVE::JSONSchema::check_format($fmt, $value);
 446             return $value;
 447         }
 448         return $value;
 449     } else {
 450         die "internal error"
 451     }
 452 }
 453
 454 sub parse_pct_config {
 455     my ($filename, $raw) = @_;
 456
 457     return undef if !defined($raw);
 458
 459     my $res = {
 460         digest => Digest::SHA::sha1_hex($raw),
 461         snapshots => {},
 462     };
 463
 464     $filename =~ m|/lxc/(\d+).conf$|
 465         || die "got strange filename '$filename'";
 466
 467     my $vmid = $1;
 468
 469     my $conf = $res;
 470     my $descr = '';
 471     my $section = '';
 472
 473     my @lines = split(/\n/, $raw);
 474     foreach my $line (@lines) {
 475         next if $line =~ m/^\s*$/;
 476
 477         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 478             $section = $1;
 479             $conf->{description} = $descr if $descr;
 480             $descr = '';
 481             $conf = $res->{snapshots}->{$section} = {};
 482             next;
 483         }
 484
 485         if ($line =~ m/^\#(.*)\s*$/) {
 486             $descr .= PVE::Tools::decode_text($1) . "\n";
 487             next;
 488         }
 489
 490         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 491             my $key = $1;
 492             my $value = $3;
 493             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 494                 push @{$conf->{lxc}}, [$key, $value];
 495             } else {
 496                 warn "vm $vmid - unable to parse config: $line\n";
 497             }
 498         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 499             $descr .= PVE::Tools::decode_text($2);
 500         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 501             $conf->{snapstate} = $1;
 502         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 503             my $key = $1;
 504             my $value = $2;
 505             eval { $value = check_type($key, $value); };
 506             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 507             $conf->{$key} = $value;
 508         } else {
 509             warn "vm $vmid - unable to parse config: $line\n";
 510         }
 511     }
 512
 513     $conf->{description} = $descr if $descr;
 514
 515     delete $res->{snapstate}; # just to be sure
 516
 517     return $res;
 518 }
 519
 520 sub config_list {
 521     my $vmlist = PVE::Cluster::get_vmlist();
 522     my $res = {};
 523     return $res if !$vmlist || !$vmlist->{ids};
 524     my $ids = $vmlist->{ids};
 525
 526     foreach my $vmid (keys %$ids) {
 527         next if !$vmid; # skip CT0
 528         my $d = $ids->{$vmid};
 529         next if !$d->{node} || $d->{node} ne $nodename;
 530         next if !$d->{type} || $d->{type} ne 'lxc';
 531         $res->{$vmid}->{type} = 'lxc';
 532     }
 533     return $res;
 534 }
 535
 536 sub cfs_config_path {
 537     my ($vmid, $node) = @_;
 538
 539     $node = $nodename if !$node;
 540     return "nodes/$node/lxc/$vmid.conf";
 541 }
 542
 543 sub config_file {
 544     my ($vmid, $node) = @_;
 545
 546     my $cfspath = cfs_config_path($vmid, $node);
 547     return "/etc/pve/$cfspath";
 548 }
 549
 550 sub load_config {
 551     my ($vmid, $node) = @_;
 552
 553     $node = $nodename if !$node;
 554     my $cfspath = cfs_config_path($vmid, $node);
 555
 556     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 557     die "container $vmid does not exists\n" if !defined($conf);
 558
 559     return $conf;
 560 }
 561
 562 sub create_config {
 563     my ($vmid, $conf) = @_;
 564
 565     my $dir = "/etc/pve/nodes/$nodename/lxc";
 566     mkdir $dir;
 567
 568     write_config($vmid, $conf);
 569 }
 570
 571 sub destroy_config {
 572     my ($vmid) = @_;
 573
 574     unlink config_file($vmid, $nodename);
 575 }
 576
 577 sub write_config {
 578     my ($vmid, $conf) = @_;
 579
 580     my $cfspath = cfs_config_path($vmid);
 581
 582     PVE::Cluster::cfs_write_file($cfspath, $conf);
 583 }
 584
 585 # flock: we use one file handle per process, so lock file
 586 # can be called multiple times and succeeds for the same process.
 587
 588 my $lock_handles =  {};
 589 my $lockdir = "/run/lock/lxc";
 590
 591 sub lock_filename {
 592     my ($vmid) = @_;
 593
 594     return "$lockdir/pve-config-${vmid}.lock";
 595 }
 596
 597 sub lock_aquire {
 598     my ($vmid, $timeout) = @_;
 599
 600     $timeout = 10 if !$timeout;
 601     my $mode = LOCK_EX;
 602
 603     my $filename = lock_filename($vmid);
 604
 605     mkdir $lockdir if !-d $lockdir;
 606
 607     my $lock_func = sub {
 608         if (!$lock_handles->{$$}->{$filename}) {
 609             my $fh = new IO::File(">>$filename") ||
 610                 die "can't open file - $!\n";
 611             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 612         }
 613
 614         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 615             print STDERR "trying to aquire lock...";
 616             my $success;
 617             while(1) {
 618                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 619                 # try again on EINTR (see bug #273)
 620                 if ($success || ($! != EINTR)) {
 621                     last;
 622                 }
 623             }
 624             if (!$success) {
 625                 print STDERR " failed\n";
 626                 die "can't aquire lock - $!\n";
 627             }
 628
 629             print STDERR " OK\n";
 630         }
 631
 632         $lock_handles->{$$}->{$filename}->{refcount}++;
 633     };
 634
 635     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 636     my $err = $@;
 637     if ($err) {
 638         die "can't lock file '$filename' - $err";
 639     }
 640 }
 641
 642 sub lock_release {
 643     my ($vmid) = @_;
 644
 645     my $filename = lock_filename($vmid);
 646
 647     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 648         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 649         if ($refcount <= 0) {
 650             $lock_handles->{$$}->{$filename} = undef;
 651             close ($fh);
 652         }
 653     }
 654 }
 655
 656 sub lock_container {
 657     my ($vmid, $timeout, $code, @param) = @_;
 658
 659     my $res;
 660
 661     lock_aquire($vmid, $timeout);
 662     eval { $res = &$code(@param) };
 663     my $err = $@;
 664     lock_release($vmid);
 665
 666     die $err if $err;
 667
 668     return $res;
 669 }
 670
 671 sub option_exists {
 672     my ($name) = @_;
 673
 674     return defined($confdesc->{$name});
 675 }
 676
 677 # add JSON properties for create and set function
 678 sub json_config_properties {
 679     my $prop = shift;
 680
 681     foreach my $opt (keys %$confdesc) {
 682         next if $opt eq 'parent' || $opt eq 'snaptime';
 683         next if $prop->{$opt};
 684         $prop->{$opt} = $confdesc->{$opt};
 685     }
 686
 687     return $prop;
 688 }
 689
 690 sub json_config_properties_no_rootfs {
 691     my $prop = shift;
 692
 693     foreach my $opt (keys %$confdesc) {
 694         next if $prop->{$opt};
 695         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 696         $prop->{$opt} = $confdesc->{$opt};
 697     }
 698
 699     return $prop;
 700 }
 701
 702 # container status helpers
 703
 704 sub list_active_containers {
 705
 706     my $filename = "/proc/net/unix";
 707
 708     # similar test is used by lcxcontainers.c: list_active_containers
 709     my $res = {};
 710
 711     my $fh = IO::File->new ($filename, "r");
 712     return $res if !$fh;
 713
 714     while (defined(my $line = <$fh>)) {
 715         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 716             my $path = $1;
 717             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 718                 $res->{$1} = 1;
 719             }
 720         }
 721     }
 722
 723     close($fh);
 724
 725     return $res;
 726 }
 727
 728 # warning: this is slow
 729 sub check_running {
 730     my ($vmid) = @_;
 731
 732     my $active_hash = list_active_containers();
 733
 734     return 1 if defined($active_hash->{$vmid});
 735
 736     return undef;
 737 }
 738
 739 sub get_container_disk_usage {
 740     my ($vmid, $pid) = @_;
 741
 742     return PVE::Tools::df("/proc/$pid/root/", 1);
 743 }
 744
 745 my $last_proc_vmid_stat;
 746
 747 my $parse_cpuacct_stat = sub {
 748     my ($vmid) = @_;
 749
 750     my $raw = read_cgroup_value('cpuacct', $vmid, 'cpuacct.stat', 1);
 751
 752     my $stat = {};
 753
 754     if ($raw =~ m/^user (\d+)\nsystem (\d+)\n/) {
 755
 756         $stat->{utime} = $1;
 757         $stat->{stime} = $2;
 758
 759     }
 760
 761     return $stat;
 762 };
 763
 764 sub vmstatus {
 765     my ($opt_vmid) = @_;
 766
 767     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 768
 769     my $active_hash = list_active_containers();
 770
 771     my $cpucount = $cpuinfo->{cpus} || 1;
 772
 773     my $cdtime = gettimeofday;
 774
 775     my $uptime = (PVE::ProcFSTools::read_proc_uptime(1))[0];
 776
 777     foreach my $vmid (keys %$list) {
 778         my $d = $list->{$vmid};
 779
 780         eval { $d->{pid} = find_lxc_pid($vmid) if defined($active_hash->{$vmid}); };
 781         warn $@ if $@; # ignore errors (consider them stopped)
 782
 783         $d->{status} = $d->{pid} ? 'running' : 'stopped';
 784
 785         my $cfspath = cfs_config_path($vmid);
 786         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 787
 788         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 789         $d->{name} =~ s/[\s]//g;
 790
 791         $d->{cpus} = $conf->{cpulimit} || $cpucount;
 792
 793         if ($d->{pid}) {
 794             my $res = get_container_disk_usage($vmid, $d->{pid});
 795             $d->{disk} = $res->{used};
 796             $d->{maxdisk} = $res->{total};
 797         } else {
 798             $d->{disk} = 0;
 799             # use 4GB by default ??
 800             if (my $rootfs = $conf->{rootfs}) {
 801                 my $rootinfo = parse_ct_mountpoint($rootfs);
 802                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 803             } else {
 804                 $d->{maxdisk} = 4*1024*1024*1024;
 805             }
 806         }
 807
 808         $d->{mem} = 0;
 809         $d->{swap} = 0;
 810         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 811         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 812
 813         $d->{uptime} = 0;
 814         $d->{cpu} = 0;
 815
 816         $d->{netout} = 0;
 817         $d->{netin} = 0;
 818
 819         $d->{diskread} = 0;
 820         $d->{diskwrite} = 0;
 821
 822         $d->{template} = is_template($conf);
 823     }
 824
 825     foreach my $vmid (keys %$list) {
 826         my $d = $list->{$vmid};
 827         my $pid = $d->{pid};
 828
 829         next if !$pid; # skip stopped CTs
 830
 831         my $ctime = (stat("/proc/$pid"))[10]; # 10 = ctime
 832         $d->{uptime} = time - $ctime; # the method lxcfs uses
 833
 834         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 835         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 836
 837         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 838         my @bytes = split(/\n/, $blkio_bytes);
 839         foreach my $byte (@bytes) {
 840             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 841                 $d->{diskread} = $2 if $key eq 'Read';
 842                 $d->{diskwrite} = $2 if $key eq 'Write';
 843             }
 844         }
 845
 846         my $pstat = &$parse_cpuacct_stat($vmid);
 847
 848         my $used = $pstat->{utime} + $pstat->{stime};
 849
 850         my $old = $last_proc_vmid_stat->{$vmid};
 851         if (!$old) {
 852             $last_proc_vmid_stat->{$vmid} = {
 853                 time => $cdtime,
 854                 used => $used,
 855                 cpu => 0,
 856             };
 857             next;
 858         }
 859
 860         my $dtime = ($cdtime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
 861
 862         if ($dtime > 1000) {
 863             my $dutime = $used -  $old->{used};
 864
 865             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
 866             $last_proc_vmid_stat->{$vmid} = {
 867                 time => $cdtime,
 868                 used => $used,
 869                 cpu => $d->{cpu},
 870             };
 871         } else {
 872             $d->{cpu} = $old->{cpu};
 873         }
 874     }
 875
 876     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 877
 878     foreach my $dev (keys %$netdev) {
 879         next if $dev !~ m/^veth([1-9]\d*)i/;
 880         my $vmid = $1;
 881         my $d = $list->{$vmid};
 882
 883         next if !$d;
 884
 885         $d->{netout} += $netdev->{$dev}->{receive};
 886         $d->{netin} += $netdev->{$dev}->{transmit};
 887
 888     }
 889
 890     return $list;
 891 }
 892
 893 sub parse_ct_mountpoint {
 894     my ($data, $noerr) = @_;
 895
 896     $data //= '';
 897
 898     my $res;
 899     eval { $res = PVE::JSONSchema::parse_property_string($mp_desc, $data) };
 900     if ($@) {
 901         return undef if $noerr;
 902         die $@;
 903     }
 904
 905     if (defined(my $size = $res->{size})) {
 906         $size = PVE::JSONSchema::parse_size($size);
 907         if (!defined($size)) {
 908             return undef if $noerr;
 909             die "invalid size: $size\n";
 910         }
 911         $res->{size} = $size;
 912     }
 913
 914     return $res;
 915 }
 916
 917 sub print_ct_mountpoint {
 918     my ($info, $nomp) = @_;
 919     my $skip = $nomp ? ['mp'] : [];
 920     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
 921 }
 922
 923 sub print_lxc_network {
 924     my $net = shift;
 925     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
 926 }
 927
 928 sub parse_lxc_network {
 929     my ($data) = @_;
 930
 931     my $res = {};
 932
 933     return $res if !$data;
 934
 935     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
 936
 937     $res->{type} = 'veth';
 938     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 939
 940     return $res;
 941 }
 942
 943 sub read_cgroup_value {
 944     my ($group, $vmid, $name, $full) = @_;
 945
 946     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 947
 948     return PVE::Tools::file_get_contents($path) if $full;
 949
 950     return PVE::Tools::file_read_firstline($path);
 951 }
 952
 953 sub write_cgroup_value {
 954    my ($group, $vmid, $name, $value) = @_;
 955
 956    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 957    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 958
 959 }
 960
 961 sub find_lxc_console_pids {
 962
 963     my $res = {};
 964
 965     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 966         my ($pid) = @_;
 967
 968         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 969         return if !$cmdline;
 970
 971         my @args = split(/\0/, $cmdline);
 972
 973         # serach for lxc-console -n <vmid>
 974         return if scalar(@args) != 3;
 975         return if $args[1] ne '-n';
 976         return if $args[2] !~ m/^\d+$/;
 977         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 978
 979         my $vmid = $args[2];
 980
 981         push @{$res->{$vmid}}, $pid;
 982     });
 983
 984     return $res;
 985 }
 986
 987 sub find_lxc_pid {
 988     my ($vmid) = @_;
 989
 990     my $pid = undef;
 991     my $parser = sub {
 992         my $line = shift;
 993         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 994     };
 995     PVE::Tools::run_command(['lxc-info', '-n', $vmid, '-p'], outfunc => $parser);
 996
 997     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 998
 999     return $pid;
1000 }
1001
1002 # Note: we cannot use Net:IP, because that only allows strict
1003 # CIDR networks
1004 sub parse_ipv4_cidr {
1005     my ($cidr, $noerr) = @_;
1006
1007     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 <= 32)) {
1008         return { address => $1, netmask => $PVE::Network::ipv4_reverse_mask->[$2] };
1009     }
1010
1011     return undef if $noerr;
1012
1013     die "unable to parse ipv4 address/mask\n";
1014 }
1015
1016 sub check_lock {
1017     my ($conf) = @_;
1018
1019     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
1020 }
1021
1022 sub check_protection {
1023     my ($vm_conf, $err_msg) = @_;
1024
1025     if ($vm_conf->{protection}) {
1026         die "$err_msg - protection mode enabled\n";
1027     }
1028 }
1029
1030 sub update_lxc_config {
1031     my ($storage_cfg, $vmid, $conf) = @_;
1032
1033     my $dir = "/var/lib/lxc/$vmid";
1034
1035     if ($conf->{template}) {
1036
1037         unlink "$dir/config";
1038
1039         return;
1040     }
1041
1042     my $raw = '';
1043
1044     die "missing 'arch' - internal error" if !$conf->{arch};
1045     $raw .= "lxc.arch = $conf->{arch}\n";
1046
1047     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
1048     if ($ostype =~ /^(?:debian | ubuntu | centos | archlinux)$/x) {
1049         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
1050     } else {
1051         die "implement me";
1052     }
1053
1054     if (!has_dev_console($conf)) {
1055         $raw .= "lxc.console = none\n";
1056         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
1057     }
1058
1059     my $ttycount = get_tty_count($conf);
1060     $raw .= "lxc.tty = $ttycount\n";
1061
1062     # some init scripts expects a linux terminal (turnkey).
1063     $raw .= "lxc.environment = TERM=linux\n";
1064
1065     my $utsname = $conf->{hostname} || "CT$vmid";
1066     $raw .= "lxc.utsname = $utsname\n";
1067
1068     my $memory = $conf->{memory} || 512;
1069     my $swap = $conf->{swap} // 0;
1070
1071     my $lxcmem = int($memory*1024*1024);
1072     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
1073
1074     my $lxcswap = int(($memory + $swap)*1024*1024);
1075     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
1076
1077     if (my $cpulimit = $conf->{cpulimit}) {
1078         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
1079         my $value = int(100000*$cpulimit);
1080         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
1081     }
1082
1083     my $shares = $conf->{cpuunits} || 1024;
1084     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
1085
1086     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1087     $mountpoint->{mp} = '/';
1088
1089     my ($path, $use_loopdev) = mountpoint_mount_path($mountpoint, $storage_cfg);
1090     $path = "loop:$path" if $use_loopdev;
1091
1092     $raw .= "lxc.rootfs = $path\n";
1093
1094     my $netcount = 0;
1095     foreach my $k (keys %$conf) {
1096         next if $k !~ m/^net(\d+)$/;
1097         my $ind = $1;
1098         my $d = parse_lxc_network($conf->{$k});
1099         $netcount++;
1100         $raw .= "lxc.network.type = veth\n";
1101         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1102         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1103         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1104         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1105     }
1106
1107     if (my $lxcconf = $conf->{lxc}) {
1108         foreach my $entry (@$lxcconf) {
1109             my ($k, $v) = @$entry;
1110             $netcount++ if $k eq 'lxc.network.type';
1111             $raw .= "$k = $v\n";
1112         }
1113     }
1114
1115     $raw .= "lxc.network.type = empty\n" if !$netcount;
1116
1117     File::Path::mkpath("$dir/rootfs");
1118
1119     PVE::Tools::file_set_contents("$dir/config", $raw);
1120 }
1121
1122 # verify and cleanup nameserver list (replace \0 with ' ')
1123 sub verify_nameserver_list {
1124     my ($nameserver_list) = @_;
1125
1126     my @list = ();
1127     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1128         PVE::JSONSchema::pve_verify_ip($server);
1129         push @list, $server;
1130     }
1131
1132     return join(' ', @list);
1133 }
1134
1135 sub verify_searchdomain_list {
1136     my ($searchdomain_list) = @_;
1137
1138     my @list = ();
1139     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1140         # todo: should we add checks for valid dns domains?
1141         push @list, $server;
1142     }
1143
1144     return join(' ', @list);
1145 }
1146
1147 sub add_unused_volume {
1148     my ($config, $volid) = @_;
1149
1150     # skip bind mounts and block devices
1151     return if $volid =~ m|^/|;
1152
1153     my $key;
1154     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1155         my $test = "unused$ind";
1156         if (my $vid = $config->{$test}) {
1157             return if $vid eq $volid; # do not add duplicates
1158         } else {
1159             $key = $test;
1160         }
1161     }
1162
1163     die "To many unused volume - please delete them first.\n" if !$key;
1164
1165     $config->{$key} = $volid;
1166
1167     return $key;
1168 }
1169
1170 sub update_pct_config {
1171     my ($vmid, $conf, $running, $param, $delete) = @_;
1172
1173     my @nohotplug;
1174
1175     my $new_disks = 0;
1176     my @deleted_volumes;
1177
1178     my $rootdir;
1179     if ($running) {
1180         my $pid = find_lxc_pid($vmid);
1181         $rootdir = "/proc/$pid/root";
1182     }
1183
1184     my $hotplug_error = sub {
1185         if ($running) {
1186             push @nohotplug, @_;
1187             return 1;
1188         } else {
1189             return 0;
1190         }
1191     };
1192
1193     if (defined($delete)) {
1194         foreach my $opt (@$delete) {
1195             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1196                 die "unable to delete required option '$opt'\n";
1197             } elsif ($opt eq 'swap') {
1198                 delete $conf->{$opt};
1199                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1200             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1201                 delete $conf->{$opt};
1202             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1203                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1204                 next if $hotplug_error->($opt);
1205                 delete $conf->{$opt};
1206             } elsif ($opt =~ m/^net(\d)$/) {
1207                 delete $conf->{$opt};
1208                 next if !$running;
1209                 my $netid = $1;
1210                 PVE::Network::veth_delete("veth${vmid}i$netid");
1211             } elsif ($opt eq 'protection') {
1212                 delete $conf->{$opt};
1213             } elsif ($opt =~ m/^unused(\d+)$/) {
1214                 next if $hotplug_error->($opt);
1215                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1216                 push @deleted_volumes, $conf->{$opt};
1217                 delete $conf->{$opt};
1218             } elsif ($opt =~ m/^mp(\d+)$/) {
1219                 next if $hotplug_error->($opt);
1220                 check_protection($conf, "can't remove CT $vmid drive '$opt'");
1221                 my $mountpoint = parse_ct_mountpoint($conf->{$opt});
1222                 add_unused_volume($conf, $mountpoint->{volume});
1223                 delete $conf->{$opt};
1224             } else {
1225                 die "implement me"
1226             }
1227             write_config($vmid, $conf) if $running;
1228         }
1229     }
1230
1231     # There's no separate swap size to configure, there's memory and "total"
1232     # memory (iow. memory+swap). This means we have to change them together.
1233     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1234     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1235     if (defined($wanted_memory) || defined($wanted_swap)) {
1236
1237         $wanted_memory //= ($conf->{memory} || 512);
1238         $wanted_swap //=  ($conf->{swap} || 0);
1239
1240         my $total = $wanted_memory + $wanted_swap;
1241         if ($running) {
1242             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1243             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1244         }
1245         $conf->{memory} = $wanted_memory;
1246         $conf->{swap} = $wanted_swap;
1247
1248         write_config($vmid, $conf) if $running;
1249     }
1250
1251     foreach my $opt (keys %$param) {
1252         my $value = $param->{$opt};
1253         if ($opt eq 'hostname') {
1254             $conf->{$opt} = $value;
1255         } elsif ($opt eq 'onboot') {
1256             $conf->{$opt} = $value ? 1 : 0;
1257         } elsif ($opt eq 'startup') {
1258             $conf->{$opt} = $value;
1259         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1260             next if $hotplug_error->($opt);
1261             $conf->{$opt} = $value;
1262         } elsif ($opt eq 'nameserver') {
1263             next if $hotplug_error->($opt);
1264             my $list = verify_nameserver_list($value);
1265             $conf->{$opt} = $list;
1266         } elsif ($opt eq 'searchdomain') {
1267             next if $hotplug_error->($opt);
1268             my $list = verify_searchdomain_list($value);
1269             $conf->{$opt} = $list;
1270         } elsif ($opt eq 'cpulimit') {
1271             next if $hotplug_error->($opt); # FIXME: hotplug
1272             $conf->{$opt} = $value;
1273         } elsif ($opt eq 'cpuunits') {
1274             $conf->{$opt} = $value;
1275             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1276         } elsif ($opt eq 'description') {
1277             $conf->{$opt} = PVE::Tools::encode_text($value);
1278         } elsif ($opt =~ m/^net(\d+)$/) {
1279             my $netid = $1;
1280             my $net = parse_lxc_network($value);
1281             if (!$running) {
1282                 $conf->{$opt} = print_lxc_network($net);
1283             } else {
1284                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1285             }
1286         } elsif ($opt eq 'protection') {
1287             $conf->{$opt} = $value ? 1 : 0;
1288         } elsif ($opt =~ m/^mp(\d+)$/) {
1289             next if $hotplug_error->($opt);
1290             check_protection($conf, "can't update CT $vmid drive '$opt'");
1291             $conf->{$opt} = $value;
1292             $new_disks = 1;
1293         } elsif ($opt eq 'rootfs') {
1294             check_protection($conf, "can't update CT $vmid drive '$opt'");
1295             die "implement me: $opt";
1296         } else {
1297             die "implement me: $opt";
1298         }
1299         write_config($vmid, $conf) if $running;
1300     }
1301
1302     if (@deleted_volumes) {
1303         my $storage_cfg = PVE::Storage::config();
1304         foreach my $volume (@deleted_volumes) {
1305             delete_mountpoint_volume($storage_cfg, $vmid, $volume);
1306         }
1307     }
1308
1309     if ($new_disks) {
1310         my $storage_cfg = PVE::Storage::config();
1311         create_disks($storage_cfg, $vmid, $conf, $conf);
1312     }
1313
1314     # This should be the last thing we do here
1315     if ($running && scalar(@nohotplug)) {
1316         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1317     }
1318 }
1319
1320 sub has_dev_console {
1321     my ($conf) = @_;
1322
1323     return !(defined($conf->{console}) && !$conf->{console});
1324 }
1325
1326 sub get_tty_count {
1327     my ($conf) = @_;
1328
1329     return $conf->{tty} // $confdesc->{tty}->{default};
1330 }
1331
1332 sub get_cmode {
1333     my ($conf) = @_;
1334
1335     return $conf->{cmode} // $confdesc->{cmode}->{default};
1336 }
1337
1338 sub get_console_command {
1339     my ($vmid, $conf) = @_;
1340
1341     my $cmode = get_cmode($conf);
1342
1343     if ($cmode eq 'console') {
1344         return ['lxc-console', '-n',  $vmid, '-t', 0];
1345     } elsif ($cmode eq 'tty') {
1346         return ['lxc-console', '-n',  $vmid];
1347     } elsif ($cmode eq 'shell') {
1348         return ['lxc-attach', '--clear-env', '-n', $vmid];
1349     } else {
1350         die "internal error";
1351     }
1352 }
1353
1354 sub get_primary_ips {
1355     my ($conf) = @_;
1356
1357     # return data from net0
1358
1359     return undef if !defined($conf->{net0});
1360     my $net = parse_lxc_network($conf->{net0});
1361
1362     my $ipv4 = $net->{ip};
1363     if ($ipv4) {
1364         if ($ipv4 =~ /^(dhcp|manual)$/) {
1365             $ipv4 = undef
1366         } else {
1367             $ipv4 =~ s!/\d+$!!;
1368         }
1369     }
1370     my $ipv6 = $net->{ip6};
1371     if ($ipv6) {
1372         if ($ipv6 =~ /^(auto|dhcp|manual)$/) {
1373             $ipv6 = undef;
1374         } else {
1375             $ipv6 =~ s!/\d+$!!;
1376         }
1377     }
1378
1379     return ($ipv4, $ipv6);
1380 }
1381
1382 sub delete_mountpoint_volume {
1383     my ($storage_cfg, $vmid, $volume) = @_;
1384
1385     # skip bind mounts and block devices
1386     if ($volume =~ m|^/|) {
1387             return;
1388     }
1389
1390     my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $volume);
1391     PVE::Storage::vdisk_free($storage_cfg, $volume) if $vmid == $owner;
1392 }
1393
1394 sub destroy_lxc_container {
1395     my ($storage_cfg, $vmid, $conf) = @_;
1396
1397     foreach_mountpoint($conf, sub {
1398         my ($ms, $mountpoint) = @_;
1399         delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
1400     });
1401
1402     rmdir "/var/lib/lxc/$vmid/rootfs";
1403     unlink "/var/lib/lxc/$vmid/config";
1404     rmdir "/var/lib/lxc/$vmid";
1405     destroy_config($vmid);
1406
1407     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1408     #PVE::Tools::run_command($cmd);
1409 }
1410
1411 sub vm_stop_cleanup {
1412     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1413
1414     eval {
1415         if (!$keepActive) {
1416
1417             my $vollist = get_vm_volumes($conf);
1418             PVE::Storage::deactivate_volumes($storage_cfg, $vollist);
1419         }
1420     };
1421     warn $@ if $@; # avoid errors - just warn
1422 }
1423
1424 my $safe_num_ne = sub {
1425     my ($a, $b) = @_;
1426
1427     return 0 if !defined($a) && !defined($b);
1428     return 1 if !defined($a);
1429     return 1 if !defined($b);
1430
1431     return $a != $b;
1432 };
1433
1434 my $safe_string_ne = sub {
1435     my ($a, $b) = @_;
1436
1437     return 0 if !defined($a) && !defined($b);
1438     return 1 if !defined($a);
1439     return 1 if !defined($b);
1440
1441     return $a ne $b;
1442 };
1443
1444 sub update_net {
1445     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1446
1447     if ($newnet->{type} ne 'veth') {
1448         # for when there are physical interfaces
1449         die "cannot update interface of type $newnet->{type}";
1450     }
1451
1452     my $veth = "veth${vmid}i${netid}";
1453     my $eth = $newnet->{name};
1454
1455     if (my $oldnetcfg = $conf->{$opt}) {
1456         my $oldnet = parse_lxc_network($oldnetcfg);
1457
1458         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1459             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1460
1461             PVE::Network::veth_delete($veth);
1462             delete $conf->{$opt};
1463             write_config($vmid, $conf);
1464
1465             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1466
1467         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1468                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1469                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1470
1471                 if ($oldnet->{bridge}) {
1472                     PVE::Network::tap_unplug($veth);
1473                     foreach (qw(bridge tag firewall)) {
1474                         delete $oldnet->{$_};
1475                     }
1476                     $conf->{$opt} = print_lxc_network($oldnet);
1477                     write_config($vmid, $conf);
1478                 }
1479
1480                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1481                 foreach (qw(bridge tag firewall)) {
1482                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1483                 }
1484                 $conf->{$opt} = print_lxc_network($oldnet);
1485                 write_config($vmid, $conf);
1486         }
1487     } else {
1488         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1489     }
1490
1491     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1492 }
1493
1494 sub hotplug_net {
1495     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1496
1497     my $veth = "veth${vmid}i${netid}";
1498     my $vethpeer = $veth . "p";
1499     my $eth = $newnet->{name};
1500
1501     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1502     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1503
1504     # attach peer in container
1505     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1506     PVE::Tools::run_command($cmd);
1507
1508     # link up peer in container
1509     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1510     PVE::Tools::run_command($cmd);
1511
1512     my $done = { type => 'veth' };
1513     foreach (qw(bridge tag firewall hwaddr name)) {
1514         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1515     }
1516     $conf->{$opt} = print_lxc_network($done);
1517
1518     write_config($vmid, $conf);
1519 }
1520
1521 sub update_ipconfig {
1522     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1523
1524     my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1525
1526     my $optdata = parse_lxc_network($conf->{$opt});
1527     my $deleted = [];
1528     my $added = [];
1529     my $nscmd = sub {
1530         my $cmdargs = shift;
1531         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1532     };
1533     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1534
1535     my $change_ip_config = sub {
1536         my ($ipversion) = @_;
1537
1538         my $family_opt = "-$ipversion";
1539         my $suffix = $ipversion == 4 ? '' : $ipversion;
1540         my $gw= "gw$suffix";
1541         my $ip= "ip$suffix";
1542
1543         my $newip = $newnet->{$ip};
1544         my $newgw = $newnet->{$gw};
1545         my $oldip = $optdata->{$ip};
1546
1547         my $change_ip = &$safe_string_ne($oldip, $newip);
1548         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1549
1550         return if !$change_ip && !$change_gw;
1551
1552         # step 1: add new IP, if this fails we cancel
1553         my $is_real_ip = ($newip && $newip !~ /^(?:auto|dhcp|manual)$/);
1554         if ($change_ip && $is_real_ip) {
1555             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1556             if (my $err = $@) {
1557                 warn $err;
1558                 return;
1559             }
1560         }
1561
1562         # step 2: replace gateway
1563         #   If this fails we delete the added IP and cancel.
1564         #   If it succeeds we save the config and delete the old IP, ignoring
1565         #   errors. The config is then saved.
1566         # Note: 'ip route replace' can add
1567         if ($change_gw) {
1568             if ($newgw) {
1569                 eval {
1570                     if ($is_real_ip && !PVE::Network::is_ip_in_cidr($newgw, $newip, $ipversion)) {
1571                         &$ipcmd($family_opt, 'route', 'add', $newgw, 'dev', $eth);
1572                     }
1573                     &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw);
1574                 };
1575                 if (my $err = $@) {
1576                     warn $err;
1577                     # the route was not replaced, the old IP is still available
1578                     # rollback (delete new IP) and cancel
1579                     if ($change_ip) {
1580                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1581                         warn $@ if $@; # no need to die here
1582                     }
1583                     return;
1584                 }
1585             } else {
1586                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1587                 # if the route was not deleted, the guest might have deleted it manually
1588                 # warn and continue
1589                 warn $@ if $@;
1590             }
1591         }
1592
1593         # from this point on we save the configuration
1594         # step 3: delete old IP ignoring errors
1595         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1596             # We need to enable promote_secondaries, otherwise our newly added
1597             # address will be removed along with the old one.
1598             my $promote = 0;
1599             eval {
1600                 if ($ipversion == 4) {
1601                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1602                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1603                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1604                 }
1605                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1606             };
1607             warn $@ if $@; # no need to die here
1608
1609             if ($ipversion == 4) {
1610                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1611             }
1612         }
1613
1614         foreach my $property ($ip, $gw) {
1615             if ($newnet->{$property}) {
1616                 $optdata->{$property} = $newnet->{$property};
1617             } else {
1618                 delete $optdata->{$property};
1619             }
1620         }
1621         $conf->{$opt} = print_lxc_network($optdata);
1622         write_config($vmid, $conf);
1623         $lxc_setup->setup_network($conf);
1624     };
1625
1626     &$change_ip_config(4);
1627     &$change_ip_config(6);
1628
1629 }
1630
1631 # Internal snapshots
1632
1633 # NOTE: Snapshot create/delete involves several non-atomic
1634 # action, and can take a long time.
1635 # So we try to avoid locking the file and use 'lock' variable
1636 # inside the config file instead.
1637
1638 my $snapshot_copy_config = sub {
1639     my ($source, $dest) = @_;
1640
1641     foreach my $k (keys %$source) {
1642         next if $k eq 'snapshots';
1643         next if $k eq 'snapstate';
1644         next if $k eq 'snaptime';
1645         next if $k eq 'vmstate';
1646         next if $k eq 'lock';
1647         next if $k eq 'digest';
1648         next if $k eq 'description';
1649
1650         $dest->{$k} = $source->{$k};
1651     }
1652 };
1653
1654 my $snapshot_prepare = sub {
1655     my ($vmid, $snapname, $comment) = @_;
1656
1657     my $snap;
1658
1659     my $updatefn =  sub {
1660
1661         my $conf = load_config($vmid);
1662
1663         die "you can't take a snapshot if it's a template\n"
1664             if is_template($conf);
1665
1666         check_lock($conf);
1667
1668         $conf->{lock} = 'snapshot';
1669
1670         die "snapshot name '$snapname' already used\n"
1671             if defined($conf->{snapshots}->{$snapname});
1672
1673         my $storecfg = PVE::Storage::config();
1674         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1675
1676         $snap = $conf->{snapshots}->{$snapname} = {};
1677
1678         &$snapshot_copy_config($conf, $snap);
1679
1680         $snap->{'snapstate'} = "prepare";
1681         $snap->{'snaptime'} = time();
1682         $snap->{'description'} = $comment if $comment;
1683         $conf->{snapshots}->{$snapname} = $snap;
1684
1685         write_config($vmid, $conf);
1686     };
1687
1688     lock_container($vmid, 10, $updatefn);
1689
1690     return $snap;
1691 };
1692
1693 my $snapshot_commit = sub {
1694     my ($vmid, $snapname) = @_;
1695
1696     my $updatefn = sub {
1697
1698         my $conf = load_config($vmid);
1699
1700         die "missing snapshot lock\n"
1701             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1702
1703         die "snapshot '$snapname' does not exist\n"
1704             if !defined($conf->{snapshots}->{$snapname});
1705
1706         die "wrong snapshot state\n"
1707             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1708                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1709
1710         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1711         delete $conf->{lock};
1712         $conf->{parent} = $snapname;
1713
1714         write_config($vmid, $conf);
1715     };
1716
1717     lock_container($vmid, 10 ,$updatefn);
1718 };
1719
1720 sub has_feature {
1721     my ($feature, $conf, $storecfg, $snapname) = @_;
1722
1723     my $err;
1724
1725     foreach_mountpoint($conf, sub {
1726         my ($ms, $mountpoint) = @_;
1727
1728         return if $err; # skip further test
1729
1730         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $mountpoint->{volume}, $snapname);
1731
1732         # TODO: implement support for mountpoints
1733         die "unable to handle mountpoint '$ms' - feature not implemented\n"
1734             if $ms ne 'rootfs';
1735     });
1736
1737     return $err ? 0 : 1;
1738 }
1739
1740 sub snapshot_create {
1741     my ($vmid, $snapname, $comment) = @_;
1742
1743     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1744
1745     my $conf = load_config($vmid);
1746
1747     my $running = check_running($vmid);
1748     eval {
1749         if ($running) {
1750             PVE::Tools::run_command(['/usr/bin/lxc-freeze', '-n', $vmid]);
1751             PVE::Tools::run_command(['/bin/sync']);
1752         };
1753
1754         my $storecfg = PVE::Storage::config();
1755         my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1756         my $volid = $rootinfo->{volume};
1757
1758         if ($running) {
1759             PVE::Tools::run_command(['/usr/bin/lxc-unfreeze', '-n', $vmid]);
1760         };
1761
1762         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1763         &$snapshot_commit($vmid, $snapname);
1764     };
1765     if(my $err = $@) {
1766         snapshot_delete($vmid, $snapname, 1);
1767         die "$err\n";
1768     }
1769 }
1770
1771 sub snapshot_delete {
1772     my ($vmid, $snapname, $force) = @_;
1773
1774     my $snap;
1775
1776     my $conf;
1777
1778     my $updatefn =  sub {
1779
1780         $conf = load_config($vmid);
1781
1782         die "you can't delete a snapshot if vm is a template\n"
1783             if is_template($conf);
1784
1785         $snap = $conf->{snapshots}->{$snapname};
1786
1787         check_lock($conf);
1788
1789         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1790
1791         $snap->{snapstate} = 'delete';
1792
1793         write_config($vmid, $conf);
1794     };
1795
1796     lock_container($vmid, 10, $updatefn);
1797
1798     my $storecfg = PVE::Storage::config();
1799
1800     my $del_snap =  sub {
1801
1802         check_lock($conf);
1803
1804         if ($conf->{parent} eq $snapname) {
1805             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1806                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1807             } else {
1808                 delete $conf->{parent};
1809             }
1810         }
1811
1812         delete $conf->{snapshots}->{$snapname};
1813
1814         write_config($vmid, $conf);
1815     };
1816
1817     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1818     my $rootinfo = parse_ct_mountpoint($rootfs);
1819     my $volid = $rootinfo->{volume};
1820
1821     eval {
1822         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1823     };
1824     my $err = $@;
1825
1826     if(!$err || ($err && $force)) {
1827         lock_container($vmid, 10, $del_snap);
1828         if ($err) {
1829             die "Can't delete snapshot: $vmid $snapname $err\n";
1830         }
1831     }
1832 }
1833
1834 sub snapshot_rollback {
1835     my ($vmid, $snapname) = @_;
1836
1837     my $storecfg = PVE::Storage::config();
1838
1839     my $conf = load_config($vmid);
1840
1841     die "you can't rollback if vm is a template\n" if is_template($conf);
1842
1843     my $snap = $conf->{snapshots}->{$snapname};
1844
1845     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1846
1847     my $rootfs = $snap->{rootfs};
1848     my $rootinfo = parse_ct_mountpoint($rootfs);
1849     my $volid = $rootinfo->{volume};
1850
1851     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1852
1853     my $updatefn = sub {
1854
1855         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1856             if $snap->{snapstate};
1857
1858         check_lock($conf);
1859
1860         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1861
1862         die "unable to rollback vm $vmid: vm is running\n"
1863             if check_running($vmid);
1864
1865         $conf->{lock} = 'rollback';
1866
1867         my $forcemachine;
1868
1869         # copy snapshot config to current config
1870
1871         my $tmp_conf = $conf;
1872         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1873         $conf->{snapshots} = $tmp_conf->{snapshots};
1874         delete $conf->{snaptime};
1875         delete $conf->{snapname};
1876         $conf->{parent} = $snapname;
1877
1878         write_config($vmid, $conf);
1879     };
1880
1881     my $unlockfn = sub {
1882         delete $conf->{lock};
1883         write_config($vmid, $conf);
1884     };
1885
1886     lock_container($vmid, 10, $updatefn);
1887
1888     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1889
1890     lock_container($vmid, 5, $unlockfn);
1891 }
1892
1893 sub template_create {
1894     my ($vmid, $conf) = @_;
1895
1896     my $storecfg = PVE::Storage::config();
1897
1898     my $rootinfo = parse_ct_mountpoint($conf->{rootfs});
1899     my $volid = $rootinfo->{volume};
1900
1901     die "Template feature is not available for '$volid'\n"
1902         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1903
1904     PVE::Storage::activate_volumes($storecfg, [$volid]);
1905
1906     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1907     $rootinfo->{volume} = $template_volid;
1908     $conf->{rootfs} = print_ct_mountpoint($rootinfo, 1);
1909
1910     write_config($vmid, $conf);
1911 }
1912
1913 sub is_template {
1914     my ($conf) = @_;
1915
1916     return 1 if defined $conf->{template} && $conf->{template} == 1;
1917 }
1918
1919 sub mountpoint_names {
1920     my ($reverse) = @_;
1921
1922     my @names = ('rootfs');
1923
1924     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1925         push @names, "mp$i";
1926     }
1927
1928     return $reverse ? reverse @names : @names;
1929 }
1930
1931 # The container might have *different* symlinks than the host. realpath/abs_path
1932 # use the actual filesystem to resolve links.
1933 sub sanitize_mountpoint {
1934     my ($mp) = @_;
1935     $mp = '/' . $mp; # we always start with a slash
1936     $mp =~ s@/{2,}@/@g; # collapse sequences of slashes
1937     $mp =~ s@/\./@@g; # collapse /./
1938     $mp =~ s@/\.(/)?$@$1@; # collapse a trailing /. or /./
1939     $mp =~ s@(.*)/[^/]+/\.\./@$1/@g; # collapse /../ without regard for symlinks
1940     $mp =~ s@/\.\.(/)?$@$1@; # collapse trailing /.. or /../ disregarding symlinks
1941     return $mp;
1942 }
1943
1944 sub foreach_mountpoint_full {
1945     my ($conf, $reverse, $func) = @_;
1946
1947     foreach my $key (mountpoint_names($reverse)) {
1948         my $value = $conf->{$key};
1949         next if !defined($value);
1950         my $mountpoint = parse_ct_mountpoint($value, 1);
1951         next if !defined($mountpoint);
1952
1953         # just to be sure: rootfs is /
1954         my $path = $key eq 'rootfs' ? '/' : $mountpoint->{mp};
1955         $mountpoint->{mp} = sanitize_mountpoint($path);
1956
1957         $path = $mountpoint->{volume};
1958         $mountpoint->{volume} = sanitize_mountpoint($path) if $path =~ m|^/|;
1959
1960         &$func($key, $mountpoint);
1961     }
1962 }
1963
1964 sub foreach_mountpoint {
1965     my ($conf, $func) = @_;
1966
1967     foreach_mountpoint_full($conf, 0, $func);
1968 }
1969
1970 sub foreach_mountpoint_reverse {
1971     my ($conf, $func) = @_;
1972
1973     foreach_mountpoint_full($conf, 1, $func);
1974 }
1975
1976 sub check_ct_modify_config_perm {
1977     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
1978
1979     return 1 if $authuser ne 'root@pam';
1980
1981     foreach my $opt (@$key_list) {
1982
1983         if ($opt eq 'cpus' || $opt eq 'cpuunits' || $opt eq 'cpulimit') {
1984             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
1985         } elsif ($opt eq 'rootfs' || $opt =~ /^mp\d+$/) {
1986             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
1987         } elsif ($opt eq 'memory' || $opt eq 'swap') {
1988             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
1989         } elsif ($opt =~ m/^net\d+$/ || $opt eq 'nameserver' ||
1990                  $opt eq 'searchdomain' || $opt eq 'hostname') {
1991             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
1992         } else {
1993             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
1994         }
1995     }
1996
1997     return 1;
1998 }
1999
2000 sub umount_all {
2001     my ($vmid, $storage_cfg, $conf, $noerr) = @_;
2002
2003     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2004     my $volid_list = get_vm_volumes($conf);
2005
2006     foreach_mountpoint_reverse($conf, sub {
2007         my ($ms, $mountpoint) = @_;
2008
2009         my $volid = $mountpoint->{volume};
2010         my $mount = $mountpoint->{mp};
2011
2012         return if !$volid || !$mount;
2013
2014         my $mount_path = "$rootdir/$mount";
2015         $mount_path =~ s!/+!/!g;
2016
2017         return if !PVE::ProcFSTools::is_mounted($mount_path);
2018
2019         eval {
2020             PVE::Tools::run_command(['umount', '-d', $mount_path]);
2021         };
2022         if (my $err = $@) {
2023             if ($noerr) {
2024                 warn $err;
2025             } else {
2026                 die $err;
2027             }
2028         }
2029     });
2030 }
2031
2032 sub mount_all {
2033     my ($vmid, $storage_cfg, $conf) = @_;
2034
2035     my $rootdir = "/var/lib/lxc/$vmid/rootfs";
2036     File::Path::make_path($rootdir);
2037
2038     my $volid_list = get_vm_volumes($conf);
2039     PVE::Storage::activate_volumes($storage_cfg, $volid_list);
2040
2041     eval {
2042         foreach_mountpoint($conf, sub {
2043             my ($ms, $mountpoint) = @_;
2044
2045             my $volid = $mountpoint->{volume};
2046             my $mount = $mountpoint->{mp};
2047
2048             return if !$volid || !$mount;
2049
2050             my $image_path = PVE::Storage::path($storage_cfg, $volid);
2051             my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2052                 PVE::Storage::parse_volname($storage_cfg, $volid);
2053
2054             die "unable to mount base volume - internal error" if $isBase;
2055
2056             mountpoint_mount($mountpoint, $rootdir, $storage_cfg);
2057         });
2058     };
2059     if (my $err = $@) {
2060         warn "mounting container failed - $err";
2061         umount_all($vmid, $storage_cfg, $conf, 1);
2062     }
2063
2064     return $rootdir;
2065 }
2066
2067
2068 sub mountpoint_mount_path {
2069     my ($mountpoint, $storage_cfg, $snapname) = @_;
2070
2071     return mountpoint_mount($mountpoint, undef, $storage_cfg, $snapname);
2072 }
2073
2074 my $check_mount_path = sub {
2075     my ($path) = @_;
2076     $path = File::Spec->canonpath($path);
2077     my $real = Cwd::realpath($path);
2078     if ($real ne $path) {
2079         die "mount path modified by symlink: $path != $real";
2080     }
2081 };
2082
2083 # use $rootdir = undef to just return the corresponding mount path
2084 sub mountpoint_mount {
2085     my ($mountpoint, $rootdir, $storage_cfg, $snapname) = @_;
2086
2087     my $volid = $mountpoint->{volume};
2088     my $mount = $mountpoint->{mp};
2089
2090     return if !$volid || !$mount;
2091
2092     my $mount_path;
2093
2094     if (defined($rootdir)) {
2095         $rootdir =~ s!/+$!!;
2096         $mount_path = "$rootdir/$mount";
2097         $mount_path =~ s!/+!/!g;
2098         &$check_mount_path($mount_path);
2099         File::Path::mkpath($mount_path);
2100     }
2101
2102     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2103
2104     die "unknown snapshot path for '$volid'" if !$storage && defined($snapname);
2105
2106     if ($storage) {
2107
2108         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
2109         my $path = PVE::Storage::path($storage_cfg, $volid, $snapname);
2110
2111         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2112             PVE::Storage::parse_volname($storage_cfg, $volid);
2113
2114         $format = 'iso' if $vtype eq 'iso'; # allow to handle iso files
2115
2116         if ($format eq 'subvol') {
2117             if ($mount_path) {
2118                 if ($snapname) {
2119                     if ($scfg->{type} eq 'zfspool') {
2120                         my $path_arg = $path;
2121                         $path_arg =~ s!^/+!!;
2122                         PVE::Tools::run_command(['mount', '-o', 'ro', '-t', 'zfs', $path_arg, $mount_path]);
2123                     } else {
2124                         die "cannot mount subvol snapshots for storage type '$scfg->{type}'\n";
2125                     }
2126                 } else {
2127                     PVE::Tools::run_command(['mount', '-o', 'bind', $path, $mount_path]);
2128                 }
2129             }
2130             return wantarray ? ($path, 0) : $path;
2131         } elsif ($format eq 'raw' || $format eq 'iso') {
2132             my $use_loopdev = 0;
2133             my @extra_opts;
2134             if ($scfg->{path}) {
2135                 push @extra_opts, '-o', 'loop';
2136                 $use_loopdev = 1;
2137             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm' || $scfg->{type} eq 'rbd') {
2138                 # do nothing
2139             } else {
2140                 die "unsupported storage type '$scfg->{type}'\n";
2141             }
2142             if ($mount_path) {
2143                 if ($format eq 'iso') {
2144                     PVE::Tools::run_command(['mount', '-o', 'ro', @extra_opts, $path, $mount_path]);
2145                 } elsif ($isBase || defined($snapname)) {
2146                     PVE::Tools::run_command(['mount', '-o', 'ro,noload', @extra_opts, $path, $mount_path]);
2147                 } else {
2148                     PVE::Tools::run_command(['mount', @extra_opts, $path, $mount_path]);
2149                 }
2150             }
2151             return wantarray ? ($path, $use_loopdev) : $path;
2152         } else {
2153             die "unsupported image format '$format'\n";
2154         }
2155     } elsif ($volid =~ m|^/dev/.+|) {
2156         PVE::Tools::run_command(['mount', $volid, $mount_path]) if $mount_path;
2157         return wantarray ? ($volid, 0) : $volid;
2158     } elsif ($volid !~ m|^/dev/.+| && $volid =~ m|^/.+| && -d $volid) {
2159         &$check_mount_path($volid);
2160         PVE::Tools::run_command(['mount', '-o', 'bind', $volid, $mount_path]) if $mount_path;
2161         return wantarray ? ($volid, 0) : $volid;
2162     }
2163
2164     die "unsupported storage";
2165 }
2166
2167 sub get_vm_volumes {
2168     my ($conf, $excludes) = @_;
2169
2170     my $vollist = [];
2171
2172     foreach_mountpoint($conf, sub {
2173         my ($ms, $mountpoint) = @_;
2174
2175         return if $excludes && $ms eq $excludes;
2176
2177         my $volid = $mountpoint->{volume};
2178
2179         return if !$volid || $volid =~ m|^/|;
2180
2181         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2182         return if !$sid;
2183
2184         push @$vollist, $volid;
2185     });
2186
2187     return $vollist;
2188 }
2189
2190 sub mkfs {
2191     my ($dev) = @_;
2192
2193     PVE::Tools::run_command(['mkfs.ext4', '-O', 'mmp', $dev]);
2194 }
2195
2196 sub format_disk {
2197     my ($storage_cfg, $volid) = @_;
2198
2199     if ($volid =~ m!^/dev/.+!) {
2200         mkfs($volid);
2201         return;
2202     }
2203
2204     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2205
2206     die "cannot format volume '$volid' with no storage\n" if !$storage;
2207
2208     PVE::Storage::activate_volumes($storage_cfg, [$volid]);
2209
2210     my $path = PVE::Storage::path($storage_cfg, $volid);
2211
2212     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
2213         PVE::Storage::parse_volname($storage_cfg, $volid);
2214
2215     die "cannot format volume '$volid' (format == $format)\n"
2216         if $format ne 'raw';
2217
2218     mkfs($path);
2219 }
2220
2221 sub destroy_disks {
2222     my ($storecfg, $vollist) = @_;
2223
2224     foreach my $volid (@$vollist) {
2225         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
2226         warn $@ if $@;
2227     }
2228 }
2229
2230 sub create_disks {
2231     my ($storecfg, $vmid, $settings, $conf) = @_;
2232
2233     my $vollist = [];
2234
2235     eval {
2236         foreach_mountpoint($settings, sub {
2237             my ($ms, $mountpoint) = @_;
2238
2239             my $volid = $mountpoint->{volume};
2240             my $mp = $mountpoint->{mp};
2241
2242             my ($storage, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2243
2244             return if !$storage;
2245
2246             if ($volid =~ m/^([^:\s]+):(\d+(\.\d+)?)$/) {
2247                 my ($storeid, $size_gb) = ($1, $2);
2248
2249                 my $size_kb = int(${size_gb}*1024) * 1024;
2250
2251                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
2252                 # fixme: use better naming ct-$vmid-disk-X.raw?
2253
2254                 if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
2255                     if ($size_kb > 0) {
2256                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw',
2257                                                            undef, $size_kb);
2258                         format_disk($storecfg, $volid);
2259                     } else {
2260                         $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2261                                                            undef, 0);
2262                     }
2263                 } elsif ($scfg->{type} eq 'zfspool') {
2264
2265                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'subvol',
2266                                                        undef, $size_kb);
2267                 } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'lvm') {
2268
2269                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2270                     format_disk($storecfg, $volid);
2271
2272                 } elsif ($scfg->{type} eq 'rbd') {
2273
2274                     die "krbd option must be enabled on storage type '$scfg->{type}'\n" if !$scfg->{krbd};
2275                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storage, $vmid, 'raw', undef, $size_kb);
2276                     format_disk($storecfg, $volid);
2277                 } else {
2278                     die "unable to create containers on storage type '$scfg->{type}'\n";
2279                 }
2280                 push @$vollist, $volid;
2281                 my $new_mountpoint = { volume => $volid, size => $size_kb*1024, mp => $mp };
2282                 $conf->{$ms} = print_ct_mountpoint($new_mountpoint, $ms eq 'rootfs');
2283             } else {
2284                 # use specified/existing volid
2285             }
2286         });
2287     };
2288     # free allocated images on error
2289     if (my $err = $@) {
2290         destroy_disks($storecfg, $vollist);
2291         die $err;
2292     }
2293     return $vollist;
2294 }
2295
2296 # bash completion helper
2297
2298 sub complete_os_templates {
2299     my ($cmdname, $pname, $cvalue) = @_;
2300
2301     my $cfg = PVE::Storage::config();
2302
2303     my $storeid;
2304
2305     if ($cvalue =~ m/^([^:]+):/) {
2306         $storeid = $1;
2307     }
2308
2309     my $vtype = $cmdname eq 'restore' ? 'backup' : 'vztmpl';
2310     my $data = PVE::Storage::template_list($cfg, $storeid, $vtype);
2311
2312     my $res = [];
2313     foreach my $id (keys %$data) {
2314         foreach my $item (@{$data->{$id}}) {
2315             push @$res, $item->{volid} if defined($item->{volid});
2316         }
2317     }
2318
2319     return $res;
2320 }
2321
2322 my $complete_ctid_full = sub {
2323     my ($running) = @_;
2324
2325     my $idlist = vmstatus();
2326
2327     my $active_hash = list_active_containers();
2328
2329     my $res = [];
2330
2331     foreach my $id (keys %$idlist) {
2332         my $d = $idlist->{$id};
2333         if (defined($running)) {
2334             next if $d->{template};
2335             next if $running && !$active_hash->{$id};
2336             next if !$running && $active_hash->{$id};
2337         }
2338         push @$res, $id;
2339
2340     }
2341     return $res;
2342 };
2343
2344 sub complete_ctid {
2345     return &$complete_ctid_full();
2346 }
2347
2348 sub complete_ctid_stopped {
2349     return &$complete_ctid_full(0);
2350 }
2351
2352 sub complete_ctid_running {
2353     return &$complete_ctid_full(1);
2354 }
2355
2356 1;