src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use Fcntl ':flock';
   9
  10 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  11 use PVE::Storage;
  12 use PVE::SafeSyslog;
  13 use PVE::INotify;
  14 use PVE::JSONSchema qw(get_standard_option);
  15 use PVE::Tools qw($IPV6RE $IPV4RE dir_glob_foreach);
  16 use PVE::Network;
  17
  18 use Data::Dumper;
  19
  20 my $nodename = PVE::INotify::nodename();
  21
  22 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  23
  24 PVE::JSONSchema::register_format('pve-lxc-network', \&verify_lxc_network);
  25 sub verify_lxc_network {
  26     my ($value, $noerr) = @_;
  27
  28     return $value if parse_lxc_network($value);
  29
  30     return undef if $noerr;
  31
  32     die "unable to parse network setting\n";
  33 }
  34
  35 PVE::JSONSchema::register_format('pve-ct-mountpoint', \&verify_ct_mountpoint);
  36 sub verify_ct_mountpoint {
  37     my ($value, $noerr) = @_;
  38
  39     return $value if parse_ct_mountpoint($value);
  40
  41     return undef if $noerr;
  42
  43     die "unable to parse CT mountpoint options\n";
  44 }
  45
  46 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  47     type => 'string', format => 'pve-ct-mountpoint',
  48     typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+]',
  49     description => "Use volume as container root.",
  50     optional => 1,
  51 });
  52
  53 my $confdesc = {
  54     lock => {
  55         optional => 1,
  56         type => 'string',
  57         description => "Lock/unlock the VM.",
  58         enum => [qw(migrate backup snapshot rollback)],
  59     },
  60     onboot => {
  61         optional => 1,
  62         type => 'boolean',
  63         description => "Specifies whether a VM will be started during system bootup.",
  64         default => 0,
  65     },
  66     startup => get_standard_option('pve-startup-order'),
  67     template => {
  68         optional => 1,
  69         type => 'boolean',
  70         description => "Enable/disable Template.",
  71         default => 0,
  72     },
  73     arch => {
  74         optional => 1,
  75         type => 'string',
  76         enum => ['amd64', 'i386'],
  77         description => "OS architecture type.",
  78         default => 'amd64',
  79     },
  80     ostype => {
  81         optional => 1,
  82         type => 'string',
  83         enum => ['debian', 'ubuntu', 'centos'],
  84         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  85     },
  86     console => {
  87         optional => 1,
  88         type => 'boolean',
  89         description => "Attach a console device (/dev/console) to the container.",
  90         default => 1,
  91     },
  92     tty => {
  93         optional => 1,
  94         type => 'integer',
  95         description => "Specify the number of tty available to the container",
  96         minimum => 0,
  97         maximum => 6,
  98         default => 2,
  99     },
 100     cpulimit => {
 101         optional => 1,
 102         type => 'number',
 103         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 104         minimum => 0,
 105         maximum => 128,
 106         default => 0,
 107     },
 108     cpuunits => {
 109         optional => 1,
 110         type => 'integer',
 111         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 112         minimum => 0,
 113         maximum => 500000,
 114         default => 1024,
 115     },
 116     memory => {
 117         optional => 1,
 118         type => 'integer',
 119         description => "Amount of RAM for the VM in MB.",
 120         minimum => 16,
 121         default => 512,
 122     },
 123     swap => {
 124         optional => 1,
 125         type => 'integer',
 126         description => "Amount of SWAP for the VM in MB.",
 127         minimum => 0,
 128         default => 512,
 129     },
 130     hostname => {
 131         optional => 1,
 132         description => "Set a host name for the container.",
 133         type => 'string',
 134         maxLength => 255,
 135     },
 136     description => {
 137         optional => 1,
 138         type => 'string',
 139         description => "Container description. Only used on the configuration web interface.",
 140     },
 141     searchdomain => {
 142         optional => 1,
 143         type => 'string',
 144         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 145     },
 146     nameserver => {
 147         optional => 1,
 148         type => 'string',
 149         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 150     },
 151     rootfs => get_standard_option('pve-ct-rootfs'),
 152     parent => {
 153         optional => 1,
 154         type => 'string', format => 'pve-configid',
 155         maxLength => 40,
 156         description => "Parent snapshot name. This is used internally, and should not be modified.",
 157     },
 158     snaptime => {
 159         optional => 1,
 160         description => "Timestamp for snapshots.",
 161         type => 'integer',
 162         minimum => 0,
 163     },
 164     cmode => {
 165         optional => 1,
 166         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 167         type => 'string',
 168         enum => ['shell', 'console', 'tty'],
 169         default => 'tty',
 170     },
 171 };
 172
 173 my $valid_lxc_conf_keys = {
 174     'lxc.include' => 1,
 175     'lxc.arch' => 1,
 176     'lxc.utsname' => 1,
 177     'lxc.haltsignal' => 1,
 178     'lxc.rebootsignal' => 1,
 179     'lxc.stopsignal' => 1,
 180     'lxc.init_cmd' => 1,
 181     'lxc.network.type' => 1,
 182     'lxc.network.flags' => 1,
 183     'lxc.network.link' => 1,
 184     'lxc.network.mtu' => 1,
 185     'lxc.network.name' => 1,
 186     'lxc.network.hwaddr' => 1,
 187     'lxc.network.ipv4' => 1,
 188     'lxc.network.ipv4.gateway' => 1,
 189     'lxc.network.ipv6' => 1,
 190     'lxc.network.ipv6.gateway' => 1,
 191     'lxc.network.script.up' => 1,
 192     'lxc.network.script.down' => 1,
 193     'lxc.pts' => 1,
 194     'lxc.console.logfile' => 1,
 195     'lxc.console' => 1,
 196     'lxc.tty' => 1,
 197     'lxc.devttydir' => 1,
 198     'lxc.hook.autodev' => 1,
 199     'lxc.autodev' => 1,
 200     'lxc.kmsg' => 1,
 201     'lxc.mount' => 1,
 202     'lxc.mount.entry' => 1,
 203     'lxc.mount.auto' => 1,
 204     'lxc.rootfs' => 1,
 205     'lxc.rootfs.mount' => 1,
 206     'lxc.rootfs.options' => 1,
 207     # lxc.cgroup.*
 208     'lxc.cap.drop' => 1,
 209     'lxc.cap.keep' => 1,
 210     'lxc.aa_profile' => 1,
 211     'lxc.aa_allow_incomplete' => 1,
 212     'lxc.se_context' => 1,
 213     'lxc.seccomp' => 1,
 214     'lxc.id_map' => 1,
 215     'lxc.hook.pre-start' => 1,
 216     'lxc.hook.pre-mount' => 1,
 217     'lxc.hook.mount' => 1,
 218     'lxc.hook.start' => 1,
 219     'lxc.hook.post-stop' => 1,
 220     'lxc.hook.clone' => 1,
 221     'lxc.hook.destroy' => 1,
 222     'lxc.loglevel' => 1,
 223     'lxc.logfile' => 1,
 224     'lxc.start.auto' => 1,
 225     'lxc.start.delay' => 1,
 226     'lxc.start.order' => 1,
 227     'lxc.group' => 1,
 228     'lxc.environment' => 1,
 229     'lxc.' => 1,
 230     'lxc.' => 1,
 231     'lxc.' => 1,
 232     'lxc.' => 1,
 233 };
 234
 235 my $MAX_LXC_NETWORKS = 10;
 236 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 237     $confdesc->{"net$i"} = {
 238         optional => 1,
 239         type => 'string', format => 'pve-lxc-network',
 240         description => "Specifies network interfaces for the container.\n\n".
 241             "The string should have the follow format:\n\n".
 242             "-net<[0-9]> bridge=<vmbr<Nummber>>[,hwaddr=<MAC>]\n".
 243             "[,mtu=<Number>][,name=<String>][,ip=<IPv4Format/CIDR>]\n".
 244             ",ip6=<IPv6Format/CIDR>][,gw=<GatwayIPv4>]\n".
 245             ",gw6=<GatwayIPv6>][,firewall=<[1|0]>][,tag=<VlanNo>]",
 246     };
 247 }
 248
 249 my $MAX_MOUNT_POINTS = 10;
 250 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 251     $confdesc->{"mp$i"} = {
 252         optional => 1,
 253         type => 'string', format => 'pve-ct-mountpoint',
 254         typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+] [,mp=mountpoint]',
 255         description => "Use volume as container mount point.",
 256         optional => 1,
 257     };
 258 }
 259
 260 sub write_pct_config {
 261     my ($filename, $conf) = @_;
 262
 263     delete $conf->{snapstate}; # just to be sure
 264
 265     my $generate_raw_config = sub {
 266         my ($conf) = @_;
 267
 268         my $raw = '';
 269
 270         # add description as comment to top of file
 271         my $descr = $conf->{description} || '';
 272         foreach my $cl (split(/\n/, $descr)) {
 273             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 274         }
 275
 276         foreach my $key (sort keys %$conf) {
 277             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 278                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 279             $raw .= "$key: $conf->{$key}\n";
 280         }
 281
 282         if (my $lxcconf = $conf->{lxc}) {
 283             foreach my $entry (@$lxcconf) {
 284                 my ($k, $v) = @$entry;
 285                 $raw .= "$k: $v\n";
 286             }
 287         }
 288
 289         return $raw;
 290     };
 291
 292     my $raw = &$generate_raw_config($conf);
 293
 294     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 295         $raw .= "\n[$snapname]\n";
 296         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 297     }
 298
 299     return $raw;
 300 }
 301
 302 sub check_type {
 303     my ($key, $value) = @_;
 304
 305     die "unknown setting '$key'\n" if !$confdesc->{$key};
 306
 307     my $type = $confdesc->{$key}->{type};
 308
 309     if (!defined($value)) {
 310         die "got undefined value\n";
 311     }
 312
 313     if ($value =~ m/[\n\r]/) {
 314         die "property contains a line feed\n";
 315     }
 316
 317     if ($type eq 'boolean') {
 318         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 319         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 320         die "type check ('boolean') failed - got '$value'\n";
 321     } elsif ($type eq 'integer') {
 322         return int($1) if $value =~ m/^(\d+)$/;
 323         die "type check ('integer') failed - got '$value'\n";
 324     } elsif ($type eq 'number') {
 325         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 326         die "type check ('number') failed - got '$value'\n";
 327     } elsif ($type eq 'string') {
 328         if (my $fmt = $confdesc->{$key}->{format}) {
 329             PVE::JSONSchema::check_format($fmt, $value);
 330             return $value;
 331         }
 332         return $value;
 333     } else {
 334         die "internal error"
 335     }
 336 }
 337
 338 sub parse_pct_config {
 339     my ($filename, $raw) = @_;
 340
 341     return undef if !defined($raw);
 342
 343     my $res = {
 344         digest => Digest::SHA::sha1_hex($raw),
 345         snapshots => {},
 346     };
 347
 348     $filename =~ m|/lxc/(\d+).conf$|
 349         || die "got strange filename '$filename'";
 350
 351     my $vmid = $1;
 352
 353     my $conf = $res;
 354     my $descr = '';
 355     my $section = '';
 356
 357     my @lines = split(/\n/, $raw);
 358     foreach my $line (@lines) {
 359         next if $line =~ m/^\s*$/;
 360
 361         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 362             $section = $1;
 363             $conf->{description} = $descr if $descr;
 364             $descr = '';
 365             $conf = $res->{snapshots}->{$section} = {};
 366             next;
 367         }
 368
 369         if ($line =~ m/^\#(.*)\s*$/) {
 370             $descr .= PVE::Tools::decode_text($1) . "\n";
 371             next;
 372         }
 373
 374         if ($line =~ m/^(lxc\.[a-z0-9\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 375             my $key = $1;
 376             my $value = $3;
 377             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 378                 push @{$conf->{lxc}}, [$key, $value];
 379             } else {
 380                 warn "vm $vmid - unable to parse config: $line\n";
 381             }
 382         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 383             $descr .= PVE::Tools::decode_text($2);
 384         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 385             $conf->{snapstate} = $1;
 386         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
 387             my $key = $1;
 388             my $value = $2;
 389             eval { $value = check_type($key, $value); };
 390             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 391             $conf->{$key} = $value;
 392         } else {
 393             warn "vm $vmid - unable to parse config: $line\n";
 394         }
 395     }
 396
 397     $conf->{description} = $descr if $descr;
 398
 399     delete $res->{snapstate}; # just to be sure
 400
 401     return $res;
 402 }
 403
 404 sub config_list {
 405     my $vmlist = PVE::Cluster::get_vmlist();
 406     my $res = {};
 407     return $res if !$vmlist || !$vmlist->{ids};
 408     my $ids = $vmlist->{ids};
 409
 410     foreach my $vmid (keys %$ids) {
 411         next if !$vmid; # skip CT0
 412         my $d = $ids->{$vmid};
 413         next if !$d->{node} || $d->{node} ne $nodename;
 414         next if !$d->{type} || $d->{type} ne 'lxc';
 415         $res->{$vmid}->{type} = 'lxc';
 416     }
 417     return $res;
 418 }
 419
 420 sub cfs_config_path {
 421     my ($vmid, $node) = @_;
 422
 423     $node = $nodename if !$node;
 424     return "nodes/$node/lxc/$vmid.conf";
 425 }
 426
 427 sub config_file {
 428     my ($vmid, $node) = @_;
 429
 430     my $cfspath = cfs_config_path($vmid, $node);
 431     return "/etc/pve/$cfspath";
 432 }
 433
 434 sub load_config {
 435     my ($vmid) = @_;
 436
 437     my $cfspath = cfs_config_path($vmid);
 438
 439     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 440     die "container $vmid does not exists\n" if !defined($conf);
 441
 442     return $conf;
 443 }
 444
 445 sub create_config {
 446     my ($vmid, $conf) = @_;
 447
 448     my $dir = "/etc/pve/nodes/$nodename/lxc";
 449     mkdir $dir;
 450
 451     write_config($vmid, $conf);
 452 }
 453
 454 sub destroy_config {
 455     my ($vmid) = @_;
 456
 457     unlink config_file($vmid, $nodename);
 458 }
 459
 460 sub write_config {
 461     my ($vmid, $conf) = @_;
 462
 463     my $cfspath = cfs_config_path($vmid);
 464
 465     PVE::Cluster::cfs_write_file($cfspath, $conf);
 466 }
 467
 468 # flock: we use one file handle per process, so lock file
 469 # can be called multiple times and succeeds for the same process.
 470
 471 my $lock_handles =  {};
 472 my $lockdir = "/run/lock/lxc";
 473
 474 sub lock_filename {
 475     my ($vmid) = @_;
 476
 477     return "$lockdir/pve-config-{$vmid}.lock";
 478 }
 479
 480 sub lock_aquire {
 481     my ($vmid, $timeout) = @_;
 482
 483     $timeout = 10 if !$timeout;
 484     my $mode = LOCK_EX;
 485
 486     my $filename = lock_filename($vmid);
 487
 488     mkdir $lockdir if !-d $lockdir;
 489
 490     my $lock_func = sub {
 491         if (!$lock_handles->{$$}->{$filename}) {
 492             my $fh = new IO::File(">>$filename") ||
 493                 die "can't open file - $!\n";
 494             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 495         }
 496
 497         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 498             print STDERR "trying to aquire lock...";
 499             my $success;
 500             while(1) {
 501                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 502                 # try again on EINTR (see bug #273)
 503                 if ($success || ($! != EINTR)) {
 504                     last;
 505                 }
 506             }
 507             if (!$success) {
 508                 print STDERR " failed\n";
 509                 die "can't aquire lock - $!\n";
 510             }
 511
 512             $lock_handles->{$$}->{$filename}->{refcount}++;
 513
 514             print STDERR " OK\n";
 515         }
 516     };
 517
 518     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 519     my $err = $@;
 520     if ($err) {
 521         die "can't lock file '$filename' - $err";
 522     }
 523 }
 524
 525 sub lock_release {
 526     my ($vmid) = @_;
 527
 528     my $filename = lock_filename($vmid);
 529
 530     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 531         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 532         if ($refcount <= 0) {
 533             $lock_handles->{$$}->{$filename} = undef;
 534             close ($fh);
 535         }
 536     }
 537 }
 538
 539 sub lock_container {
 540     my ($vmid, $timeout, $code, @param) = @_;
 541
 542     my $res;
 543
 544     lock_aquire($vmid, $timeout);
 545     eval { $res = &$code(@param) };
 546     my $err = $@;
 547     lock_release($vmid);
 548
 549     die $err if $err;
 550
 551     return $res;
 552 }
 553
 554 sub option_exists {
 555     my ($name) = @_;
 556
 557     return defined($confdesc->{$name});
 558 }
 559
 560 # add JSON properties for create and set function
 561 sub json_config_properties {
 562     my $prop = shift;
 563
 564     foreach my $opt (keys %$confdesc) {
 565         next if $opt eq 'parent' || $opt eq 'snaptime';
 566         next if $prop->{$opt};
 567         $prop->{$opt} = $confdesc->{$opt};
 568     }
 569
 570     return $prop;
 571 }
 572
 573 sub json_config_properties_no_rootfs {
 574     my $prop = shift;
 575
 576     foreach my $opt (keys %$confdesc) {
 577         next if $prop->{$opt};
 578         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 579         $prop->{$opt} = $confdesc->{$opt};
 580     }
 581
 582     return $prop;
 583 }
 584
 585 # container status helpers
 586
 587 sub list_active_containers {
 588
 589     my $filename = "/proc/net/unix";
 590
 591     # similar test is used by lcxcontainers.c: list_active_containers
 592     my $res = {};
 593
 594     my $fh = IO::File->new ($filename, "r");
 595     return $res if !$fh;
 596
 597     while (defined(my $line = <$fh>)) {
 598         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 599             my $path = $1;
 600             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 601                 $res->{$1} = 1;
 602             }
 603         }
 604     }
 605
 606     close($fh);
 607
 608     return $res;
 609 }
 610
 611 # warning: this is slow
 612 sub check_running {
 613     my ($vmid) = @_;
 614
 615     my $active_hash = list_active_containers();
 616
 617     return 1 if defined($active_hash->{$vmid});
 618
 619     return undef;
 620 }
 621
 622 sub get_container_disk_usage {
 623     my ($vmid) = @_;
 624
 625     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 626
 627     my $res = {
 628         total => 0,
 629         used => 0,
 630         avail => 0,
 631     };
 632
 633     my $parser = sub {
 634         my $line = shift;
 635         if (my ($fsid, $total, $used, $avail) = $line =~
 636             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 637             $res = {
 638                 total => $total,
 639                 used => $used,
 640                 avail => $avail,
 641             };
 642         }
 643     };
 644     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 645     warn $@ if $@;
 646
 647     return $res;
 648 }
 649
 650 sub vmstatus {
 651     my ($opt_vmid) = @_;
 652
 653     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 654
 655     my $active_hash = list_active_containers();
 656
 657     foreach my $vmid (keys %$list) {
 658         my $d = $list->{$vmid};
 659
 660         my $running = defined($active_hash->{$vmid});
 661
 662         $d->{status} = $running ? 'running' : 'stopped';
 663
 664         my $cfspath = cfs_config_path($vmid);
 665         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 666
 667         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 668         $d->{name} =~ s/[\s]//g;
 669
 670         $d->{cpus} = $conf->{cpulimit} // 0;
 671
 672         if ($running) {
 673             my $res = get_container_disk_usage($vmid);
 674             $d->{disk} = $res->{used};
 675             $d->{maxdisk} = $res->{total};
 676         } else {
 677             $d->{disk} = 0;
 678             # use 4GB by default ??
 679             if (my $rootfs = $conf->{rootfs}) {
 680                 my $rootinfo = parse_ct_mountpoint($rootfs);
 681                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 682             } else {
 683                 $d->{maxdisk} = 4*1024*1024*1024;
 684             }
 685         }
 686
 687         $d->{mem} = 0;
 688         $d->{swap} = 0;
 689         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 690         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 691
 692         $d->{uptime} = 0;
 693         $d->{cpu} = 0;
 694
 695         $d->{netout} = 0;
 696         $d->{netin} = 0;
 697
 698         $d->{diskread} = 0;
 699         $d->{diskwrite} = 0;
 700
 701         $d->{template} = is_template($conf);
 702     }
 703
 704     foreach my $vmid (keys %$list) {
 705         my $d = $list->{$vmid};
 706         next if $d->{status} ne 'running';
 707
 708         $d->{uptime} = 100; # fixme:
 709
 710         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 711         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 712
 713         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 714         my @bytes = split(/\n/, $blkio_bytes);
 715         foreach my $byte (@bytes) {
 716             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 717                 $d->{diskread} = $2 if $key eq 'Read';
 718                 $d->{diskwrite} = $2 if $key eq 'Write';
 719             }
 720         }
 721     }
 722
 723     return $list;
 724 }
 725
 726 my $parse_size = sub {
 727     my ($value) = @_;
 728
 729     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 730     my ($size, $unit) = ($1, $3);
 731     if ($unit) {
 732         if ($unit eq 'K') {
 733             $size = $size * 1024;
 734         } elsif ($unit eq 'M') {
 735             $size = $size * 1024 * 1024;
 736         } elsif ($unit eq 'G') {
 737             $size = $size * 1024 * 1024 * 1024;
 738         }
 739     }
 740     return int($size);
 741 };
 742
 743 sub parse_ct_mountpoint {
 744     my ($data) = @_;
 745
 746     $data //= '';
 747
 748     my $res = {};
 749
 750     foreach my $p (split (/,/, $data)) {
 751         next if $p =~ m/^\s*$/;
 752
 753         if ($p =~ m/^(volume|backup|size|mp)=(.+)$/) {
 754             my ($k, $v) = ($1, $2);
 755             return undef if defined($res->{$k});
 756             $res->{$k} = $v;
 757         } else {
 758             if (!$res->{volume} && $p !~ m/=/) {
 759                 $res->{volume} = $p;
 760             } else {
 761                 return undef;
 762             }
 763         }
 764     }
 765
 766     return undef if !$res->{volume};
 767
 768     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 769
 770     if ($res->{size}) {
 771         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 772     }
 773
 774     return $res;
 775 }
 776
 777 sub print_ct_mountpoint {
 778     my ($info) = @_;
 779
 780     my $opts = '';
 781
 782     die "missing volume\n" if !$info->{volume};
 783
 784     foreach my $o ('size', 'backup') {
 785         $opts .= ",$o=$info->{$o}" if defined($info->{$o});
 786     }
 787
 788     return "$info->{volume}$opts";
 789 }
 790
 791 sub print_lxc_network {
 792     my $net = shift;
 793
 794     die "no network name defined\n" if !$net->{name};
 795
 796     my $res = "name=$net->{name}";
 797
 798     foreach my $k (qw(hwaddr mtu bridge ip gw ip6 gw6 firewall tag)) {
 799         next if !defined($net->{$k});
 800         $res .= ",$k=$net->{$k}";
 801     }
 802
 803     return $res;
 804 }
 805
 806 sub parse_lxc_network {
 807     my ($data) = @_;
 808
 809     my $res = {};
 810
 811     return $res if !$data;
 812
 813     foreach my $pv (split (/,/, $data)) {
 814         if ($pv =~ m/^(bridge|hwaddr|mtu|name|ip|ip6|gw|gw6|firewall|tag)=(\S+)$/) {
 815             $res->{$1} = $2;
 816         } else {
 817             return undef;
 818         }
 819     }
 820
 821     $res->{type} = 'veth';
 822     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 823
 824     return $res;
 825 }
 826
 827 sub read_cgroup_value {
 828     my ($group, $vmid, $name, $full) = @_;
 829
 830     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 831
 832     return PVE::Tools::file_get_contents($path) if $full;
 833
 834     return PVE::Tools::file_read_firstline($path);
 835 }
 836
 837 sub write_cgroup_value {
 838    my ($group, $vmid, $name, $value) = @_;
 839
 840    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 841    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 842
 843 }
 844
 845 sub find_lxc_console_pids {
 846
 847     my $res = {};
 848
 849     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 850         my ($pid) = @_;
 851
 852         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 853         return if !$cmdline;
 854
 855         my @args = split(/\0/, $cmdline);
 856
 857         # serach for lxc-console -n <vmid>
 858         return if scalar(@args) != 3;
 859         return if $args[1] ne '-n';
 860         return if $args[2] !~ m/^\d+$/;
 861         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 862
 863         my $vmid = $args[2];
 864
 865         push @{$res->{$vmid}}, $pid;
 866     });
 867
 868     return $res;
 869 }
 870
 871 sub find_lxc_pid {
 872     my ($vmid) = @_;
 873
 874     my $pid = undef;
 875     my $parser = sub {
 876         my $line = shift;
 877         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 878     };
 879     PVE::Tools::run_command(['lxc-info', '-n', $vmid], outfunc => $parser);
 880
 881     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 882
 883     return $pid;
 884 }
 885
 886 my $ipv4_reverse_mask = [
 887     '0.0.0.0',
 888     '128.0.0.0',
 889     '192.0.0.0',
 890     '224.0.0.0',
 891     '240.0.0.0',
 892     '248.0.0.0',
 893     '252.0.0.0',
 894     '254.0.0.0',
 895     '255.0.0.0',
 896     '255.128.0.0',
 897     '255.192.0.0',
 898     '255.224.0.0',
 899     '255.240.0.0',
 900     '255.248.0.0',
 901     '255.252.0.0',
 902     '255.254.0.0',
 903     '255.255.0.0',
 904     '255.255.128.0',
 905     '255.255.192.0',
 906     '255.255.224.0',
 907     '255.255.240.0',
 908     '255.255.248.0',
 909     '255.255.252.0',
 910     '255.255.254.0',
 911     '255.255.255.0',
 912     '255.255.255.128',
 913     '255.255.255.192',
 914     '255.255.255.224',
 915     '255.255.255.240',
 916     '255.255.255.248',
 917     '255.255.255.252',
 918     '255.255.255.254',
 919     '255.255.255.255',
 920 ];
 921
 922 # Note: we cannot use Net:IP, because that only allows strict
 923 # CIDR networks
 924 sub parse_ipv4_cidr {
 925     my ($cidr, $noerr) = @_;
 926
 927     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
 928         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
 929     }
 930
 931     return undef if $noerr;
 932
 933     die "unable to parse ipv4 address/mask\n";
 934 }
 935
 936 sub check_lock {
 937     my ($conf) = @_;
 938
 939     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
 940 }
 941
 942 sub update_lxc_config {
 943     my ($storage_cfg, $vmid, $conf) = @_;
 944
 945     my $dir = "/var/lib/lxc/$vmid";
 946
 947     if ($conf->{template}) {
 948
 949         unlink "$dir/config";
 950
 951         return;
 952     }
 953
 954     my $raw = '';
 955
 956     die "missing 'arch' - internal error" if !$conf->{arch};
 957     $raw .= "lxc.arch = $conf->{arch}\n";
 958
 959     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
 960     if ($ostype eq 'debian' || $ostype eq 'ubuntu' || $ostype eq 'centos') {
 961         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
 962     } else {
 963         die "implement me";
 964     }
 965
 966     if (!has_dev_console($conf)) {
 967         $raw .= "lxc.console = none\n";
 968         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
 969     }
 970
 971     my $ttycount = get_tty_count($conf);
 972     $raw .= "lxc.tty = $ttycount\n";
 973
 974     my $utsname = $conf->{hostname} || "CT$vmid";
 975     $raw .= "lxc.utsname = $utsname\n";
 976
 977     my $memory = $conf->{memory} || 512;
 978     my $swap = $conf->{swap} // 0;
 979
 980     my $lxcmem = int($memory*1024*1024);
 981     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
 982
 983     my $lxcswap = int(($memory + $swap)*1024*1024);
 984     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
 985
 986     if (my $cpulimit = $conf->{cpulimit}) {
 987         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
 988         my $value = int(100000*$cpulimit);
 989         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
 990     }
 991
 992     my $shares = $conf->{cpuunits} || 1024;
 993     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
 994
 995     PVE::LXC::foreach_mountpoint($conf, sub {
 996         my ($ms, $mountpoint) = @_;
 997
 998         my $volid = $mountpoint->{volume};
 999         return if !$volid;
1000
1001         my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
1002
1003         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1004         my $path = PVE::Storage::path($storage_cfg, $volid);
1005
1006         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1007             PVE::Storage::parse_volname($storage_cfg, $volid);
1008
1009         die "unable to use template as mountpoint\n" if $isBase;
1010
1011         if ($format eq 'subvol') {
1012             $mountpoint->{mp} =~ s/^\///s;
1013             if ($ms eq 'rootfs') {
1014                 $raw .= "lxc.rootfs = $path\n";
1015             } else {
1016                 $raw .= "lxc.mount.entry = $path $mountpoint->{mp} none defaults,bind 0 0\n";
1017             }
1018         } elsif ($format eq 'raw') {
1019
1020             if ($scfg->{path}) {
1021                 $raw .= "lxc.rootfs = loop:$path\n" if $ms eq 'rootfs';
1022             } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'rbd') {
1023                 $raw .= "lxc.rootfs = $path\n" if $ms eq 'rootfs';
1024             } else {
1025                 die "unsupported storage type '$scfg->{type}'\n";
1026             }
1027         } else {
1028             die "unsupported image format '$format'\n";
1029         }
1030
1031     });
1032
1033     my $netcount = 0;
1034     foreach my $k (keys %$conf) {
1035         next if $k !~ m/^net(\d+)$/;
1036         my $ind = $1;
1037         my $d = parse_lxc_network($conf->{$k});
1038         $netcount++;
1039         $raw .= "lxc.network.type = veth\n";
1040         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1041         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1042         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1043         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1044     }
1045
1046     if (my $lxcconf = $conf->{lxc}) {
1047         foreach my $entry (@$lxcconf) {
1048             my ($k, $v) = @$entry;
1049             $netcount++ if $k eq 'lxc.network.type';
1050             $raw .= "$k = $v\n";
1051         }
1052     }
1053
1054     $raw .= "lxc.network.type = empty\n" if !$netcount;
1055
1056     File::Path::mkpath("$dir/rootfs");
1057
1058     PVE::Tools::file_set_contents("$dir/config", $raw);
1059 }
1060
1061 # verify and cleanup nameserver list (replace \0 with ' ')
1062 sub verify_nameserver_list {
1063     my ($nameserver_list) = @_;
1064
1065     my @list = ();
1066     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1067         PVE::JSONSchema::pve_verify_ip($server);
1068         push @list, $server;
1069     }
1070
1071     return join(' ', @list);
1072 }
1073
1074 sub verify_searchdomain_list {
1075     my ($searchdomain_list) = @_;
1076
1077     my @list = ();
1078     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1079         # todo: should we add checks for valid dns domains?
1080         push @list, $server;
1081     }
1082
1083     return join(' ', @list);
1084 }
1085
1086 sub update_pct_config {
1087     my ($vmid, $conf, $running, $param, $delete) = @_;
1088
1089     my @nohotplug;
1090
1091     my $rootdir;
1092     if ($running) {
1093         my $pid = find_lxc_pid($vmid);
1094         $rootdir = "/proc/$pid/root";
1095     }
1096
1097     if (defined($delete)) {
1098         foreach my $opt (@$delete) {
1099             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1100                 die "unable to delete required option '$opt'\n";
1101             } elsif ($opt eq 'swap') {
1102                 delete $conf->{$opt};
1103                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1104             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1105                 delete $conf->{$opt};
1106             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1107                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1108                 delete $conf->{$opt};
1109                 push @nohotplug, $opt;
1110                 next if $running;
1111             } elsif ($opt =~ m/^net(\d)$/) {
1112                 delete $conf->{$opt};
1113                 next if !$running;
1114                 my $netid = $1;
1115                 PVE::Network::veth_delete("veth${vmid}i$netid");
1116             } else {
1117                 die "implement me"
1118             }
1119             PVE::LXC::write_config($vmid, $conf) if $running;
1120         }
1121     }
1122
1123     # There's no separate swap size to configure, there's memory and "total"
1124     # memory (iow. memory+swap). This means we have to change them together.
1125     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1126     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1127     if (defined($wanted_memory) || defined($wanted_swap)) {
1128
1129         $wanted_memory //= ($conf->{memory} || 512);
1130         $wanted_swap //=  ($conf->{swap} || 0);
1131
1132         my $total = $wanted_memory + $wanted_swap;
1133         if ($running) {
1134             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1135             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1136         }
1137         $conf->{memory} = $wanted_memory;
1138         $conf->{swap} = $wanted_swap;
1139
1140         PVE::LXC::write_config($vmid, $conf) if $running;
1141     }
1142
1143     foreach my $opt (keys %$param) {
1144         my $value = $param->{$opt};
1145         if ($opt eq 'hostname') {
1146             $conf->{$opt} = $value;
1147         } elsif ($opt eq 'onboot') {
1148             $conf->{$opt} = $value ? 1 : 0;
1149         } elsif ($opt eq 'startup') {
1150             $conf->{$opt} = $value;
1151         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1152             $conf->{$opt} = $value;
1153             push @nohotplug, $opt;
1154             next if $running;
1155         } elsif ($opt eq 'nameserver') {
1156             my $list = verify_nameserver_list($value);
1157             $conf->{$opt} = $list;
1158             push @nohotplug, $opt;
1159             next if $running;
1160         } elsif ($opt eq 'searchdomain') {
1161             my $list = verify_searchdomain_list($value);
1162             $conf->{$opt} = $list;
1163             push @nohotplug, $opt;
1164             next if $running;
1165         } elsif ($opt eq 'cpulimit') {
1166             $conf->{$opt} = $value;
1167             push @nohotplug, $opt; # fixme: hotplug
1168             next;
1169         } elsif ($opt eq 'cpuunits') {
1170             $conf->{$opt} = $value;
1171             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1172         } elsif ($opt eq 'description') {
1173             $conf->{$opt} = PVE::Tools::encode_text($value);
1174         } elsif ($opt =~ m/^net(\d+)$/) {
1175             my $netid = $1;
1176             my $net = parse_lxc_network($value);
1177             if (!$running) {
1178                 $conf->{$opt} = print_lxc_network($net);
1179             } else {
1180                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1181             }
1182         } else {
1183             die "implement me: $opt";
1184         }
1185         PVE::LXC::write_config($vmid, $conf) if $running;
1186     }
1187
1188     if ($running && scalar(@nohotplug)) {
1189         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1190     }
1191 }
1192
1193 sub has_dev_console {
1194     my ($conf) = @_;
1195
1196     return !(defined($conf->{console}) && !$conf->{console});
1197 }
1198
1199 sub get_tty_count {
1200     my ($conf) = @_;
1201
1202     return $conf->{tty} // $confdesc->{tty}->{default};
1203 }
1204
1205 sub get_cmode {
1206     my ($conf) = @_;
1207
1208     return $conf->{cmode} // $confdesc->{cmode}->{default};
1209 }
1210
1211 sub get_console_command {
1212     my ($vmid, $conf) = @_;
1213
1214     my $cmode = get_cmode($conf);
1215
1216     if ($cmode eq 'console') {
1217         return ['lxc-console', '-n',  $vmid, '-t', 0];
1218     } elsif ($cmode eq 'tty') {
1219         return ['lxc-console', '-n',  $vmid];
1220     } elsif ($cmode eq 'shell') {
1221         return ['lxc-attach', '--clear-env', '-n', $vmid];
1222     } else {
1223         die "internal error";
1224     }
1225 }
1226
1227 sub get_primary_ips {
1228     my ($conf) = @_;
1229
1230     # return data from net0
1231
1232     return undef if !defined($conf->{net0});
1233     my $net = parse_lxc_network($conf->{net0});
1234
1235     my $ipv4 = $net->{ip};
1236     if ($ipv4) {
1237         if ($ipv4 =~ /^(dhcp|manual)$/) {
1238             $ipv4 = undef
1239         } else {
1240             $ipv4 =~ s!/\d+$!!;
1241         }
1242     }
1243     my $ipv6 = $net->{ip6};
1244     if ($ipv6) {
1245         if ($ipv6 =~ /^(dhcp|manual)$/) {
1246             $ipv6 = undef;
1247         } else {
1248             $ipv6 =~ s!/\d+$!!;
1249         }
1250     }
1251
1252     return ($ipv4, $ipv6);
1253 }
1254
1255
1256 sub destroy_lxc_container {
1257     my ($storage_cfg, $vmid, $conf) = @_;
1258
1259     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1260     if (defined($rootinfo->{volume})) {
1261         my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $rootinfo->{volume});
1262         PVE::Storage::vdisk_free($storage_cfg, $rootinfo->{volume}) if $vmid == $owner;;
1263     }
1264     rmdir "/var/lib/lxc/$vmid/rootfs";
1265     unlink "/var/lib/lxc/$vmid/config";
1266     rmdir "/var/lib/lxc/$vmid";
1267     destroy_config($vmid);
1268
1269     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1270     #PVE::Tools::run_command($cmd);
1271 }
1272
1273 sub vm_stop_cleanup {
1274     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1275
1276     eval {
1277         if (!$keepActive) {
1278
1279             my $loopdevs = loopdevices_list();
1280
1281             PVE::LXC::foreach_mountpoint($conf, sub {
1282                 my ($ms, $mountpoint) = @_;
1283
1284                 my $volid = $mountpoint->{volume};
1285                 #detach loopdevices of non rootfs mountpoints
1286                 my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
1287                 my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1288                 my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1289                     PVE::Storage::parse_volname($storage_cfg, $volid);
1290
1291                 if($ms ne 'rootfs' && $format eq 'raw' && ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs')) {
1292                     my $path = PVE::Storage::path($storage_cfg, $volid);
1293                     foreach my $dev (keys %$loopdevs){
1294                         PVE::Tools::run_command(['losetup', '-d', $dev]) if $loopdevs->{$dev} eq $path;
1295                     }
1296                 }
1297
1298                 PVE::Storage::deactivate_volumes($storage_cfg, [$volid]);
1299
1300             });
1301         }
1302     };
1303     warn $@ if $@; # avoid errors - just warn
1304 }
1305
1306 my $safe_num_ne = sub {
1307     my ($a, $b) = @_;
1308
1309     return 0 if !defined($a) && !defined($b);
1310     return 1 if !defined($a);
1311     return 1 if !defined($b);
1312
1313     return $a != $b;
1314 };
1315
1316 my $safe_string_ne = sub {
1317     my ($a, $b) = @_;
1318
1319     return 0 if !defined($a) && !defined($b);
1320     return 1 if !defined($a);
1321     return 1 if !defined($b);
1322
1323     return $a ne $b;
1324 };
1325
1326 sub update_net {
1327     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1328
1329     if ($newnet->{type} ne 'veth') {
1330         # for when there are physical interfaces
1331         die "cannot update interface of type $newnet->{type}";
1332     }
1333
1334     my $veth = "veth${vmid}i${netid}";
1335     my $eth = $newnet->{name};
1336
1337     if (my $oldnetcfg = $conf->{$opt}) {
1338         my $oldnet = parse_lxc_network($oldnetcfg);
1339
1340         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1341             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1342
1343             PVE::Network::veth_delete($veth);
1344             delete $conf->{$opt};
1345             PVE::LXC::write_config($vmid, $conf);
1346
1347             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1348
1349         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1350                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1351                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1352
1353                 if ($oldnet->{bridge}) {
1354                     PVE::Network::tap_unplug($veth);
1355                     foreach (qw(bridge tag firewall)) {
1356                         delete $oldnet->{$_};
1357                     }
1358                     $conf->{$opt} = print_lxc_network($oldnet);
1359                     PVE::LXC::write_config($vmid, $conf);
1360                 }
1361
1362                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1363                 foreach (qw(bridge tag firewall)) {
1364                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1365                 }
1366                 $conf->{$opt} = print_lxc_network($oldnet);
1367                 PVE::LXC::write_config($vmid, $conf);
1368         }
1369     } else {
1370         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1371     }
1372
1373     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1374 }
1375
1376 sub hotplug_net {
1377     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1378
1379     my $veth = "veth${vmid}i${netid}";
1380     my $vethpeer = $veth . "p";
1381     my $eth = $newnet->{name};
1382
1383     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1384     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1385
1386     # attach peer in container
1387     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1388     PVE::Tools::run_command($cmd);
1389
1390     # link up peer in container
1391     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1392     PVE::Tools::run_command($cmd);
1393
1394     my $done = { type => 'veth' };
1395     foreach (qw(bridge tag firewall hwaddr name)) {
1396         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1397     }
1398     $conf->{$opt} = print_lxc_network($done);
1399
1400     PVE::LXC::write_config($vmid, $conf);
1401 }
1402
1403 sub update_ipconfig {
1404     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1405
1406     my $lxc_setup = PVE::LXCSetup->new($conf, $rootdir);
1407
1408     my $optdata = parse_lxc_network($conf->{$opt});
1409     my $deleted = [];
1410     my $added = [];
1411     my $nscmd = sub {
1412         my $cmdargs = shift;
1413         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1414     };
1415     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1416
1417     my $change_ip_config = sub {
1418         my ($ipversion) = @_;
1419
1420         my $family_opt = "-$ipversion";
1421         my $suffix = $ipversion == 4 ? '' : $ipversion;
1422         my $gw= "gw$suffix";
1423         my $ip= "ip$suffix";
1424
1425         my $newip = $newnet->{$ip};
1426         my $newgw = $newnet->{$gw};
1427         my $oldip = $optdata->{$ip};
1428
1429         my $change_ip = &$safe_string_ne($oldip, $newip);
1430         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1431
1432         return if !$change_ip && !$change_gw;
1433
1434         # step 1: add new IP, if this fails we cancel
1435         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1436             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1437             if (my $err = $@) {
1438                 warn $err;
1439                 return;
1440             }
1441         }
1442
1443         # step 2: replace gateway
1444         #   If this fails we delete the added IP and cancel.
1445         #   If it succeeds we save the config and delete the old IP, ignoring
1446         #   errors. The config is then saved.
1447         # Note: 'ip route replace' can add
1448         if ($change_gw) {
1449             if ($newgw) {
1450                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1451                 if (my $err = $@) {
1452                     warn $err;
1453                     # the route was not replaced, the old IP is still available
1454                     # rollback (delete new IP) and cancel
1455                     if ($change_ip) {
1456                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1457                         warn $@ if $@; # no need to die here
1458                     }
1459                     return;
1460                 }
1461             } else {
1462                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1463                 # if the route was not deleted, the guest might have deleted it manually
1464                 # warn and continue
1465                 warn $@ if $@;
1466             }
1467         }
1468
1469         # from this point on we save the configuration
1470         # step 3: delete old IP ignoring errors
1471         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1472             # We need to enable promote_secondaries, otherwise our newly added
1473             # address will be removed along with the old one.
1474             my $promote = 0;
1475             eval {
1476                 if ($ipversion == 4) {
1477                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1478                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1479                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1480                 }
1481                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1482             };
1483             warn $@ if $@; # no need to die here
1484
1485             if ($ipversion == 4) {
1486                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1487             }
1488         }
1489
1490         foreach my $property ($ip, $gw) {
1491             if ($newnet->{$property}) {
1492                 $optdata->{$property} = $newnet->{$property};
1493             } else {
1494                 delete $optdata->{$property};
1495             }
1496         }
1497         $conf->{$opt} = print_lxc_network($optdata);
1498         PVE::LXC::write_config($vmid, $conf);
1499         $lxc_setup->setup_network($conf);
1500     };
1501
1502     &$change_ip_config(4);
1503     &$change_ip_config(6);
1504
1505 }
1506
1507 # Internal snapshots
1508
1509 # NOTE: Snapshot create/delete involves several non-atomic
1510 # action, and can take a long time.
1511 # So we try to avoid locking the file and use 'lock' variable
1512 # inside the config file instead.
1513
1514 my $snapshot_copy_config = sub {
1515     my ($source, $dest) = @_;
1516
1517     foreach my $k (keys %$source) {
1518         next if $k eq 'snapshots';
1519         next if $k eq 'snapstate';
1520         next if $k eq 'snaptime';
1521         next if $k eq 'vmstate';
1522         next if $k eq 'lock';
1523         next if $k eq 'digest';
1524         next if $k eq 'description';
1525
1526         $dest->{$k} = $source->{$k};
1527     }
1528 };
1529
1530 my $snapshot_prepare = sub {
1531     my ($vmid, $snapname, $comment) = @_;
1532
1533     my $snap;
1534
1535     my $updatefn =  sub {
1536
1537         my $conf = load_config($vmid);
1538
1539         die "you can't take a snapshot if it's a template\n"
1540             if is_template($conf);
1541
1542         check_lock($conf);
1543
1544         $conf->{lock} = 'snapshot';
1545
1546         die "snapshot name '$snapname' already used\n"
1547             if defined($conf->{snapshots}->{$snapname});
1548
1549         my $storecfg = PVE::Storage::config();
1550         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1551
1552         $snap = $conf->{snapshots}->{$snapname} = {};
1553
1554         &$snapshot_copy_config($conf, $snap);
1555
1556         $snap->{'snapstate'} = "prepare";
1557         $snap->{'snaptime'} = time();
1558         $snap->{'description'} = $comment if $comment;
1559         $conf->{snapshots}->{$snapname} = $snap;
1560
1561         PVE::LXC::write_config($vmid, $conf);
1562     };
1563
1564     lock_container($vmid, 10, $updatefn);
1565
1566     return $snap;
1567 };
1568
1569 my $snapshot_commit = sub {
1570     my ($vmid, $snapname) = @_;
1571
1572     my $updatefn = sub {
1573
1574         my $conf = load_config($vmid);
1575
1576         die "missing snapshot lock\n"
1577             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1578
1579         die "snapshot '$snapname' does not exist\n"
1580             if !defined($conf->{snapshots}->{$snapname});
1581
1582         die "wrong snapshot state\n"
1583             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1584                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1585
1586         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1587         delete $conf->{lock};
1588         $conf->{parent} = $snapname;
1589
1590         PVE::LXC::write_config($vmid, $conf);
1591     };
1592
1593     lock_container($vmid, 10 ,$updatefn);
1594 };
1595
1596 sub has_feature {
1597     my ($feature, $conf, $storecfg, $snapname) = @_;
1598
1599     #Fixme add other drives if necessary.
1600     my $err;
1601
1602     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1603     $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $rootinfo->{volume}, $snapname);
1604
1605     return $err ? 0 : 1;
1606 }
1607
1608 sub snapshot_create {
1609     my ($vmid, $snapname, $comment) = @_;
1610
1611     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1612
1613     my $conf = load_config($vmid);
1614
1615     my $cmd = "/usr/bin/lxc-freeze -n $vmid";
1616     my $running = check_running($vmid);
1617     eval {
1618         if ($running) {
1619             PVE::Tools::run_command($cmd);
1620         };
1621
1622         my $storecfg = PVE::Storage::config();
1623         my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1624         my $volid = $rootinfo->{volume};
1625
1626         $cmd = "/usr/bin/lxc-unfreeze -n $vmid";
1627         if ($running) {
1628             PVE::Tools::run_command($cmd);
1629         };
1630
1631         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1632         &$snapshot_commit($vmid, $snapname);
1633     };
1634     if(my $err = $@) {
1635         snapshot_delete($vmid, $snapname, 1);
1636         die "$err\n";
1637     }
1638 }
1639
1640 sub snapshot_delete {
1641     my ($vmid, $snapname, $force) = @_;
1642
1643     my $snap;
1644
1645     my $conf;
1646
1647     my $updatefn =  sub {
1648
1649         $conf = load_config($vmid);
1650
1651         die "you can't delete a snapshot if vm is a template\n"
1652             if is_template($conf);
1653
1654         $snap = $conf->{snapshots}->{$snapname};
1655
1656         check_lock($conf);
1657
1658         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1659
1660         $snap->{snapstate} = 'delete';
1661
1662         PVE::LXC::write_config($vmid, $conf);
1663     };
1664
1665     lock_container($vmid, 10, $updatefn);
1666
1667     my $storecfg = PVE::Storage::config();
1668
1669     my $del_snap =  sub {
1670
1671         check_lock($conf);
1672
1673         if ($conf->{parent} eq $snapname) {
1674             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1675                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1676             } else {
1677                 delete $conf->{parent};
1678             }
1679         }
1680
1681         delete $conf->{snapshots}->{$snapname};
1682
1683         PVE::LXC::write_config($vmid, $conf);
1684     };
1685
1686     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1687     my $rootinfo = PVE::LXC::parse_ct_mountpoint($rootfs);
1688     my $volid = $rootinfo->{volume};
1689
1690     eval {
1691         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1692     };
1693     my $err = $@;
1694
1695     if(!$err || ($err && $force)) {
1696         lock_container($vmid, 10, $del_snap);
1697         if ($err) {
1698             die "Can't delete snapshot: $vmid $snapname $err\n";
1699         }
1700     }
1701 }
1702
1703 sub snapshot_rollback {
1704     my ($vmid, $snapname) = @_;
1705
1706     my $storecfg = PVE::Storage::config();
1707
1708     my $conf = load_config($vmid);
1709
1710     die "you can't rollback if vm is a template\n" if is_template($conf);
1711
1712     my $snap = $conf->{snapshots}->{$snapname};
1713
1714     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1715
1716     my $rootfs = $snap->{rootfs};
1717     my $rootinfo = PVE::LXC::parse_ct_mountpoint($rootfs);
1718     my $volid = $rootinfo->{volume};
1719
1720     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1721
1722     my $updatefn = sub {
1723
1724         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1725             if $snap->{snapstate};
1726
1727         check_lock($conf);
1728
1729         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1730
1731         die "unable to rollback vm $vmid: vm is running\n"
1732             if check_running($vmid);
1733
1734         $conf->{lock} = 'rollback';
1735
1736         my $forcemachine;
1737
1738         # copy snapshot config to current config
1739
1740         my $tmp_conf = $conf;
1741         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1742         $conf->{snapshots} = $tmp_conf->{snapshots};
1743         delete $conf->{snaptime};
1744         delete $conf->{snapname};
1745         $conf->{parent} = $snapname;
1746
1747         PVE::LXC::write_config($vmid, $conf);
1748     };
1749
1750     my $unlockfn = sub {
1751         delete $conf->{lock};
1752         PVE::LXC::write_config($vmid, $conf);
1753     };
1754
1755     lock_container($vmid, 10, $updatefn);
1756
1757     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1758
1759     lock_container($vmid, 5, $unlockfn);
1760 }
1761
1762 sub template_create {
1763     my ($vmid, $conf) = @_;
1764
1765     my $storecfg = PVE::Storage::config();
1766
1767     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1768     my $volid = $rootinfo->{volume};
1769
1770     die "Template feature is not available for '$volid'\n"
1771         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1772
1773     PVE::Storage::activate_volumes($storecfg, [$volid]);
1774
1775     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1776     $rootinfo->{volume} = $template_volid;
1777     $conf->{rootfs} = print_ct_mountpoint($rootinfo);
1778
1779     write_config($vmid, $conf);
1780 }
1781
1782 sub is_template {
1783     my ($conf) = @_;
1784
1785     return 1 if defined $conf->{template} && $conf->{template} == 1;
1786 }
1787
1788 sub foreach_mountpoint {
1789     my ($conf, $func) = @_;
1790
1791     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1792     $mountpoint->{mp} = '/'; # just to be sure
1793     &$func('rootfs', $mountpoint);
1794
1795     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1796         my $key = "mp$i";
1797         next if !defined($conf->{$key});
1798         $mountpoint = parse_ct_mountpoint($conf->{$key});
1799         &$func($key, $mountpoint);
1800     }
1801 }
1802
1803 sub loopdevices_list {
1804
1805     my $loopdev = {};
1806     my $parser = sub {
1807         my $line = shift;
1808         if ($line =~ m/^(\/dev\/loop\d+)\s+\d\s+\d\s+\d\s+\d\s(\S+)$/) {
1809             $loopdev->{$1} = $2;
1810         }
1811     };
1812
1813     PVE::Tools::run_command(['losetup'], outfunc => $parser);
1814
1815     return $loopdev;
1816 }
1817
1818 sub blockdevices_list {
1819
1820     my $bdevs = {};
1821     dir_glob_foreach("/sys/dev/block/", '(\d+):(\d+)', sub {
1822         my (undef, $major, $minor) = @_;
1823         my $bdev = readlink("/sys/dev/block/$major:$minor");
1824         $bdev =~ s/\.\.\/\.\.\/devices\/virtual\/block\//\/dev\//;
1825         $bdevs->{$bdev}->{major} = $major;
1826         $bdevs->{$bdev}->{minor} = $minor;
1827     });
1828     return $bdevs;
1829 }
1830
1831 sub find_loopdev {
1832     my ($loopdevs, $path) = @_;
1833
1834     foreach my $dev (keys %$loopdevs){
1835         return $dev if $loopdevs->{$dev} eq $path;
1836     }
1837 }
1838
1839 sub setup_mountpoints {
1840     my ($conf, $vmid, $rootdir) = @_;
1841
1842     my $fn = "/etc/pve/storage.cfg";
1843
1844     return if ! -f $fn;
1845
1846     my $raw = PVE::Tools::file_get_contents($fn);
1847     my $storage_cfg = PVE::Storage::Plugin->parse_config($fn, $raw);
1848
1849     my $bdevs = PVE::LXC::blockdevices_list();
1850     my $loopdevs = PVE::LXC::loopdevices_list();
1851
1852     PVE::LXC::foreach_mountpoint($conf, sub {
1853         my ($ms, $mountpoint) = @_;
1854
1855         my $volid = $mountpoint->{volume};
1856         return if !$volid;
1857
1858         my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
1859
1860         my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1861
1862         my $path = PVE::Storage::path($storage_cfg, $volid);
1863
1864         my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1865             PVE::Storage::parse_volname($storage_cfg, $volid);
1866
1867         if ($format ne 'subvol') {
1868
1869             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') {
1870                 $path = PVE::LXC::find_loopdev($loopdevs, $path);
1871             }
1872
1873             if(-l $path) {
1874                 $path = readlink($path);
1875                 $path =~ s/\.\.\/\.\.\//\/dev\//;
1876             }
1877
1878             if ($bdevs->{$path}) {
1879                 PVE::Tools::run_command(['mknod', '-m', '666', $rootdir.$path, 'b',  $bdevs->{$path}->{major}, $bdevs->{$path}->{minor}]);
1880                 write_cgroup_value("devices", $vmid, "devices.allow", "b $bdevs->{$path}->{major}:$bdevs->{$path}->{minor} rwm");
1881             }
1882
1883             PVE::Tools::run_command(['mount', $path, $rootdir.$mountpoint->{mp}]) if $mountpoint->{mp} && $ms ne 'rootfs';
1884         }
1885
1886     });
1887 }
1888
1889 1;