src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use Fcntl ':flock';
   9
  10 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  11 use PVE::Storage;
  12 use PVE::SafeSyslog;
  13 use PVE::INotify;
  14 use PVE::JSONSchema qw(get_standard_option);
  15 use PVE::Tools qw($IPV6RE $IPV4RE);
  16 use PVE::Network;
  17
  18 use Data::Dumper;
  19
  20 my $nodename = PVE::INotify::nodename();
  21
  22 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  23
  24 PVE::JSONSchema::register_format('pve-lxc-network', \&verify_lxc_network);
  25 sub verify_lxc_network {
  26     my ($value, $noerr) = @_;
  27
  28     return $value if parse_lxc_network($value);
  29
  30     return undef if $noerr;
  31
  32     die "unable to parse network setting\n";
  33 }
  34
  35 PVE::JSONSchema::register_format('pve-ct-mountpoint', \&verify_ct_mountpoint);
  36 sub verify_ct_mountpoint {
  37     my ($value, $noerr) = @_;
  38
  39     return $value if parse_ct_mountpoint($value);
  40
  41     return undef if $noerr;
  42
  43     die "unable to parse CT mountpoint options\n";
  44 }
  45
  46 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  47     type => 'string', format => 'pve-ct-mountpoint',
  48     typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+]',
  49     description => "Use volume as container root.",
  50     optional => 1,
  51 });
  52
  53 my $confdesc = {
  54     lock => {
  55         optional => 1,
  56         type => 'string',
  57         description => "Lock/unlock the VM.",
  58         enum => [qw(migrate backup snapshot rollback)],
  59     },
  60     onboot => {
  61         optional => 1,
  62         type => 'boolean',
  63         description => "Specifies whether a VM will be started during system bootup.",
  64         default => 0,
  65     },
  66     startup => get_standard_option('pve-startup-order'),
  67     template => {
  68         optional => 1,
  69         type => 'boolean',
  70         description => "Enable/disable Template.",
  71         default => 0,
  72     },
  73     arch => {
  74         optional => 1,
  75         type => 'string',
  76         enum => ['amd64', 'i386'],
  77         description => "OS architecture type.",
  78         default => 'amd64',
  79     },
  80     ostype => {
  81         optional => 1,
  82         type => 'string',
  83         enum => ['debian', 'ubuntu', 'centos'],
  84         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  85     },
  86     console => {
  87         optional => 1,
  88         type => 'boolean',
  89         description => "Attach a console device (/dev/console) to the container.",
  90         default => 1,
  91     },
  92     tty => {
  93         optional => 1,
  94         type => 'integer',
  95         description => "Specify the number of tty available to the container",
  96         minimum => 0,
  97         maximum => 6,
  98         default => 2,
  99     },
 100     cpulimit => {
 101         optional => 1,
 102         type => 'number',
 103         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 104         minimum => 0,
 105         maximum => 128,
 106         default => 0,
 107     },
 108     cpuunits => {
 109         optional => 1,
 110         type => 'integer',
 111         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 112         minimum => 0,
 113         maximum => 500000,
 114         default => 1024,
 115     },
 116     memory => {
 117         optional => 1,
 118         type => 'integer',
 119         description => "Amount of RAM for the VM in MB.",
 120         minimum => 16,
 121         default => 512,
 122     },
 123     swap => {
 124         optional => 1,
 125         type => 'integer',
 126         description => "Amount of SWAP for the VM in MB.",
 127         minimum => 0,
 128         default => 512,
 129     },
 130     hostname => {
 131         optional => 1,
 132         description => "Set a host name for the container.",
 133         type => 'string',
 134         maxLength => 255,
 135     },
 136     description => {
 137         optional => 1,
 138         type => 'string',
 139         description => "Container description. Only used on the configuration web interface.",
 140     },
 141     searchdomain => {
 142         optional => 1,
 143         type => 'string',
 144         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 145     },
 146     nameserver => {
 147         optional => 1,
 148         type => 'string',
 149         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 150     },
 151     rootfs => get_standard_option('pve-ct-rootfs'),
 152     parent => {
 153         optional => 1,
 154         type => 'string', format => 'pve-configid',
 155         maxLength => 40,
 156         description => "Parent snapshot name. This is used internally, and should not be modified.",
 157     },
 158     snaptime => {
 159         optional => 1,
 160         description => "Timestamp for snapshots.",
 161         type => 'integer',
 162         minimum => 0,
 163     },
 164     cmode => {
 165         optional => 1,
 166         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 167         type => 'string',
 168         enum => ['shell', 'console', 'tty'],
 169         default => 'tty',
 170     },
 171 };
 172
 173 my $valid_lxc_conf_keys = {
 174     'lxc.include' => 1,
 175     'lxc.arch' => 1,
 176     'lxc.utsname' => 1,
 177     'lxc.haltsignal' => 1,
 178     'lxc.rebootsignal' => 1,
 179     'lxc.stopsignal' => 1,
 180     'lxc.init_cmd' => 1,
 181     'lxc.network.type' => 1,
 182     'lxc.network.flags' => 1,
 183     'lxc.network.link' => 1,
 184     'lxc.network.mtu' => 1,
 185     'lxc.network.name' => 1,
 186     'lxc.network.hwaddr' => 1,
 187     'lxc.network.ipv4' => 1,
 188     'lxc.network.ipv4.gateway' => 1,
 189     'lxc.network.ipv6' => 1,
 190     'lxc.network.ipv6.gateway' => 1,
 191     'lxc.network.script.up' => 1,
 192     'lxc.network.script.down' => 1,
 193     'lxc.pts' => 1,
 194     'lxc.console.logfile' => 1,
 195     'lxc.console' => 1,
 196     'lxc.tty' => 1,
 197     'lxc.devttydir' => 1,
 198     'lxc.hook.autodev' => 1,
 199     'lxc.autodev' => 1,
 200     'lxc.kmsg' => 1,
 201     'lxc.mount' => 1,
 202     'lxc.mount.entry' => 1,
 203     'lxc.mount.auto' => 1,
 204     'lxc.rootfs' => 1,
 205     'lxc.rootfs.mount' => 1,
 206     'lxc.rootfs.options' => 1,
 207     # lxc.cgroup.*
 208     'lxc.cap.drop' => 1,
 209     'lxc.cap.keep' => 1,
 210     'lxc.aa_profile' => 1,
 211     'lxc.aa_allow_incomplete' => 1,
 212     'lxc.se_context' => 1,
 213     'lxc.seccomp' => 1,
 214     'lxc.id_map' => 1,
 215     'lxc.hook.pre-start' => 1,
 216     'lxc.hook.pre-mount' => 1,
 217     'lxc.hook.mount' => 1,
 218     'lxc.hook.start' => 1,
 219     'lxc.hook.post-stop' => 1,
 220     'lxc.hook.clone' => 1,
 221     'lxc.hook.destroy' => 1,
 222     'lxc.loglevel' => 1,
 223     'lxc.logfile' => 1,
 224     'lxc.start.auto' => 1,
 225     'lxc.start.delay' => 1,
 226     'lxc.start.order' => 1,
 227     'lxc.group' => 1,
 228     'lxc.environment' => 1,
 229     'lxc.' => 1,
 230     'lxc.' => 1,
 231     'lxc.' => 1,
 232     'lxc.' => 1,
 233 };
 234
 235 my $MAX_LXC_NETWORKS = 10;
 236 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 237     $confdesc->{"net$i"} = {
 238         optional => 1,
 239         type => 'string', format => 'pve-lxc-network',
 240         description => "Specifies network interfaces for the container.\n\n".
 241             "The string should have the follow format:\n\n".
 242             "-net<[0-9]> bridge=<vmbr<Nummber>>[,hwaddr=<MAC>]\n".
 243             "[,mtu=<Number>][,name=<String>][,ip=<IPv4Format/CIDR>]\n".
 244             ",ip6=<IPv6Format/CIDR>][,gw=<GatwayIPv4>]\n".
 245             ",gw6=<GatwayIPv6>][,firewall=<[1|0]>][,tag=<VlanNo>]",
 246     };
 247 }
 248
 249 my $MAX_MOUNT_POINTS = 10;
 250 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 251     $confdesc->{"mp$i"} = {
 252         optional => 1,
 253         type => 'string', format => 'pve-ct-mountpoint',
 254         typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+] [,mp=mountpoint]',
 255         description => "Use volume as container mount point.",
 256         optional => 1,
 257     };
 258 }
 259
 260 sub write_pct_config {
 261     my ($filename, $conf) = @_;
 262
 263     delete $conf->{snapstate}; # just to be sure
 264
 265     my $generate_raw_config = sub {
 266         my ($conf) = @_;
 267
 268         my $raw = '';
 269
 270         # add description as comment to top of file
 271         my $descr = $conf->{description} || '';
 272         foreach my $cl (split(/\n/, $descr)) {
 273             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 274         }
 275
 276         foreach my $key (sort keys %$conf) {
 277             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 278                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 279             $raw .= "$key: $conf->{$key}\n";
 280         }
 281
 282         if (my $lxcconf = $conf->{lxc}) {
 283             foreach my $entry (@$lxcconf) {
 284                 my ($k, $v) = @$entry;
 285                 $raw .= "$k: $v\n";
 286             }
 287         }
 288
 289         return $raw;
 290     };
 291
 292     my $raw = &$generate_raw_config($conf);
 293
 294     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 295         $raw .= "\n[$snapname]\n";
 296         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 297     }
 298
 299     return $raw;
 300 }
 301
 302 sub check_type {
 303     my ($key, $value) = @_;
 304
 305     die "unknown setting '$key'\n" if !$confdesc->{$key};
 306
 307     my $type = $confdesc->{$key}->{type};
 308
 309     if (!defined($value)) {
 310         die "got undefined value\n";
 311     }
 312
 313     if ($value =~ m/[\n\r]/) {
 314         die "property contains a line feed\n";
 315     }
 316
 317     if ($type eq 'boolean') {
 318         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 319         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 320         die "type check ('boolean') failed - got '$value'\n";
 321     } elsif ($type eq 'integer') {
 322         return int($1) if $value =~ m/^(\d+)$/;
 323         die "type check ('integer') failed - got '$value'\n";
 324     } elsif ($type eq 'number') {
 325         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 326         die "type check ('number') failed - got '$value'\n";
 327     } elsif ($type eq 'string') {
 328         if (my $fmt = $confdesc->{$key}->{format}) {
 329             PVE::JSONSchema::check_format($fmt, $value);
 330             return $value;
 331         }
 332         return $value;
 333     } else {
 334         die "internal error"
 335     }
 336 }
 337
 338 sub parse_pct_config {
 339     my ($filename, $raw) = @_;
 340
 341     return undef if !defined($raw);
 342
 343     my $res = {
 344         digest => Digest::SHA::sha1_hex($raw),
 345         snapshots => {},
 346     };
 347
 348     $filename =~ m|/lxc/(\d+).conf$|
 349         || die "got strange filename '$filename'";
 350
 351     my $vmid = $1;
 352
 353     my $conf = $res;
 354     my $descr = '';
 355     my $section = '';
 356
 357     my @lines = split(/\n/, $raw);
 358     foreach my $line (@lines) {
 359         next if $line =~ m/^\s*$/;
 360
 361         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 362             $section = $1;
 363             $conf->{description} = $descr if $descr;
 364             $descr = '';
 365             $conf = $res->{snapshots}->{$section} = {};
 366             next;
 367         }
 368
 369         if ($line =~ m/^\#(.*)\s*$/) {
 370             $descr .= PVE::Tools::decode_text($1) . "\n";
 371             next;
 372         }
 373
 374         if ($line =~ m/^(lxc\.[a-z0-9\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 375             my $key = $1;
 376             my $value = $3;
 377             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 378                 push @{$conf->{lxc}}, [$key, $value];
 379             } else {
 380                 warn "vm $vmid - unable to parse config: $line\n";
 381             }
 382         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 383             $descr .= PVE::Tools::decode_text($2);
 384         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 385             $conf->{snapstate} = $1;
 386         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
 387             my $key = $1;
 388             my $value = $2;
 389             eval { $value = check_type($key, $value); };
 390             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 391             $conf->{$key} = $value;
 392         } else {
 393             warn "vm $vmid - unable to parse config: $line\n";
 394         }
 395     }
 396
 397     $conf->{description} = $descr if $descr;
 398
 399     delete $res->{snapstate}; # just to be sure
 400
 401     return $res;
 402 }
 403
 404 sub config_list {
 405     my $vmlist = PVE::Cluster::get_vmlist();
 406     my $res = {};
 407     return $res if !$vmlist || !$vmlist->{ids};
 408     my $ids = $vmlist->{ids};
 409
 410     foreach my $vmid (keys %$ids) {
 411         next if !$vmid; # skip CT0
 412         my $d = $ids->{$vmid};
 413         next if !$d->{node} || $d->{node} ne $nodename;
 414         next if !$d->{type} || $d->{type} ne 'lxc';
 415         $res->{$vmid}->{type} = 'lxc';
 416     }
 417     return $res;
 418 }
 419
 420 sub cfs_config_path {
 421     my ($vmid, $node) = @_;
 422
 423     $node = $nodename if !$node;
 424     return "nodes/$node/lxc/$vmid.conf";
 425 }
 426
 427 sub config_file {
 428     my ($vmid, $node) = @_;
 429
 430     my $cfspath = cfs_config_path($vmid, $node);
 431     return "/etc/pve/$cfspath";
 432 }
 433
 434 sub load_config {
 435     my ($vmid) = @_;
 436
 437     my $cfspath = cfs_config_path($vmid);
 438
 439     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 440     die "container $vmid does not exists\n" if !defined($conf);
 441
 442     return $conf;
 443 }
 444
 445 sub create_config {
 446     my ($vmid, $conf) = @_;
 447
 448     my $dir = "/etc/pve/nodes/$nodename/lxc";
 449     mkdir $dir;
 450
 451     write_config($vmid, $conf);
 452 }
 453
 454 sub destroy_config {
 455     my ($vmid) = @_;
 456
 457     unlink config_file($vmid, $nodename);
 458 }
 459
 460 sub write_config {
 461     my ($vmid, $conf) = @_;
 462
 463     my $cfspath = cfs_config_path($vmid);
 464
 465     PVE::Cluster::cfs_write_file($cfspath, $conf);
 466 }
 467
 468 # flock: we use one file handle per process, so lock file
 469 # can be called multiple times and succeeds for the same process.
 470
 471 my $lock_handles =  {};
 472 my $lockdir = "/run/lock/lxc";
 473
 474 sub lock_filename {
 475     my ($vmid) = @_;
 476
 477     return "$lockdir/pve-config-{$vmid}.lock";
 478 }
 479
 480 sub lock_aquire {
 481     my ($vmid, $timeout) = @_;
 482
 483     $timeout = 10 if !$timeout;
 484     my $mode = LOCK_EX;
 485
 486     my $filename = lock_filename($vmid);
 487
 488     mkdir $lockdir if !-d $lockdir;
 489
 490     my $lock_func = sub {
 491         if (!$lock_handles->{$$}->{$filename}) {
 492             my $fh = new IO::File(">>$filename") ||
 493                 die "can't open file - $!\n";
 494             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 495         }
 496
 497         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 498             print STDERR "trying to aquire lock...";
 499             my $success;
 500             while(1) {
 501                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 502                 # try again on EINTR (see bug #273)
 503                 if ($success || ($! != EINTR)) {
 504                     last;
 505                 }
 506             }
 507             if (!$success) {
 508                 print STDERR " failed\n";
 509                 die "can't aquire lock - $!\n";
 510             }
 511
 512             $lock_handles->{$$}->{$filename}->{refcount}++;
 513
 514             print STDERR " OK\n";
 515         }
 516     };
 517
 518     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 519     my $err = $@;
 520     if ($err) {
 521         die "can't lock file '$filename' - $err";
 522     }
 523 }
 524
 525 sub lock_release {
 526     my ($vmid) = @_;
 527
 528     my $filename = lock_filename($vmid);
 529
 530     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 531         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 532         if ($refcount <= 0) {
 533             $lock_handles->{$$}->{$filename} = undef;
 534             close ($fh);
 535         }
 536     }
 537 }
 538
 539 sub lock_container {
 540     my ($vmid, $timeout, $code, @param) = @_;
 541
 542     my $res;
 543
 544     lock_aquire($vmid, $timeout);
 545     eval { $res = &$code(@param) };
 546     my $err = $@;
 547     lock_release($vmid);
 548
 549     die $err if $err;
 550
 551     return $res;
 552 }
 553
 554 sub option_exists {
 555     my ($name) = @_;
 556
 557     return defined($confdesc->{$name});
 558 }
 559
 560 # add JSON properties for create and set function
 561 sub json_config_properties {
 562     my $prop = shift;
 563
 564     foreach my $opt (keys %$confdesc) {
 565         next if $opt eq 'parent' || $opt eq 'snaptime';
 566         next if $prop->{$opt};
 567         $prop->{$opt} = $confdesc->{$opt};
 568     }
 569
 570     return $prop;
 571 }
 572
 573 sub json_config_properties_no_rootfs {
 574     my $prop = shift;
 575
 576     foreach my $opt (keys %$confdesc) {
 577         next if $prop->{$opt};
 578         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 579         $prop->{$opt} = $confdesc->{$opt};
 580     }
 581
 582     return $prop;
 583 }
 584
 585 # container status helpers
 586
 587 sub list_active_containers {
 588
 589     my $filename = "/proc/net/unix";
 590
 591     # similar test is used by lcxcontainers.c: list_active_containers
 592     my $res = {};
 593
 594     my $fh = IO::File->new ($filename, "r");
 595     return $res if !$fh;
 596
 597     while (defined(my $line = <$fh>)) {
 598         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 599             my $path = $1;
 600             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 601                 $res->{$1} = 1;
 602             }
 603         }
 604     }
 605
 606     close($fh);
 607
 608     return $res;
 609 }
 610
 611 # warning: this is slow
 612 sub check_running {
 613     my ($vmid) = @_;
 614
 615     my $active_hash = list_active_containers();
 616
 617     return 1 if defined($active_hash->{$vmid});
 618
 619     return undef;
 620 }
 621
 622 sub get_container_disk_usage {
 623     my ($vmid) = @_;
 624
 625     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 626
 627     my $res = {
 628         total => 0,
 629         used => 0,
 630         avail => 0,
 631     };
 632
 633     my $parser = sub {
 634         my $line = shift;
 635         if (my ($fsid, $total, $used, $avail) = $line =~
 636             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 637             $res = {
 638                 total => $total,
 639                 used => $used,
 640                 avail => $avail,
 641             };
 642         }
 643     };
 644     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 645     warn $@ if $@;
 646
 647     return $res;
 648 }
 649
 650 sub vmstatus {
 651     my ($opt_vmid) = @_;
 652
 653     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 654
 655     my $active_hash = list_active_containers();
 656
 657     foreach my $vmid (keys %$list) {
 658         my $d = $list->{$vmid};
 659
 660         my $running = defined($active_hash->{$vmid});
 661
 662         $d->{status} = $running ? 'running' : 'stopped';
 663
 664         my $cfspath = cfs_config_path($vmid);
 665         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 666
 667         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 668         $d->{name} =~ s/[\s]//g;
 669
 670         $d->{cpus} = $conf->{cpulimit} // 0;
 671
 672         if ($running) {
 673             my $res = get_container_disk_usage($vmid);
 674             $d->{disk} = $res->{used};
 675             $d->{maxdisk} = $res->{total};
 676         } else {
 677             $d->{disk} = 0;
 678             # use 4GB by default ??
 679             if (my $rootfs = $conf->{rootfs}) {
 680                 my $rootinfo = parse_ct_mountpoint($rootfs);
 681                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 682             } else {
 683                 $d->{maxdisk} = 4*1024*1024*1024;
 684             }
 685         }
 686
 687         $d->{mem} = 0;
 688         $d->{swap} = 0;
 689         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 690         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 691
 692         $d->{uptime} = 0;
 693         $d->{cpu} = 0;
 694
 695         $d->{netout} = 0;
 696         $d->{netin} = 0;
 697
 698         $d->{diskread} = 0;
 699         $d->{diskwrite} = 0;
 700
 701         $d->{template} = is_template($conf);
 702     }
 703
 704     foreach my $vmid (keys %$list) {
 705         my $d = $list->{$vmid};
 706         next if $d->{status} ne 'running';
 707
 708         $d->{uptime} = 100; # fixme:
 709
 710         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 711         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 712
 713         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 714         my @bytes = split(/\n/, $blkio_bytes);
 715         foreach my $byte (@bytes) {
 716             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 717                 $d->{diskread} = $2 if $key eq 'Read';
 718                 $d->{diskwrite} = $2 if $key eq 'Write';
 719             }
 720         }
 721     }
 722
 723     return $list;
 724 }
 725
 726 my $parse_size = sub {
 727     my ($value) = @_;
 728
 729     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 730     my ($size, $unit) = ($1, $3);
 731     if ($unit) {
 732         if ($unit eq 'K') {
 733             $size = $size * 1024;
 734         } elsif ($unit eq 'M') {
 735             $size = $size * 1024 * 1024;
 736         } elsif ($unit eq 'G') {
 737             $size = $size * 1024 * 1024 * 1024;
 738         }
 739     }
 740     return int($size);
 741 };
 742
 743 sub parse_ct_mountpoint {
 744     my ($data) = @_;
 745
 746     $data //= '';
 747
 748     my $res = {};
 749
 750     foreach my $p (split (/,/, $data)) {
 751         next if $p =~ m/^\s*$/;
 752
 753         if ($p =~ m/^(volume|backup|size|mp)=(.+)$/) {
 754             my ($k, $v) = ($1, $2);
 755             return undef if defined($res->{$k});
 756             $res->{$k} = $v;
 757         } else {
 758             if (!$res->{volume} && $p !~ m/=/) {
 759                 $res->{volume} = $p;
 760             } else {
 761                 return undef;
 762             }
 763         }
 764     }
 765
 766     return undef if !$res->{volume};
 767
 768     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 769
 770     if ($res->{size}) {
 771         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 772     }
 773
 774     return $res;
 775 }
 776
 777 sub print_ct_mountpoint {
 778     my ($info) = @_;
 779
 780     my $opts = '';
 781
 782     die "missing volume\n" if !$info->{volume};
 783
 784     foreach my $o ('size', 'backup') {
 785         $opts .= ",$o=$info->{$o}" if defined($info->{$o});
 786     }
 787
 788     return "$info->{volume}$opts";
 789 }
 790
 791 sub print_lxc_network {
 792     my $net = shift;
 793
 794     die "no network name defined\n" if !$net->{name};
 795
 796     my $res = "name=$net->{name}";
 797
 798     foreach my $k (qw(hwaddr mtu bridge ip gw ip6 gw6 firewall tag)) {
 799         next if !defined($net->{$k});
 800         $res .= ",$k=$net->{$k}";
 801     }
 802
 803     return $res;
 804 }
 805
 806 sub parse_lxc_network {
 807     my ($data) = @_;
 808
 809     my $res = {};
 810
 811     return $res if !$data;
 812
 813     foreach my $pv (split (/,/, $data)) {
 814         if ($pv =~ m/^(bridge|hwaddr|mtu|name|ip|ip6|gw|gw6|firewall|tag)=(\S+)$/) {
 815             $res->{$1} = $2;
 816         } else {
 817             return undef;
 818         }
 819     }
 820
 821     $res->{type} = 'veth';
 822     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 823
 824     return $res;
 825 }
 826
 827 sub read_cgroup_value {
 828     my ($group, $vmid, $name, $full) = @_;
 829
 830     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 831
 832     return PVE::Tools::file_get_contents($path) if $full;
 833
 834     return PVE::Tools::file_read_firstline($path);
 835 }
 836
 837 sub write_cgroup_value {
 838    my ($group, $vmid, $name, $value) = @_;
 839
 840    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 841    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 842
 843 }
 844
 845 sub find_lxc_console_pids {
 846
 847     my $res = {};
 848
 849     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 850         my ($pid) = @_;
 851
 852         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 853         return if !$cmdline;
 854
 855         my @args = split(/\0/, $cmdline);
 856
 857         # serach for lxc-console -n <vmid>
 858         return if scalar(@args) != 3;
 859         return if $args[1] ne '-n';
 860         return if $args[2] !~ m/^\d+$/;
 861         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 862
 863         my $vmid = $args[2];
 864
 865         push @{$res->{$vmid}}, $pid;
 866     });
 867
 868     return $res;
 869 }
 870
 871 sub find_lxc_pid {
 872     my ($vmid) = @_;
 873
 874     my $pid = undef;
 875     my $parser = sub {
 876         my $line = shift;
 877         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 878     };
 879     PVE::Tools::run_command(['lxc-info', '-n', $vmid], outfunc => $parser);
 880
 881     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 882
 883     return $pid;
 884 }
 885
 886 my $ipv4_reverse_mask = [
 887     '0.0.0.0',
 888     '128.0.0.0',
 889     '192.0.0.0',
 890     '224.0.0.0',
 891     '240.0.0.0',
 892     '248.0.0.0',
 893     '252.0.0.0',
 894     '254.0.0.0',
 895     '255.0.0.0',
 896     '255.128.0.0',
 897     '255.192.0.0',
 898     '255.224.0.0',
 899     '255.240.0.0',
 900     '255.248.0.0',
 901     '255.252.0.0',
 902     '255.254.0.0',
 903     '255.255.0.0',
 904     '255.255.128.0',
 905     '255.255.192.0',
 906     '255.255.224.0',
 907     '255.255.240.0',
 908     '255.255.248.0',
 909     '255.255.252.0',
 910     '255.255.254.0',
 911     '255.255.255.0',
 912     '255.255.255.128',
 913     '255.255.255.192',
 914     '255.255.255.224',
 915     '255.255.255.240',
 916     '255.255.255.248',
 917     '255.255.255.252',
 918     '255.255.255.254',
 919     '255.255.255.255',
 920 ];
 921
 922 # Note: we cannot use Net:IP, because that only allows strict
 923 # CIDR networks
 924 sub parse_ipv4_cidr {
 925     my ($cidr, $noerr) = @_;
 926
 927     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
 928         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
 929     }
 930
 931     return undef if $noerr;
 932
 933     die "unable to parse ipv4 address/mask\n";
 934 }
 935
 936 sub check_lock {
 937     my ($conf) = @_;
 938
 939     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
 940 }
 941
 942 sub update_lxc_config {
 943     my ($storage_cfg, $vmid, $conf) = @_;
 944
 945     my $dir = "/var/lib/lxc/$vmid";
 946
 947     if ($conf->{template}) {
 948
 949         unlink "$dir/config";
 950
 951         return;
 952     }
 953
 954     my $raw = '';
 955
 956     die "missing 'arch' - internal error" if !$conf->{arch};
 957     $raw .= "lxc.arch = $conf->{arch}\n";
 958
 959     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
 960     if ($ostype eq 'debian' || $ostype eq 'ubuntu' || $ostype eq 'centos') {
 961         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
 962     } else {
 963         die "implement me";
 964     }
 965
 966     if (!has_dev_console($conf)) {
 967         $raw .= "lxc.console = none\n";
 968         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
 969     }
 970
 971     my $ttycount = get_tty_count($conf);
 972     $raw .= "lxc.tty = $ttycount\n";
 973
 974     my $utsname = $conf->{hostname} || "CT$vmid";
 975     $raw .= "lxc.utsname = $utsname\n";
 976
 977     my $memory = $conf->{memory} || 512;
 978     my $swap = $conf->{swap} // 0;
 979
 980     my $lxcmem = int($memory*1024*1024);
 981     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
 982
 983     my $lxcswap = int(($memory + $swap)*1024*1024);
 984     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
 985
 986     if (my $cpulimit = $conf->{cpulimit}) {
 987         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
 988         my $value = int(100000*$cpulimit);
 989         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
 990     }
 991
 992     my $shares = $conf->{cpuunits} || 1024;
 993     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
 994
 995     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
 996     my $volid = $rootinfo->{volume};
 997     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
 998
 999     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1000         PVE::Storage::parse_volname($storage_cfg, $volid);
1001
1002     die "unable to use template as rootfs\n" if $isBase;
1003
1004     my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1005     my $path = PVE::Storage::path($storage_cfg, $volid);
1006
1007     if ($format eq 'subvol') {
1008         $raw .= "lxc.rootfs = $path\n";
1009     } elsif ($format eq 'raw') {
1010         if ($scfg->{path}) {
1011             $raw .= "lxc.rootfs = loop:$path\n";
1012         } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'rbd') {
1013             $raw .= "lxc.rootfs = $path\n";
1014         } else {
1015             die "unsupported storage type '$scfg->{type}'\n";
1016         }
1017     } else {
1018         die "unsupported image format '$format'\n";
1019     }
1020
1021     my $netcount = 0;
1022     foreach my $k (keys %$conf) {
1023         next if $k !~ m/^net(\d+)$/;
1024         my $ind = $1;
1025         my $d = parse_lxc_network($conf->{$k});
1026         $netcount++;
1027         $raw .= "lxc.network.type = veth\n";
1028         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1029         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1030         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1031         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1032     }
1033
1034     if (my $lxcconf = $conf->{lxc}) {
1035         foreach my $entry (@$lxcconf) {
1036             my ($k, $v) = @$entry;
1037             $netcount++ if $k eq 'lxc.network.type';
1038             $raw .= "$k = $v\n";
1039         }
1040     }
1041
1042     $raw .= "lxc.network.type = empty\n" if !$netcount;
1043
1044     File::Path::mkpath("$dir/rootfs");
1045
1046     PVE::Tools::file_set_contents("$dir/config", $raw);
1047 }
1048
1049 # verify and cleanup nameserver list (replace \0 with ' ')
1050 sub verify_nameserver_list {
1051     my ($nameserver_list) = @_;
1052
1053     my @list = ();
1054     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1055         PVE::JSONSchema::pve_verify_ip($server);
1056         push @list, $server;
1057     }
1058
1059     return join(' ', @list);
1060 }
1061
1062 sub verify_searchdomain_list {
1063     my ($searchdomain_list) = @_;
1064
1065     my @list = ();
1066     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1067         # todo: should we add checks for valid dns domains?
1068         push @list, $server;
1069     }
1070
1071     return join(' ', @list);
1072 }
1073
1074 sub update_pct_config {
1075     my ($vmid, $conf, $running, $param, $delete) = @_;
1076
1077     my @nohotplug;
1078
1079     my $rootdir;
1080     if ($running) {
1081         my $pid = find_lxc_pid($vmid);
1082         $rootdir = "/proc/$pid/root";
1083     }
1084
1085     if (defined($delete)) {
1086         foreach my $opt (@$delete) {
1087             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1088                 die "unable to delete required option '$opt'\n";
1089             } elsif ($opt eq 'swap') {
1090                 delete $conf->{$opt};
1091                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1092             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup' ||
1093                      $opt eq 'cmode') {
1094                 delete $conf->{$opt};
1095             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1096                      $opt eq 'tty' || $opt eq 'console') {
1097                 delete $conf->{$opt};
1098                 push @nohotplug, $opt;
1099                 next if $running;
1100             } elsif ($opt =~ m/^net(\d)$/) {
1101                 delete $conf->{$opt};
1102                 next if !$running;
1103                 my $netid = $1;
1104                 PVE::Network::veth_delete("veth${vmid}i$netid");
1105             } else {
1106                 die "implement me"
1107             }
1108             PVE::LXC::write_config($vmid, $conf) if $running;
1109         }
1110     }
1111
1112     # There's no separate swap size to configure, there's memory and "total"
1113     # memory (iow. memory+swap). This means we have to change them together.
1114     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1115     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1116     if (defined($wanted_memory) || defined($wanted_swap)) {
1117
1118         $wanted_memory //= ($conf->{memory} || 512);
1119         $wanted_swap //=  ($conf->{swap} || 0);
1120
1121         my $total = $wanted_memory + $wanted_swap;
1122         if ($running) {
1123             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1124             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1125         }
1126         $conf->{memory} = $wanted_memory;
1127         $conf->{swap} = $wanted_swap;
1128
1129         PVE::LXC::write_config($vmid, $conf) if $running;
1130     }
1131
1132     foreach my $opt (keys %$param) {
1133         my $value = $param->{$opt};
1134         if ($opt eq 'hostname') {
1135             $conf->{$opt} = $value;
1136         } elsif ($opt eq 'onboot') {
1137             $conf->{$opt} = $value ? 1 : 0;
1138         } elsif ($opt eq 'startup') {
1139             $conf->{$opt} = $value;
1140         } elsif ($opt eq 'cmode') {
1141             $conf->{$opt} = $value;
1142         } elsif ($opt eq 'tty' || $opt eq 'console') {
1143             $conf->{$opt} = $value;
1144             push @nohotplug, $opt;
1145             next if $running;
1146         } elsif ($opt eq 'nameserver') {
1147             my $list = verify_nameserver_list($value);
1148             $conf->{$opt} = $list;
1149             push @nohotplug, $opt;
1150             next if $running;
1151         } elsif ($opt eq 'searchdomain') {
1152             my $list = verify_searchdomain_list($value);
1153             $conf->{$opt} = $list;
1154             push @nohotplug, $opt;
1155             next if $running;
1156         } elsif ($opt eq 'cpulimit') {
1157             $conf->{$opt} = $value;
1158             push @nohotplug, $opt; # fixme: hotplug
1159             next;
1160         } elsif ($opt eq 'cpuunits') {
1161             $conf->{$opt} = $value;
1162             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1163         } elsif ($opt eq 'description') {
1164             $conf->{$opt} = PVE::Tools::encode_text($value);
1165         } elsif ($opt =~ m/^net(\d+)$/) {
1166             my $netid = $1;
1167             my $net = parse_lxc_network($value);
1168             if (!$running) {
1169                 $conf->{$opt} = print_lxc_network($net);
1170             } else {
1171                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1172             }
1173         } else {
1174             die "implement me: $opt";
1175         }
1176         PVE::LXC::write_config($vmid, $conf) if $running;
1177     }
1178
1179     if ($running && scalar(@nohotplug)) {
1180         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1181     }
1182 }
1183
1184 sub has_dev_console {
1185     my ($conf) = @_;
1186
1187     return !(defined($conf->{console}) && !$conf->{console});
1188 }
1189
1190 sub get_tty_count {
1191     my ($conf) = @_;
1192
1193     return $conf->{tty} // $confdesc->{tty}->{default};
1194 }
1195
1196 sub get_cmode {
1197     my ($conf) = @_;
1198
1199     return $conf->{cmode} // $confdesc->{cmode}->{default};
1200 }
1201
1202 sub get_console_command {
1203     my ($vmid, $conf) = @_;
1204
1205     my $cmode = get_cmode($conf);
1206
1207     if ($cmode eq 'console') {
1208         return ['lxc-console', '-n',  $vmid, '-t', 0];
1209     } elsif ($cmode eq 'tty') {
1210         return ['lxc-console', '-n',  $vmid];
1211     } elsif ($cmode eq 'shell') {
1212         return ['lxc-attach', '--clear-env', '-n', $vmid];
1213     } else {
1214         die "internal error";
1215     }
1216 }
1217
1218 sub get_primary_ips {
1219     my ($conf) = @_;
1220
1221     # return data from net0
1222
1223     return undef if !defined($conf->{net0});
1224     my $net = parse_lxc_network($conf->{net0});
1225
1226     my $ipv4 = $net->{ip};
1227     if ($ipv4) {
1228         if ($ipv4 =~ /^(dhcp|manual)$/) {
1229             $ipv4 = undef
1230         } else {
1231             $ipv4 =~ s!/\d+$!!;
1232         }
1233     }
1234     my $ipv6 = $net->{ip6};
1235     if ($ipv6) {
1236         if ($ipv6 =~ /^(dhcp|manual)$/) {
1237             $ipv6 = undef;
1238         } else {
1239             $ipv6 =~ s!/\d+$!!;
1240         }
1241     }
1242
1243     return ($ipv4, $ipv6);
1244 }
1245
1246
1247 sub destroy_lxc_container {
1248     my ($storage_cfg, $vmid, $conf) = @_;
1249
1250     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1251     if (defined($rootinfo->{volume})) {
1252         my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $rootinfo->{volume});
1253         PVE::Storage::vdisk_free($storage_cfg, $rootinfo->{volume}) if $vmid == $owner;;
1254     }
1255     rmdir "/var/lib/lxc/$vmid/rootfs";
1256     unlink "/var/lib/lxc/$vmid/config";
1257     rmdir "/var/lib/lxc/$vmid";
1258     destroy_config($vmid);
1259
1260     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1261     #PVE::Tools::run_command($cmd);
1262 }
1263
1264 sub vm_stop_cleanup {
1265     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1266
1267     eval {
1268         if (!$keepActive) {
1269             PVE::LXC::foreach_mountpoint($conf, sub {
1270                 my ($ms, $mountpoint) = @_;
1271                 PVE::Storage::deactivate_volumes($storage_cfg, [$mountpoint->{volume}]);
1272             });
1273         }
1274     };
1275     warn $@ if $@; # avoid errors - just warn
1276 }
1277
1278 my $safe_num_ne = sub {
1279     my ($a, $b) = @_;
1280
1281     return 0 if !defined($a) && !defined($b);
1282     return 1 if !defined($a);
1283     return 1 if !defined($b);
1284
1285     return $a != $b;
1286 };
1287
1288 my $safe_string_ne = sub {
1289     my ($a, $b) = @_;
1290
1291     return 0 if !defined($a) && !defined($b);
1292     return 1 if !defined($a);
1293     return 1 if !defined($b);
1294
1295     return $a ne $b;
1296 };
1297
1298 sub update_net {
1299     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1300
1301     if ($newnet->{type} ne 'veth') {
1302         # for when there are physical interfaces
1303         die "cannot update interface of type $newnet->{type}";
1304     }
1305
1306     my $veth = "veth${vmid}i${netid}";
1307     my $eth = $newnet->{name};
1308
1309     if (my $oldnetcfg = $conf->{$opt}) {
1310         my $oldnet = parse_lxc_network($oldnetcfg);
1311
1312         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1313             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1314
1315             PVE::Network::veth_delete($veth);
1316             delete $conf->{$opt};
1317             PVE::LXC::write_config($vmid, $conf);
1318
1319             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1320
1321         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1322                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1323                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1324
1325                 if ($oldnet->{bridge}) {
1326                     PVE::Network::tap_unplug($veth);
1327                     foreach (qw(bridge tag firewall)) {
1328                         delete $oldnet->{$_};
1329                     }
1330                     $conf->{$opt} = print_lxc_network($oldnet);
1331                     PVE::LXC::write_config($vmid, $conf);
1332                 }
1333
1334                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1335                 foreach (qw(bridge tag firewall)) {
1336                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1337                 }
1338                 $conf->{$opt} = print_lxc_network($oldnet);
1339                 PVE::LXC::write_config($vmid, $conf);
1340         }
1341     } else {
1342         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1343     }
1344
1345     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1346 }
1347
1348 sub hotplug_net {
1349     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1350
1351     my $veth = "veth${vmid}i${netid}";
1352     my $vethpeer = $veth . "p";
1353     my $eth = $newnet->{name};
1354
1355     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1356     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1357
1358     # attach peer in container
1359     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1360     PVE::Tools::run_command($cmd);
1361
1362     # link up peer in container
1363     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1364     PVE::Tools::run_command($cmd);
1365
1366     my $done = { type => 'veth' };
1367     foreach (qw(bridge tag firewall hwaddr name)) {
1368         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1369     }
1370     $conf->{$opt} = print_lxc_network($done);
1371
1372     PVE::LXC::write_config($vmid, $conf);
1373 }
1374
1375 sub update_ipconfig {
1376     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1377
1378     my $lxc_setup = PVE::LXCSetup->new($conf, $rootdir);
1379
1380     my $optdata = parse_lxc_network($conf->{$opt});
1381     my $deleted = [];
1382     my $added = [];
1383     my $nscmd = sub {
1384         my $cmdargs = shift;
1385         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1386     };
1387     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1388
1389     my $change_ip_config = sub {
1390         my ($ipversion) = @_;
1391
1392         my $family_opt = "-$ipversion";
1393         my $suffix = $ipversion == 4 ? '' : $ipversion;
1394         my $gw= "gw$suffix";
1395         my $ip= "ip$suffix";
1396
1397         my $newip = $newnet->{$ip};
1398         my $newgw = $newnet->{$gw};
1399         my $oldip = $optdata->{$ip};
1400
1401         my $change_ip = &$safe_string_ne($oldip, $newip);
1402         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1403
1404         return if !$change_ip && !$change_gw;
1405
1406         # step 1: add new IP, if this fails we cancel
1407         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1408             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1409             if (my $err = $@) {
1410                 warn $err;
1411                 return;
1412             }
1413         }
1414
1415         # step 2: replace gateway
1416         #   If this fails we delete the added IP and cancel.
1417         #   If it succeeds we save the config and delete the old IP, ignoring
1418         #   errors. The config is then saved.
1419         # Note: 'ip route replace' can add
1420         if ($change_gw) {
1421             if ($newgw) {
1422                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1423                 if (my $err = $@) {
1424                     warn $err;
1425                     # the route was not replaced, the old IP is still available
1426                     # rollback (delete new IP) and cancel
1427                     if ($change_ip) {
1428                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1429                         warn $@ if $@; # no need to die here
1430                     }
1431                     return;
1432                 }
1433             } else {
1434                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1435                 # if the route was not deleted, the guest might have deleted it manually
1436                 # warn and continue
1437                 warn $@ if $@;
1438             }
1439         }
1440
1441         # from this point on we save the configuration
1442         # step 3: delete old IP ignoring errors
1443         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1444             # We need to enable promote_secondaries, otherwise our newly added
1445             # address will be removed along with the old one.
1446             my $promote = 0;
1447             eval {
1448                 if ($ipversion == 4) {
1449                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1450                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1451                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1452                 }
1453                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1454             };
1455             warn $@ if $@; # no need to die here
1456
1457             if ($ipversion == 4) {
1458                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1459             }
1460         }
1461
1462         foreach my $property ($ip, $gw) {
1463             if ($newnet->{$property}) {
1464                 $optdata->{$property} = $newnet->{$property};
1465             } else {
1466                 delete $optdata->{$property};
1467             }
1468         }
1469         $conf->{$opt} = print_lxc_network($optdata);
1470         PVE::LXC::write_config($vmid, $conf);
1471         $lxc_setup->setup_network($conf);
1472     };
1473
1474     &$change_ip_config(4);
1475     &$change_ip_config(6);
1476
1477 }
1478
1479 # Internal snapshots
1480
1481 # NOTE: Snapshot create/delete involves several non-atomic
1482 # action, and can take a long time.
1483 # So we try to avoid locking the file and use 'lock' variable
1484 # inside the config file instead.
1485
1486 my $snapshot_copy_config = sub {
1487     my ($source, $dest) = @_;
1488
1489     foreach my $k (keys %$source) {
1490         next if $k eq 'snapshots';
1491         next if $k eq 'snapstate';
1492         next if $k eq 'snaptime';
1493         next if $k eq 'vmstate';
1494         next if $k eq 'lock';
1495         next if $k eq 'digest';
1496         next if $k eq 'description';
1497
1498         $dest->{$k} = $source->{$k};
1499     }
1500 };
1501
1502 my $snapshot_prepare = sub {
1503     my ($vmid, $snapname, $comment) = @_;
1504
1505     my $snap;
1506
1507     my $updatefn =  sub {
1508
1509         my $conf = load_config($vmid);
1510
1511         die "you can't take a snapshot if it's a template\n"
1512             if is_template($conf);
1513
1514         check_lock($conf);
1515
1516         $conf->{lock} = 'snapshot';
1517
1518         die "snapshot name '$snapname' already used\n"
1519             if defined($conf->{snapshots}->{$snapname});
1520
1521         my $storecfg = PVE::Storage::config();
1522         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1523
1524         $snap = $conf->{snapshots}->{$snapname} = {};
1525
1526         &$snapshot_copy_config($conf, $snap);
1527
1528         $snap->{'snapstate'} = "prepare";
1529         $snap->{'snaptime'} = time();
1530         $snap->{'description'} = $comment if $comment;
1531         $conf->{snapshots}->{$snapname} = $snap;
1532
1533         PVE::LXC::write_config($vmid, $conf);
1534     };
1535
1536     lock_container($vmid, 10, $updatefn);
1537
1538     return $snap;
1539 };
1540
1541 my $snapshot_commit = sub {
1542     my ($vmid, $snapname) = @_;
1543
1544     my $updatefn = sub {
1545
1546         my $conf = load_config($vmid);
1547
1548         die "missing snapshot lock\n"
1549             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1550
1551         die "snapshot '$snapname' does not exist\n"
1552             if !defined($conf->{snapshots}->{$snapname});
1553
1554         die "wrong snapshot state\n"
1555             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1556                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1557
1558         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1559         delete $conf->{lock};
1560         $conf->{parent} = $snapname;
1561
1562         PVE::LXC::write_config($vmid, $conf);
1563     };
1564
1565     lock_container($vmid, 10 ,$updatefn);
1566 };
1567
1568 sub has_feature {
1569     my ($feature, $conf, $storecfg, $snapname) = @_;
1570
1571     #Fixme add other drives if necessary.
1572     my $err;
1573
1574     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1575     $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $rootinfo->{volume}, $snapname);
1576
1577     return $err ? 0 : 1;
1578 }
1579
1580 sub snapshot_create {
1581     my ($vmid, $snapname, $comment) = @_;
1582
1583     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1584
1585     my $conf = load_config($vmid);
1586
1587     my $cmd = "/usr/bin/lxc-freeze -n $vmid";
1588     my $running = check_running($vmid);
1589     eval {
1590         if ($running) {
1591             PVE::Tools::run_command($cmd);
1592         };
1593
1594         my $storecfg = PVE::Storage::config();
1595         my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1596         my $volid = $rootinfo->{volume};
1597
1598         $cmd = "/usr/bin/lxc-unfreeze -n $vmid";
1599         if ($running) {
1600             PVE::Tools::run_command($cmd);
1601         };
1602
1603         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1604         &$snapshot_commit($vmid, $snapname);
1605     };
1606     if(my $err = $@) {
1607         snapshot_delete($vmid, $snapname, 1);
1608         die "$err\n";
1609     }
1610 }
1611
1612 sub snapshot_delete {
1613     my ($vmid, $snapname, $force) = @_;
1614
1615     my $snap;
1616
1617     my $conf;
1618
1619     my $updatefn =  sub {
1620
1621         $conf = load_config($vmid);
1622
1623         die "you can't delete a snapshot if vm is a template\n"
1624             if is_template($conf);
1625
1626         $snap = $conf->{snapshots}->{$snapname};
1627
1628         check_lock($conf);
1629
1630         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1631
1632         $snap->{snapstate} = 'delete';
1633
1634         PVE::LXC::write_config($vmid, $conf);
1635     };
1636
1637     lock_container($vmid, 10, $updatefn);
1638
1639     my $storecfg = PVE::Storage::config();
1640
1641     my $del_snap =  sub {
1642
1643         check_lock($conf);
1644
1645         if ($conf->{parent} eq $snapname) {
1646             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1647                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1648             } else {
1649                 delete $conf->{parent};
1650             }
1651         }
1652
1653         delete $conf->{snapshots}->{$snapname};
1654
1655         PVE::LXC::write_config($vmid, $conf);
1656     };
1657
1658     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1659     my $rootinfo = PVE::LXC::parse_ct_mountpoint($rootfs);
1660     my $volid = $rootinfo->{volume};
1661
1662     eval {
1663         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1664     };
1665     my $err = $@;
1666
1667     if(!$err || ($err && $force)) {
1668         lock_container($vmid, 10, $del_snap);
1669         if ($err) {
1670             die "Can't delete snapshot: $vmid $snapname $err\n";
1671         }
1672     }
1673 }
1674
1675 sub snapshot_rollback {
1676     my ($vmid, $snapname) = @_;
1677
1678     my $storecfg = PVE::Storage::config();
1679
1680     my $conf = load_config($vmid);
1681
1682     die "you can't rollback if vm is a template\n" if is_template($conf);
1683
1684     my $snap = $conf->{snapshots}->{$snapname};
1685
1686     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1687
1688     my $rootfs = $snap->{rootfs};
1689     my $rootinfo = PVE::LXC::parse_ct_mountpoint($rootfs);
1690     my $volid = $rootinfo->{volume};
1691
1692     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1693
1694     my $updatefn = sub {
1695
1696         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1697             if $snap->{snapstate};
1698
1699         check_lock($conf);
1700
1701         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1702
1703         die "unable to rollback vm $vmid: vm is running\n"
1704             if check_running($vmid);
1705
1706         $conf->{lock} = 'rollback';
1707
1708         my $forcemachine;
1709
1710         # copy snapshot config to current config
1711
1712         my $tmp_conf = $conf;
1713         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1714         $conf->{snapshots} = $tmp_conf->{snapshots};
1715         delete $conf->{snaptime};
1716         delete $conf->{snapname};
1717         $conf->{parent} = $snapname;
1718
1719         PVE::LXC::write_config($vmid, $conf);
1720     };
1721
1722     my $unlockfn = sub {
1723         delete $conf->{lock};
1724         PVE::LXC::write_config($vmid, $conf);
1725     };
1726
1727     lock_container($vmid, 10, $updatefn);
1728
1729     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1730
1731     lock_container($vmid, 5, $unlockfn);
1732 }
1733
1734 sub template_create {
1735     my ($vmid, $conf) = @_;
1736
1737     my $storecfg = PVE::Storage::config();
1738
1739     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1740     my $volid = $rootinfo->{volume};
1741
1742     die "Template feature is not available for '$volid'\n"
1743         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1744
1745     PVE::Storage::activate_volumes($storecfg, [$volid]);
1746
1747     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1748     $rootinfo->{volume} = $template_volid;
1749     $conf->{rootfs} = print_ct_mountpoint($rootinfo);
1750
1751     write_config($vmid, $conf);
1752 }
1753
1754 sub is_template {
1755     my ($conf) = @_;
1756
1757     return 1 if defined $conf->{template} && $conf->{template} == 1;
1758 }
1759
1760 sub foreach_mountpoint {
1761     my ($conf, $func) = @_;
1762
1763     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1764     $mountpoint->{mp} = '/'; # just to be sure
1765     &$func('rootfs', $mountpoint);
1766
1767     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1768         my $key = "mp$i";
1769         next if !defined($conf->{$key});
1770         $mountpoint = parse_ct_mountpoint($conf->{$key});
1771         &$func($key, $mountpoint);
1772     }
1773 }
1774
1775 sub loopdevices_list {
1776
1777     my $loopdev = {};
1778     my $parser = sub {
1779         my $line = shift;
1780         if ($line =~ m/^(\/dev\/loop\d+)\s+\d\s+\d\s+\d\s+\d\s(\S+)$/) {
1781             $loopdev->{$1} = $2;
1782         }
1783     };
1784
1785     PVE::Tools::run_command(['losetup'], outfunc => $parser);
1786
1787     return $loopdev;
1788 }
1789
1790 sub blockdevices_list {
1791
1792     my $bdevs = {};
1793     dir_glob_foreach("/sys/dev/block/", '(\d+):(\d+)', sub {
1794         my (undef, $major, $minor) = @_;
1795         my $bdev = readlink("/sys/dev/block/$major:$minor");
1796         $bdev =~ s/\.\.\/\.\.\/devices\/virtual\/block\//\/dev\//;
1797         $bdevs->{$bdev}->{major} = $major;
1798         $bdevs->{$bdev}->{minor} = $minor;
1799     });
1800     return $bdevs;
1801 }
1802
1803 1;