src/PVE/LXC.pm

   1 package PVE::LXC;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(EINTR);
   6
   7 use File::Path;
   8 use Fcntl ':flock';
   9
  10 use PVE::Cluster qw(cfs_register_file cfs_read_file);
  11 use PVE::Storage;
  12 use PVE::SafeSyslog;
  13 use PVE::INotify;
  14 use PVE::JSONSchema qw(get_standard_option);
  15 use PVE::Tools qw($IPV6RE $IPV4RE);
  16 use PVE::Network;
  17
  18 use Data::Dumper;
  19
  20 my $nodename = PVE::INotify::nodename();
  21
  22 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
  23
  24 PVE::JSONSchema::register_format('pve-lxc-network', \&verify_lxc_network);
  25 sub verify_lxc_network {
  26     my ($value, $noerr) = @_;
  27
  28     return $value if parse_lxc_network($value);
  29
  30     return undef if $noerr;
  31
  32     die "unable to parse network setting\n";
  33 }
  34
  35 PVE::JSONSchema::register_format('pve-ct-mountpoint', \&verify_ct_mountpoint);
  36 sub verify_ct_mountpoint {
  37     my ($value, $noerr) = @_;
  38
  39     return $value if parse_ct_mountpoint($value);
  40
  41     return undef if $noerr;
  42
  43     die "unable to parse CT mountpoint options\n";
  44 }
  45
  46 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
  47     type => 'string', format => 'pve-ct-mountpoint',
  48     typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+]',
  49     description => "Use volume as container root.",
  50     optional => 1,
  51 });
  52
  53 my $confdesc = {
  54     lock => {
  55         optional => 1,
  56         type => 'string',
  57         description => "Lock/unlock the VM.",
  58         enum => [qw(migrate backup snapshot rollback)],
  59     },
  60     onboot => {
  61         optional => 1,
  62         type => 'boolean',
  63         description => "Specifies whether a VM will be started during system bootup.",
  64         default => 0,
  65     },
  66     startup => get_standard_option('pve-startup-order'),
  67     template => {
  68         optional => 1,
  69         type => 'boolean',
  70         description => "Enable/disable Template.",
  71         default => 0,
  72     },
  73     arch => {
  74         optional => 1,
  75         type => 'string',
  76         enum => ['amd64', 'i386'],
  77         description => "OS architecture type.",
  78         default => 'amd64',
  79     },
  80     ostype => {
  81         optional => 1,
  82         type => 'string',
  83         enum => ['debian', 'ubuntu', 'centos'],
  84         description => "OS type. Corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf.",
  85     },
  86     console => {
  87         optional => 1,
  88         type => 'boolean',
  89         description => "Attach a console device (/dev/console) to the container.",
  90         default => 1,
  91     },
  92     tty => {
  93         optional => 1,
  94         type => 'integer',
  95         description => "Specify the number of tty available to the container",
  96         minimum => 0,
  97         maximum => 6,
  98         default => 2,
  99     },
 100     cpulimit => {
 101         optional => 1,
 102         type => 'number',
 103         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 104         minimum => 0,
 105         maximum => 128,
 106         default => 0,
 107     },
 108     cpuunits => {
 109         optional => 1,
 110         type => 'integer',
 111         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 112         minimum => 0,
 113         maximum => 500000,
 114         default => 1024,
 115     },
 116     memory => {
 117         optional => 1,
 118         type => 'integer',
 119         description => "Amount of RAM for the VM in MB.",
 120         minimum => 16,
 121         default => 512,
 122     },
 123     swap => {
 124         optional => 1,
 125         type => 'integer',
 126         description => "Amount of SWAP for the VM in MB.",
 127         minimum => 0,
 128         default => 512,
 129     },
 130     hostname => {
 131         optional => 1,
 132         description => "Set a host name for the container.",
 133         type => 'string',
 134         maxLength => 255,
 135     },
 136     description => {
 137         optional => 1,
 138         type => 'string',
 139         description => "Container description. Only used on the configuration web interface.",
 140     },
 141     searchdomain => {
 142         optional => 1,
 143         type => 'string',
 144         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 145     },
 146     nameserver => {
 147         optional => 1,
 148         type => 'string',
 149         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain or nameserver.",
 150     },
 151     rootfs => get_standard_option('pve-ct-rootfs'),
 152     parent => {
 153         optional => 1,
 154         type => 'string', format => 'pve-configid',
 155         maxLength => 40,
 156         description => "Parent snapshot name. This is used internally, and should not be modified.",
 157     },
 158     snaptime => {
 159         optional => 1,
 160         description => "Timestamp for snapshots.",
 161         type => 'integer',
 162         minimum => 0,
 163     },
 164     cmode => {
 165         optional => 1,
 166         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 167         type => 'string',
 168         enum => ['shell', 'console', 'tty'],
 169         default => 'tty',
 170     },
 171 };
 172
 173 my $valid_lxc_conf_keys = {
 174     'lxc.include' => 1,
 175     'lxc.arch' => 1,
 176     'lxc.utsname' => 1,
 177     'lxc.haltsignal' => 1,
 178     'lxc.rebootsignal' => 1,
 179     'lxc.stopsignal' => 1,
 180     'lxc.init_cmd' => 1,
 181     'lxc.network.type' => 1,
 182     'lxc.network.flags' => 1,
 183     'lxc.network.link' => 1,
 184     'lxc.network.mtu' => 1,
 185     'lxc.network.name' => 1,
 186     'lxc.network.hwaddr' => 1,
 187     'lxc.network.ipv4' => 1,
 188     'lxc.network.ipv4.gateway' => 1,
 189     'lxc.network.ipv6' => 1,
 190     'lxc.network.ipv6.gateway' => 1,
 191     'lxc.network.script.up' => 1,
 192     'lxc.network.script.down' => 1,
 193     'lxc.pts' => 1,
 194     'lxc.console.logfile' => 1,
 195     'lxc.console' => 1,
 196     'lxc.tty' => 1,
 197     'lxc.devttydir' => 1,
 198     'lxc.hook.autodev' => 1,
 199     'lxc.autodev' => 1,
 200     'lxc.kmsg' => 1,
 201     'lxc.mount' => 1,
 202     'lxc.mount.entry' => 1,
 203     'lxc.mount.auto' => 1,
 204     'lxc.rootfs' => 1,
 205     'lxc.rootfs.mount' => 1,
 206     'lxc.rootfs.options' => 1,
 207     # lxc.cgroup.*
 208     'lxc.cap.drop' => 1,
 209     'lxc.cap.keep' => 1,
 210     'lxc.aa_profile' => 1,
 211     'lxc.aa_allow_incomplete' => 1,
 212     'lxc.se_context' => 1,
 213     'lxc.seccomp' => 1,
 214     'lxc.id_map' => 1,
 215     'lxc.hook.pre-start' => 1,
 216     'lxc.hook.pre-mount' => 1,
 217     'lxc.hook.mount' => 1,
 218     'lxc.hook.start' => 1,
 219     'lxc.hook.post-stop' => 1,
 220     'lxc.hook.clone' => 1,
 221     'lxc.hook.destroy' => 1,
 222     'lxc.loglevel' => 1,
 223     'lxc.logfile' => 1,
 224     'lxc.start.auto' => 1,
 225     'lxc.start.delay' => 1,
 226     'lxc.start.order' => 1,
 227     'lxc.group' => 1,
 228     'lxc.environment' => 1,
 229     'lxc.' => 1,
 230     'lxc.' => 1,
 231     'lxc.' => 1,
 232     'lxc.' => 1,
 233 };
 234
 235 my $MAX_LXC_NETWORKS = 10;
 236 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 237     $confdesc->{"net$i"} = {
 238         optional => 1,
 239         type => 'string', format => 'pve-lxc-network',
 240         description => "Specifies network interfaces for the container.\n\n".
 241             "The string should have the follow format:\n\n".
 242             "-net<[0-9]> bridge=<vmbr<Nummber>>[,hwaddr=<MAC>]\n".
 243             "[,mtu=<Number>][,name=<String>][,ip=<IPv4Format/CIDR>]\n".
 244             ",ip6=<IPv6Format/CIDR>][,gw=<GatwayIPv4>]\n".
 245             ",gw6=<GatwayIPv6>][,firewall=<[1|0]>][,tag=<VlanNo>]",
 246     };
 247 }
 248
 249 my $MAX_MOUNT_POINTS = 10;
 250 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 251     $confdesc->{"mp$i"} = {
 252         optional => 1,
 253         type => 'string', format => 'pve-ct-mountpoint',
 254         typetext => '[volume=]volume,] [,backup=yes|no] [,size=\d+] [,mp=mountpoint]',
 255         description => "Use volume as container mount point.",
 256         optional => 1,
 257     };
 258 }
 259
 260 sub write_pct_config {
 261     my ($filename, $conf) = @_;
 262
 263     delete $conf->{snapstate}; # just to be sure
 264
 265     my $generate_raw_config = sub {
 266         my ($conf) = @_;
 267
 268         my $raw = '';
 269
 270         # add description as comment to top of file
 271         my $descr = $conf->{description} || '';
 272         foreach my $cl (split(/\n/, $descr)) {
 273             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
 274         }
 275
 276         foreach my $key (sort keys %$conf) {
 277             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' ||
 278                 $key eq 'snapshots' || $key eq 'snapname' || $key eq 'lxc';
 279             $raw .= "$key: $conf->{$key}\n";
 280         }
 281
 282         if (my $lxcconf = $conf->{lxc}) {
 283             foreach my $entry (@$lxcconf) {
 284                 my ($k, $v) = @$entry;
 285                 $raw .= "$k: $v\n";
 286             }
 287         }
 288
 289         return $raw;
 290     };
 291
 292     my $raw = &$generate_raw_config($conf);
 293
 294     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 295         $raw .= "\n[$snapname]\n";
 296         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
 297     }
 298
 299     return $raw;
 300 }
 301
 302 sub check_type {
 303     my ($key, $value) = @_;
 304
 305     die "unknown setting '$key'\n" if !$confdesc->{$key};
 306
 307     my $type = $confdesc->{$key}->{type};
 308
 309     if (!defined($value)) {
 310         die "got undefined value\n";
 311     }
 312
 313     if ($value =~ m/[\n\r]/) {
 314         die "property contains a line feed\n";
 315     }
 316
 317     if ($type eq 'boolean') {
 318         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
 319         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
 320         die "type check ('boolean') failed - got '$value'\n";
 321     } elsif ($type eq 'integer') {
 322         return int($1) if $value =~ m/^(\d+)$/;
 323         die "type check ('integer') failed - got '$value'\n";
 324     } elsif ($type eq 'number') {
 325         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
 326         die "type check ('number') failed - got '$value'\n";
 327     } elsif ($type eq 'string') {
 328         if (my $fmt = $confdesc->{$key}->{format}) {
 329             PVE::JSONSchema::check_format($fmt, $value);
 330             return $value;
 331         }
 332         return $value;
 333     } else {
 334         die "internal error"
 335     }
 336 }
 337
 338 sub parse_pct_config {
 339     my ($filename, $raw) = @_;
 340
 341     return undef if !defined($raw);
 342
 343     my $res = {
 344         digest => Digest::SHA::sha1_hex($raw),
 345         snapshots => {},
 346     };
 347
 348     $filename =~ m|/lxc/(\d+).conf$|
 349         || die "got strange filename '$filename'";
 350
 351     my $vmid = $1;
 352
 353     my $conf = $res;
 354     my $descr = '';
 355     my $section = '';
 356
 357     my @lines = split(/\n/, $raw);
 358     foreach my $line (@lines) {
 359         next if $line =~ m/^\s*$/;
 360
 361         if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 362             $section = $1;
 363             $conf->{description} = $descr if $descr;
 364             $descr = '';
 365             $conf = $res->{snapshots}->{$section} = {};
 366             next;
 367         }
 368
 369         if ($line =~ m/^\#(.*)\s*$/) {
 370             $descr .= PVE::Tools::decode_text($1) . "\n";
 371             next;
 372         }
 373
 374         if ($line =~ m/^(lxc\.[a-z0-9\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 375             my $key = $1;
 376             my $value = $3;
 377             if ($valid_lxc_conf_keys->{$key} || $key =~ m/^lxc\.cgroup\./) {
 378                 push @{$conf->{lxc}}, [$key, $value];
 379             } else {
 380                 warn "vm $vmid - unable to parse config: $line\n";
 381             }
 382         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 383             $descr .= PVE::Tools::decode_text($2);
 384         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 385             $conf->{snapstate} = $1;
 386         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
 387             my $key = $1;
 388             my $value = $2;
 389             eval { $value = check_type($key, $value); };
 390             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 391             $conf->{$key} = $value;
 392         } else {
 393             warn "vm $vmid - unable to parse config: $line\n";
 394         }
 395     }
 396
 397     $conf->{description} = $descr if $descr;
 398
 399     delete $res->{snapstate}; # just to be sure
 400
 401     return $res;
 402 }
 403
 404 sub config_list {
 405     my $vmlist = PVE::Cluster::get_vmlist();
 406     my $res = {};
 407     return $res if !$vmlist || !$vmlist->{ids};
 408     my $ids = $vmlist->{ids};
 409
 410     foreach my $vmid (keys %$ids) {
 411         next if !$vmid; # skip CT0
 412         my $d = $ids->{$vmid};
 413         next if !$d->{node} || $d->{node} ne $nodename;
 414         next if !$d->{type} || $d->{type} ne 'lxc';
 415         $res->{$vmid}->{type} = 'lxc';
 416     }
 417     return $res;
 418 }
 419
 420 sub cfs_config_path {
 421     my ($vmid, $node) = @_;
 422
 423     $node = $nodename if !$node;
 424     return "nodes/$node/lxc/$vmid.conf";
 425 }
 426
 427 sub config_file {
 428     my ($vmid, $node) = @_;
 429
 430     my $cfspath = cfs_config_path($vmid, $node);
 431     return "/etc/pve/$cfspath";
 432 }
 433
 434 sub load_config {
 435     my ($vmid) = @_;
 436
 437     my $cfspath = cfs_config_path($vmid);
 438
 439     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 440     die "container $vmid does not exists\n" if !defined($conf);
 441
 442     return $conf;
 443 }
 444
 445 sub create_config {
 446     my ($vmid, $conf) = @_;
 447
 448     my $dir = "/etc/pve/nodes/$nodename/lxc";
 449     mkdir $dir;
 450
 451     write_config($vmid, $conf);
 452 }
 453
 454 sub destroy_config {
 455     my ($vmid) = @_;
 456
 457     unlink config_file($vmid, $nodename);
 458 }
 459
 460 sub write_config {
 461     my ($vmid, $conf) = @_;
 462
 463     my $cfspath = cfs_config_path($vmid);
 464
 465     PVE::Cluster::cfs_write_file($cfspath, $conf);
 466 }
 467
 468 # flock: we use one file handle per process, so lock file
 469 # can be called multiple times and succeeds for the same process.
 470
 471 my $lock_handles =  {};
 472 my $lockdir = "/run/lock/lxc";
 473
 474 sub lock_filename {
 475     my ($vmid) = @_;
 476
 477     return "$lockdir/pve-config-{$vmid}.lock";
 478 }
 479
 480 sub lock_aquire {
 481     my ($vmid, $timeout) = @_;
 482
 483     $timeout = 10 if !$timeout;
 484     my $mode = LOCK_EX;
 485
 486     my $filename = lock_filename($vmid);
 487
 488     mkdir $lockdir if !-d $lockdir;
 489
 490     my $lock_func = sub {
 491         if (!$lock_handles->{$$}->{$filename}) {
 492             my $fh = new IO::File(">>$filename") ||
 493                 die "can't open file - $!\n";
 494             $lock_handles->{$$}->{$filename} = { fh => $fh, refcount => 0};
 495         }
 496
 497         if (!flock($lock_handles->{$$}->{$filename}->{fh}, $mode |LOCK_NB)) {
 498             print STDERR "trying to aquire lock...";
 499             my $success;
 500             while(1) {
 501                 $success = flock($lock_handles->{$$}->{$filename}->{fh}, $mode);
 502                 # try again on EINTR (see bug #273)
 503                 if ($success || ($! != EINTR)) {
 504                     last;
 505                 }
 506             }
 507             if (!$success) {
 508                 print STDERR " failed\n";
 509                 die "can't aquire lock - $!\n";
 510             }
 511
 512             $lock_handles->{$$}->{$filename}->{refcount}++;
 513
 514             print STDERR " OK\n";
 515         }
 516     };
 517
 518     eval { PVE::Tools::run_with_timeout($timeout, $lock_func); };
 519     my $err = $@;
 520     if ($err) {
 521         die "can't lock file '$filename' - $err";
 522     }
 523 }
 524
 525 sub lock_release {
 526     my ($vmid) = @_;
 527
 528     my $filename = lock_filename($vmid);
 529
 530     if (my $fh = $lock_handles->{$$}->{$filename}->{fh}) {
 531         my $refcount = --$lock_handles->{$$}->{$filename}->{refcount};
 532         if ($refcount <= 0) {
 533             $lock_handles->{$$}->{$filename} = undef;
 534             close ($fh);
 535         }
 536     }
 537 }
 538
 539 sub lock_container {
 540     my ($vmid, $timeout, $code, @param) = @_;
 541
 542     my $res;
 543
 544     lock_aquire($vmid, $timeout);
 545     eval { $res = &$code(@param) };
 546     my $err = $@;
 547     lock_release($vmid);
 548
 549     die $err if $err;
 550
 551     return $res;
 552 }
 553
 554 sub option_exists {
 555     my ($name) = @_;
 556
 557     return defined($confdesc->{$name});
 558 }
 559
 560 # add JSON properties for create and set function
 561 sub json_config_properties {
 562     my $prop = shift;
 563
 564     foreach my $opt (keys %$confdesc) {
 565         next if $opt eq 'parent' || $opt eq 'snaptime';
 566         next if $prop->{$opt};
 567         $prop->{$opt} = $confdesc->{$opt};
 568     }
 569
 570     return $prop;
 571 }
 572
 573 sub json_config_properties_no_rootfs {
 574     my $prop = shift;
 575
 576     foreach my $opt (keys %$confdesc) {
 577         next if $prop->{$opt};
 578         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'rootfs';
 579         $prop->{$opt} = $confdesc->{$opt};
 580     }
 581
 582     return $prop;
 583 }
 584
 585 # container status helpers
 586
 587 sub list_active_containers {
 588
 589     my $filename = "/proc/net/unix";
 590
 591     # similar test is used by lcxcontainers.c: list_active_containers
 592     my $res = {};
 593
 594     my $fh = IO::File->new ($filename, "r");
 595     return $res if !$fh;
 596
 597     while (defined(my $line = <$fh>)) {
 598         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 599             my $path = $1;
 600             if ($path =~ m!^@/var/lib/lxc/(\d+)/command$!) {
 601                 $res->{$1} = 1;
 602             }
 603         }
 604     }
 605
 606     close($fh);
 607
 608     return $res;
 609 }
 610
 611 # warning: this is slow
 612 sub check_running {
 613     my ($vmid) = @_;
 614
 615     my $active_hash = list_active_containers();
 616
 617     return 1 if defined($active_hash->{$vmid});
 618
 619     return undef;
 620 }
 621
 622 sub get_container_disk_usage {
 623     my ($vmid) = @_;
 624
 625     my $cmd = ['lxc-attach', '-n', $vmid, '--', 'df',  '-P', '-B', '1', '/'];
 626
 627     my $res = {
 628         total => 0,
 629         used => 0,
 630         avail => 0,
 631     };
 632
 633     my $parser = sub {
 634         my $line = shift;
 635         if (my ($fsid, $total, $used, $avail) = $line =~
 636             m/^(\S+.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+\d+%\s.*$/) {
 637             $res = {
 638                 total => $total,
 639                 used => $used,
 640                 avail => $avail,
 641             };
 642         }
 643     };
 644     eval { PVE::Tools::run_command($cmd, timeout => 1, outfunc => $parser); };
 645     warn $@ if $@;
 646
 647     return $res;
 648 }
 649
 650 sub vmstatus {
 651     my ($opt_vmid) = @_;
 652
 653     my $list = $opt_vmid ? { $opt_vmid => { type => 'lxc' }} : config_list();
 654
 655     my $active_hash = list_active_containers();
 656
 657     foreach my $vmid (keys %$list) {
 658         my $d = $list->{$vmid};
 659
 660         my $running = defined($active_hash->{$vmid});
 661
 662         $d->{status} = $running ? 'running' : 'stopped';
 663
 664         my $cfspath = cfs_config_path($vmid);
 665         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 666
 667         $d->{name} = $conf->{'hostname'} || "CT$vmid";
 668         $d->{name} =~ s/[\s]//g;
 669
 670         $d->{cpus} = $conf->{cpulimit} // 0;
 671
 672         if ($running) {
 673             my $res = get_container_disk_usage($vmid);
 674             $d->{disk} = $res->{used};
 675             $d->{maxdisk} = $res->{total};
 676         } else {
 677             $d->{disk} = 0;
 678             # use 4GB by default ??
 679             if (my $rootfs = $conf->{rootfs}) {
 680                 my $rootinfo = parse_ct_mountpoint($rootfs);
 681                 $d->{maxdisk} = int(($rootinfo->{size} || 4)*1024*1024)*1024;
 682             } else {
 683                 $d->{maxdisk} = 4*1024*1024*1024;
 684             }
 685         }
 686
 687         $d->{mem} = 0;
 688         $d->{swap} = 0;
 689         $d->{maxmem} = ($conf->{memory}||512)*1024*1024;
 690         $d->{maxswap} = ($conf->{swap}//0)*1024*1024;
 691
 692         $d->{uptime} = 0;
 693         $d->{cpu} = 0;
 694
 695         $d->{netout} = 0;
 696         $d->{netin} = 0;
 697
 698         $d->{diskread} = 0;
 699         $d->{diskwrite} = 0;
 700
 701         $d->{template} = is_template($conf);
 702     }
 703
 704     foreach my $vmid (keys %$list) {
 705         my $d = $list->{$vmid};
 706         next if $d->{status} ne 'running';
 707
 708         $d->{uptime} = 100; # fixme:
 709
 710         $d->{mem} = read_cgroup_value('memory', $vmid, 'memory.usage_in_bytes');
 711         $d->{swap} = read_cgroup_value('memory', $vmid, 'memory.memsw.usage_in_bytes') - $d->{mem};
 712
 713         my $blkio_bytes = read_cgroup_value('blkio', $vmid, 'blkio.throttle.io_service_bytes', 1);
 714         my @bytes = split(/\n/, $blkio_bytes);
 715         foreach my $byte (@bytes) {
 716             if (my ($key, $value) = $byte =~ /(Read|Write)\s+(\d+)/) {
 717                 $d->{diskread} = $2 if $key eq 'Read';
 718                 $d->{diskwrite} = $2 if $key eq 'Write';
 719             }
 720         }
 721     }
 722
 723     return $list;
 724 }
 725
 726 my $parse_size = sub {
 727     my ($value) = @_;
 728
 729     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 730     my ($size, $unit) = ($1, $3);
 731     if ($unit) {
 732         if ($unit eq 'K') {
 733             $size = $size * 1024;
 734         } elsif ($unit eq 'M') {
 735             $size = $size * 1024 * 1024;
 736         } elsif ($unit eq 'G') {
 737             $size = $size * 1024 * 1024 * 1024;
 738         }
 739     }
 740     return int($size);
 741 };
 742
 743 sub parse_ct_mountpoint {
 744     my ($data) = @_;
 745
 746     $data //= '';
 747
 748     my $res = {};
 749
 750     foreach my $p (split (/,/, $data)) {
 751         next if $p =~ m/^\s*$/;
 752
 753         if ($p =~ m/^(volume|backup|size|mp)=(.+)$/) {
 754             my ($k, $v) = ($1, $2);
 755             return undef if defined($res->{$k});
 756             $res->{$k} = $v;
 757         } else {
 758             if (!$res->{volume} && $p !~ m/=/) {
 759                 $res->{volume} = $p;
 760             } else {
 761                 return undef;
 762             }
 763         }
 764     }
 765
 766     return undef if !$res->{volume};
 767
 768     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 769
 770     if ($res->{size}) {
 771         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 772     }
 773
 774     return $res;
 775 }
 776
 777 sub print_ct_mountpoint {
 778     my ($info) = @_;
 779
 780     my $opts = '';
 781
 782     die "missing volume\n" if !$info->{volume};
 783
 784     foreach my $o ('size', 'backup') {
 785         $opts .= ",$o=$info->{$o}" if defined($info->{$o});
 786     }
 787
 788     return "$info->{volume}$opts";
 789 }
 790
 791 sub print_lxc_network {
 792     my $net = shift;
 793
 794     die "no network name defined\n" if !$net->{name};
 795
 796     my $res = "name=$net->{name}";
 797
 798     foreach my $k (qw(hwaddr mtu bridge ip gw ip6 gw6 firewall tag)) {
 799         next if !defined($net->{$k});
 800         $res .= ",$k=$net->{$k}";
 801     }
 802
 803     return $res;
 804 }
 805
 806 sub parse_lxc_network {
 807     my ($data) = @_;
 808
 809     my $res = {};
 810
 811     return $res if !$data;
 812
 813     foreach my $pv (split (/,/, $data)) {
 814         if ($pv =~ m/^(bridge|hwaddr|mtu|name|ip|ip6|gw|gw6|firewall|tag)=(\S+)$/) {
 815             $res->{$1} = $2;
 816         } else {
 817             return undef;
 818         }
 819     }
 820
 821     $res->{type} = 'veth';
 822     $res->{hwaddr} = PVE::Tools::random_ether_addr() if !$res->{hwaddr};
 823
 824     return $res;
 825 }
 826
 827 sub read_cgroup_value {
 828     my ($group, $vmid, $name, $full) = @_;
 829
 830     my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 831
 832     return PVE::Tools::file_get_contents($path) if $full;
 833
 834     return PVE::Tools::file_read_firstline($path);
 835 }
 836
 837 sub write_cgroup_value {
 838    my ($group, $vmid, $name, $value) = @_;
 839
 840    my $path = "/sys/fs/cgroup/$group/lxc/$vmid/$name";
 841    PVE::ProcFSTools::write_proc_entry($path, $value) if -e $path;
 842
 843 }
 844
 845 sub find_lxc_console_pids {
 846
 847     my $res = {};
 848
 849     PVE::Tools::dir_glob_foreach('/proc', '\d+', sub {
 850         my ($pid) = @_;
 851
 852         my $cmdline = PVE::Tools::file_read_firstline("/proc/$pid/cmdline");
 853         return if !$cmdline;
 854
 855         my @args = split(/\0/, $cmdline);
 856
 857         # serach for lxc-console -n <vmid>
 858         return if scalar(@args) != 3;
 859         return if $args[1] ne '-n';
 860         return if $args[2] !~ m/^\d+$/;
 861         return if $args[0] !~ m|^(/usr/bin/)?lxc-console$|;
 862
 863         my $vmid = $args[2];
 864
 865         push @{$res->{$vmid}}, $pid;
 866     });
 867
 868     return $res;
 869 }
 870
 871 sub find_lxc_pid {
 872     my ($vmid) = @_;
 873
 874     my $pid = undef;
 875     my $parser = sub {
 876         my $line = shift;
 877         $pid = $1 if $line =~ m/^PID:\s+(\d+)$/;
 878     };
 879     PVE::Tools::run_command(['lxc-info', '-n', $vmid], outfunc => $parser);
 880
 881     die "unable to get PID for CT $vmid (not running?)\n" if !$pid;
 882
 883     return $pid;
 884 }
 885
 886 my $ipv4_reverse_mask = [
 887     '0.0.0.0',
 888     '128.0.0.0',
 889     '192.0.0.0',
 890     '224.0.0.0',
 891     '240.0.0.0',
 892     '248.0.0.0',
 893     '252.0.0.0',
 894     '254.0.0.0',
 895     '255.0.0.0',
 896     '255.128.0.0',
 897     '255.192.0.0',
 898     '255.224.0.0',
 899     '255.240.0.0',
 900     '255.248.0.0',
 901     '255.252.0.0',
 902     '255.254.0.0',
 903     '255.255.0.0',
 904     '255.255.128.0',
 905     '255.255.192.0',
 906     '255.255.224.0',
 907     '255.255.240.0',
 908     '255.255.248.0',
 909     '255.255.252.0',
 910     '255.255.254.0',
 911     '255.255.255.0',
 912     '255.255.255.128',
 913     '255.255.255.192',
 914     '255.255.255.224',
 915     '255.255.255.240',
 916     '255.255.255.248',
 917     '255.255.255.252',
 918     '255.255.255.254',
 919     '255.255.255.255',
 920 ];
 921
 922 # Note: we cannot use Net:IP, because that only allows strict
 923 # CIDR networks
 924 sub parse_ipv4_cidr {
 925     my ($cidr, $noerr) = @_;
 926
 927     if ($cidr =~ m!^($IPV4RE)(?:/(\d+))$! && ($2 > 7) &&  ($2 < 32)) {
 928         return { address => $1, netmask => $ipv4_reverse_mask->[$2] };
 929     }
 930
 931     return undef if $noerr;
 932
 933     die "unable to parse ipv4 address/mask\n";
 934 }
 935
 936 sub check_lock {
 937     my ($conf) = @_;
 938
 939     die "VM is locked ($conf->{'lock'})\n" if $conf->{'lock'};
 940 }
 941
 942 sub update_lxc_config {
 943     my ($storage_cfg, $vmid, $conf) = @_;
 944
 945     my $dir = "/var/lib/lxc/$vmid";
 946
 947     if ($conf->{template}) {
 948
 949         unlink "$dir/config";
 950
 951         return;
 952     }
 953
 954     my $raw = '';
 955
 956     die "missing 'arch' - internal error" if !$conf->{arch};
 957     $raw .= "lxc.arch = $conf->{arch}\n";
 958
 959     my $ostype = $conf->{ostype} || die "missing 'ostype' - internal error";
 960     if ($ostype eq 'debian' || $ostype eq 'ubuntu' || $ostype eq 'centos') {
 961         $raw .= "lxc.include = /usr/share/lxc/config/$ostype.common.conf\n";
 962     } else {
 963         die "implement me";
 964     }
 965
 966     if (!has_dev_console($conf)) {
 967         $raw .= "lxc.console = none\n";
 968         $raw .= "lxc.cgroup.devices.deny = c 5:1 rwm\n";
 969     }
 970
 971     my $ttycount = get_tty_count($conf);
 972     $raw .= "lxc.tty = $ttycount\n";
 973
 974     my $utsname = $conf->{hostname} || "CT$vmid";
 975     $raw .= "lxc.utsname = $utsname\n";
 976
 977     my $memory = $conf->{memory} || 512;
 978     my $swap = $conf->{swap} // 0;
 979
 980     my $lxcmem = int($memory*1024*1024);
 981     $raw .= "lxc.cgroup.memory.limit_in_bytes = $lxcmem\n";
 982
 983     my $lxcswap = int(($memory + $swap)*1024*1024);
 984     $raw .= "lxc.cgroup.memory.memsw.limit_in_bytes = $lxcswap\n";
 985
 986     if (my $cpulimit = $conf->{cpulimit}) {
 987         $raw .= "lxc.cgroup.cpu.cfs_period_us = 100000\n";
 988         my $value = int(100000*$cpulimit);
 989         $raw .= "lxc.cgroup.cpu.cfs_quota_us = $value\n";
 990     }
 991
 992     my $shares = $conf->{cpuunits} || 1024;
 993     $raw .= "lxc.cgroup.cpu.shares = $shares\n";
 994
 995     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
 996     my $volid = $rootinfo->{volume};
 997     my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
 998
 999     my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1000         PVE::Storage::parse_volname($storage_cfg, $volid);
1001
1002     die "unable to use template as rootfs\n" if $isBase;
1003
1004     my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1005     my $path = PVE::Storage::path($storage_cfg, $volid);
1006
1007     if ($format eq 'subvol') {
1008         $raw .= "lxc.rootfs = $path\n";
1009     } elsif ($format eq 'raw') {
1010         if ($scfg->{path}) {
1011             $raw .= "lxc.rootfs = loop:$path\n";
1012         } elsif ($scfg->{type} eq 'drbd' || $scfg->{type} eq 'rbd') {
1013             $raw .= "lxc.rootfs = $path\n";
1014         } else {
1015             die "unsupported storage type '$scfg->{type}'\n";
1016         }
1017     } else {
1018         die "unsupported image format '$format'\n";
1019     }
1020
1021     my $netcount = 0;
1022     foreach my $k (keys %$conf) {
1023         next if $k !~ m/^net(\d+)$/;
1024         my $ind = $1;
1025         my $d = parse_lxc_network($conf->{$k});
1026         $netcount++;
1027         $raw .= "lxc.network.type = veth\n";
1028         $raw .= "lxc.network.veth.pair = veth${vmid}i${ind}\n";
1029         $raw .= "lxc.network.hwaddr = $d->{hwaddr}\n" if defined($d->{hwaddr});
1030         $raw .= "lxc.network.name = $d->{name}\n" if defined($d->{name});
1031         $raw .= "lxc.network.mtu = $d->{mtu}\n" if defined($d->{mtu});
1032     }
1033
1034     if (my $lxcconf = $conf->{lxc}) {
1035         foreach my $entry (@$lxcconf) {
1036             my ($k, $v) = @$entry;
1037             $netcount++ if $k eq 'lxc.network.type';
1038             $raw .= "$k = $v\n";
1039         }
1040     }
1041
1042     $raw .= "lxc.network.type = empty\n" if !$netcount;
1043
1044     File::Path::mkpath("$dir/rootfs");
1045
1046     PVE::Tools::file_set_contents("$dir/config", $raw);
1047 }
1048
1049 # verify and cleanup nameserver list (replace \0 with ' ')
1050 sub verify_nameserver_list {
1051     my ($nameserver_list) = @_;
1052
1053     my @list = ();
1054     foreach my $server (PVE::Tools::split_list($nameserver_list)) {
1055         PVE::JSONSchema::pve_verify_ip($server);
1056         push @list, $server;
1057     }
1058
1059     return join(' ', @list);
1060 }
1061
1062 sub verify_searchdomain_list {
1063     my ($searchdomain_list) = @_;
1064
1065     my @list = ();
1066     foreach my $server (PVE::Tools::split_list($searchdomain_list)) {
1067         # todo: should we add checks for valid dns domains?
1068         push @list, $server;
1069     }
1070
1071     return join(' ', @list);
1072 }
1073
1074 sub update_pct_config {
1075     my ($vmid, $conf, $running, $param, $delete) = @_;
1076
1077     my @nohotplug;
1078
1079     my $rootdir;
1080     if ($running) {
1081         my $pid = find_lxc_pid($vmid);
1082         $rootdir = "/proc/$pid/root";
1083     }
1084
1085     if (defined($delete)) {
1086         foreach my $opt (@$delete) {
1087             if ($opt eq 'hostname' || $opt eq 'memory' || $opt eq 'rootfs') {
1088                 die "unable to delete required option '$opt'\n";
1089             } elsif ($opt eq 'swap') {
1090                 delete $conf->{$opt};
1091                 write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", -1);
1092             } elsif ($opt eq 'description' || $opt eq 'onboot' || $opt eq 'startup') {
1093                 delete $conf->{$opt};
1094             } elsif ($opt eq 'nameserver' || $opt eq 'searchdomain' ||
1095                      $opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1096                 delete $conf->{$opt};
1097                 push @nohotplug, $opt;
1098                 next if $running;
1099             } elsif ($opt =~ m/^net(\d)$/) {
1100                 delete $conf->{$opt};
1101                 next if !$running;
1102                 my $netid = $1;
1103                 PVE::Network::veth_delete("veth${vmid}i$netid");
1104             } else {
1105                 die "implement me"
1106             }
1107             PVE::LXC::write_config($vmid, $conf) if $running;
1108         }
1109     }
1110
1111     # There's no separate swap size to configure, there's memory and "total"
1112     # memory (iow. memory+swap). This means we have to change them together.
1113     my $wanted_memory = PVE::Tools::extract_param($param, 'memory');
1114     my $wanted_swap =  PVE::Tools::extract_param($param, 'swap');
1115     if (defined($wanted_memory) || defined($wanted_swap)) {
1116
1117         $wanted_memory //= ($conf->{memory} || 512);
1118         $wanted_swap //=  ($conf->{swap} || 0);
1119
1120         my $total = $wanted_memory + $wanted_swap;
1121         if ($running) {
1122             write_cgroup_value("memory", $vmid, "memory.limit_in_bytes", int($wanted_memory*1024*1024));
1123             write_cgroup_value("memory", $vmid, "memory.memsw.limit_in_bytes", int($total*1024*1024));
1124         }
1125         $conf->{memory} = $wanted_memory;
1126         $conf->{swap} = $wanted_swap;
1127
1128         PVE::LXC::write_config($vmid, $conf) if $running;
1129     }
1130
1131     foreach my $opt (keys %$param) {
1132         my $value = $param->{$opt};
1133         if ($opt eq 'hostname') {
1134             $conf->{$opt} = $value;
1135         } elsif ($opt eq 'onboot') {
1136             $conf->{$opt} = $value ? 1 : 0;
1137         } elsif ($opt eq 'startup') {
1138             $conf->{$opt} = $value;
1139         } elsif ($opt eq 'tty' || $opt eq 'console' || $opt eq 'cmode') {
1140             $conf->{$opt} = $value;
1141             push @nohotplug, $opt;
1142             next if $running;
1143         } elsif ($opt eq 'nameserver') {
1144             my $list = verify_nameserver_list($value);
1145             $conf->{$opt} = $list;
1146             push @nohotplug, $opt;
1147             next if $running;
1148         } elsif ($opt eq 'searchdomain') {
1149             my $list = verify_searchdomain_list($value);
1150             $conf->{$opt} = $list;
1151             push @nohotplug, $opt;
1152             next if $running;
1153         } elsif ($opt eq 'cpulimit') {
1154             $conf->{$opt} = $value;
1155             push @nohotplug, $opt; # fixme: hotplug
1156             next;
1157         } elsif ($opt eq 'cpuunits') {
1158             $conf->{$opt} = $value;
1159             write_cgroup_value("cpu", $vmid, "cpu.shares", $value);
1160         } elsif ($opt eq 'description') {
1161             $conf->{$opt} = PVE::Tools::encode_text($value);
1162         } elsif ($opt =~ m/^net(\d+)$/) {
1163             my $netid = $1;
1164             my $net = parse_lxc_network($value);
1165             if (!$running) {
1166                 $conf->{$opt} = print_lxc_network($net);
1167             } else {
1168                 update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1169             }
1170         } else {
1171             die "implement me: $opt";
1172         }
1173         PVE::LXC::write_config($vmid, $conf) if $running;
1174     }
1175
1176     if ($running && scalar(@nohotplug)) {
1177         die "unable to modify " . join(',', @nohotplug) . " while container is running\n";
1178     }
1179 }
1180
1181 sub has_dev_console {
1182     my ($conf) = @_;
1183
1184     return !(defined($conf->{console}) && !$conf->{console});
1185 }
1186
1187 sub get_tty_count {
1188     my ($conf) = @_;
1189
1190     return $conf->{tty} // $confdesc->{tty}->{default};
1191 }
1192
1193 sub get_cmode {
1194     my ($conf) = @_;
1195
1196     return $conf->{cmode} // $confdesc->{cmode}->{default};
1197 }
1198
1199 sub get_console_command {
1200     my ($vmid, $conf) = @_;
1201
1202     my $cmode = get_cmode($conf);
1203
1204     if ($cmode eq 'console') {
1205         return ['lxc-console', '-n',  $vmid, '-t', 0];
1206     } elsif ($cmode eq 'tty') {
1207         return ['lxc-console', '-n',  $vmid];
1208     } elsif ($cmode eq 'shell') {
1209         return ['lxc-attach', '--clear-env', '-n', $vmid];
1210     } else {
1211         die "internal error";
1212     }
1213 }
1214
1215 sub get_primary_ips {
1216     my ($conf) = @_;
1217
1218     # return data from net0
1219
1220     return undef if !defined($conf->{net0});
1221     my $net = parse_lxc_network($conf->{net0});
1222
1223     my $ipv4 = $net->{ip};
1224     if ($ipv4) {
1225         if ($ipv4 =~ /^(dhcp|manual)$/) {
1226             $ipv4 = undef
1227         } else {
1228             $ipv4 =~ s!/\d+$!!;
1229         }
1230     }
1231     my $ipv6 = $net->{ip6};
1232     if ($ipv6) {
1233         if ($ipv6 =~ /^(dhcp|manual)$/) {
1234             $ipv6 = undef;
1235         } else {
1236             $ipv6 =~ s!/\d+$!!;
1237         }
1238     }
1239
1240     return ($ipv4, $ipv6);
1241 }
1242
1243
1244 sub destroy_lxc_container {
1245     my ($storage_cfg, $vmid, $conf) = @_;
1246
1247     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1248     if (defined($rootinfo->{volume})) {
1249         my ($vtype, $name, $owner) = PVE::Storage::parse_volname($storage_cfg, $rootinfo->{volume});
1250         PVE::Storage::vdisk_free($storage_cfg, $rootinfo->{volume}) if $vmid == $owner;;
1251     }
1252     rmdir "/var/lib/lxc/$vmid/rootfs";
1253     unlink "/var/lib/lxc/$vmid/config";
1254     rmdir "/var/lib/lxc/$vmid";
1255     destroy_config($vmid);
1256
1257     #my $cmd = ['lxc-destroy', '-n', $vmid ];
1258     #PVE::Tools::run_command($cmd);
1259 }
1260
1261 sub vm_stop_cleanup {
1262     my ($storage_cfg, $vmid, $conf, $keepActive) = @_;
1263
1264     eval {
1265         if (!$keepActive) {
1266
1267             my $loopdevs = loopdevices_list();
1268
1269             PVE::LXC::foreach_mountpoint($conf, sub {
1270                 my ($ms, $mountpoint) = @_;
1271
1272                 my $volid = $mountpoint->{volume};
1273                 #detach loopdevices of non rootfs mountpoints
1274                 my ($storage, $volname) = PVE::Storage::parse_volume_id($volid);
1275                 my $scfg = PVE::Storage::storage_config($storage_cfg, $storage);
1276                 my ($vtype, undef, undef, undef, undef, $isBase, $format) =
1277                     PVE::Storage::parse_volname($storage_cfg, $volid);
1278
1279                 if($ms ne 'rootfs' && $format eq 'raw' && ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs')) {
1280                     my $path = PVE::Storage::path($storage_cfg, $volid);
1281                     foreach my $dev (keys %$loopdevs){
1282                         PVE::Tools::run_command(['losetup', '-d', $dev]) if $loopdevs->{$dev} eq $path;
1283                     }
1284                 }
1285
1286                 PVE::Storage::deactivate_volumes($storage_cfg, [$volid]);
1287
1288             });
1289         }
1290     };
1291     warn $@ if $@; # avoid errors - just warn
1292 }
1293
1294 my $safe_num_ne = sub {
1295     my ($a, $b) = @_;
1296
1297     return 0 if !defined($a) && !defined($b);
1298     return 1 if !defined($a);
1299     return 1 if !defined($b);
1300
1301     return $a != $b;
1302 };
1303
1304 my $safe_string_ne = sub {
1305     my ($a, $b) = @_;
1306
1307     return 0 if !defined($a) && !defined($b);
1308     return 1 if !defined($a);
1309     return 1 if !defined($b);
1310
1311     return $a ne $b;
1312 };
1313
1314 sub update_net {
1315     my ($vmid, $conf, $opt, $newnet, $netid, $rootdir) = @_;
1316
1317     if ($newnet->{type} ne 'veth') {
1318         # for when there are physical interfaces
1319         die "cannot update interface of type $newnet->{type}";
1320     }
1321
1322     my $veth = "veth${vmid}i${netid}";
1323     my $eth = $newnet->{name};
1324
1325     if (my $oldnetcfg = $conf->{$opt}) {
1326         my $oldnet = parse_lxc_network($oldnetcfg);
1327
1328         if (&$safe_string_ne($oldnet->{hwaddr}, $newnet->{hwaddr}) ||
1329             &$safe_string_ne($oldnet->{name}, $newnet->{name})) {
1330
1331             PVE::Network::veth_delete($veth);
1332             delete $conf->{$opt};
1333             PVE::LXC::write_config($vmid, $conf);
1334
1335             hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1336
1337         } elsif (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
1338                  &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
1339                  &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
1340
1341                 if ($oldnet->{bridge}) {
1342                     PVE::Network::tap_unplug($veth);
1343                     foreach (qw(bridge tag firewall)) {
1344                         delete $oldnet->{$_};
1345                     }
1346                     $conf->{$opt} = print_lxc_network($oldnet);
1347                     PVE::LXC::write_config($vmid, $conf);
1348                 }
1349
1350                 PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1351                 foreach (qw(bridge tag firewall)) {
1352                     $oldnet->{$_} = $newnet->{$_} if $newnet->{$_};
1353                 }
1354                 $conf->{$opt} = print_lxc_network($oldnet);
1355                 PVE::LXC::write_config($vmid, $conf);
1356         }
1357     } else {
1358         hotplug_net($vmid, $conf, $opt, $newnet, $netid);
1359     }
1360
1361     update_ipconfig($vmid, $conf, $opt, $eth, $newnet, $rootdir);
1362 }
1363
1364 sub hotplug_net {
1365     my ($vmid, $conf, $opt, $newnet, $netid) = @_;
1366
1367     my $veth = "veth${vmid}i${netid}";
1368     my $vethpeer = $veth . "p";
1369     my $eth = $newnet->{name};
1370
1371     PVE::Network::veth_create($veth, $vethpeer, $newnet->{bridge}, $newnet->{hwaddr});
1372     PVE::Network::tap_plug($veth, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
1373
1374     # attach peer in container
1375     my $cmd = ['lxc-device', '-n', $vmid, 'add', $vethpeer, "$eth" ];
1376     PVE::Tools::run_command($cmd);
1377
1378     # link up peer in container
1379     $cmd = ['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', '/sbin/ip', 'link', 'set', $eth ,'up'  ];
1380     PVE::Tools::run_command($cmd);
1381
1382     my $done = { type => 'veth' };
1383     foreach (qw(bridge tag firewall hwaddr name)) {
1384         $done->{$_} = $newnet->{$_} if $newnet->{$_};
1385     }
1386     $conf->{$opt} = print_lxc_network($done);
1387
1388     PVE::LXC::write_config($vmid, $conf);
1389 }
1390
1391 sub update_ipconfig {
1392     my ($vmid, $conf, $opt, $eth, $newnet, $rootdir) = @_;
1393
1394     my $lxc_setup = PVE::LXCSetup->new($conf, $rootdir);
1395
1396     my $optdata = parse_lxc_network($conf->{$opt});
1397     my $deleted = [];
1398     my $added = [];
1399     my $nscmd = sub {
1400         my $cmdargs = shift;
1401         PVE::Tools::run_command(['lxc-attach', '-n', $vmid, '-s', 'NETWORK', '--', @_], %$cmdargs);
1402     };
1403     my $ipcmd = sub { &$nscmd({}, '/sbin/ip', @_) };
1404
1405     my $change_ip_config = sub {
1406         my ($ipversion) = @_;
1407
1408         my $family_opt = "-$ipversion";
1409         my $suffix = $ipversion == 4 ? '' : $ipversion;
1410         my $gw= "gw$suffix";
1411         my $ip= "ip$suffix";
1412
1413         my $newip = $newnet->{$ip};
1414         my $newgw = $newnet->{$gw};
1415         my $oldip = $optdata->{$ip};
1416
1417         my $change_ip = &$safe_string_ne($oldip, $newip);
1418         my $change_gw = &$safe_string_ne($optdata->{$gw}, $newgw);
1419
1420         return if !$change_ip && !$change_gw;
1421
1422         # step 1: add new IP, if this fails we cancel
1423         if ($change_ip && $newip && $newip !~ /^(?:auto|dhcp)$/) {
1424             eval { &$ipcmd($family_opt, 'addr', 'add', $newip, 'dev', $eth); };
1425             if (my $err = $@) {
1426                 warn $err;
1427                 return;
1428             }
1429         }
1430
1431         # step 2: replace gateway
1432         #   If this fails we delete the added IP and cancel.
1433         #   If it succeeds we save the config and delete the old IP, ignoring
1434         #   errors. The config is then saved.
1435         # Note: 'ip route replace' can add
1436         if ($change_gw) {
1437             if ($newgw) {
1438                 eval { &$ipcmd($family_opt, 'route', 'replace', 'default', 'via', $newgw); };
1439                 if (my $err = $@) {
1440                     warn $err;
1441                     # the route was not replaced, the old IP is still available
1442                     # rollback (delete new IP) and cancel
1443                     if ($change_ip) {
1444                         eval { &$ipcmd($family_opt, 'addr', 'del', $newip, 'dev', $eth); };
1445                         warn $@ if $@; # no need to die here
1446                     }
1447                     return;
1448                 }
1449             } else {
1450                 eval { &$ipcmd($family_opt, 'route', 'del', 'default'); };
1451                 # if the route was not deleted, the guest might have deleted it manually
1452                 # warn and continue
1453                 warn $@ if $@;
1454             }
1455         }
1456
1457         # from this point on we save the configuration
1458         # step 3: delete old IP ignoring errors
1459         if ($change_ip && $oldip && $oldip !~ /^(?:auto|dhcp)$/) {
1460             # We need to enable promote_secondaries, otherwise our newly added
1461             # address will be removed along with the old one.
1462             my $promote = 0;
1463             eval {
1464                 if ($ipversion == 4) {
1465                     &$nscmd({ outfunc => sub { $promote = int(shift) } },
1466                             'cat', "/proc/sys/net/ipv4/conf/$eth/promote_secondaries");
1467                     &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=1");
1468                 }
1469                 &$ipcmd($family_opt, 'addr', 'del', $oldip, 'dev', $eth);
1470             };
1471             warn $@ if $@; # no need to die here
1472
1473             if ($ipversion == 4) {
1474                 &$nscmd({}, 'sysctl', "net.ipv4.conf.$eth.promote_secondaries=$promote");
1475             }
1476         }
1477
1478         foreach my $property ($ip, $gw) {
1479             if ($newnet->{$property}) {
1480                 $optdata->{$property} = $newnet->{$property};
1481             } else {
1482                 delete $optdata->{$property};
1483             }
1484         }
1485         $conf->{$opt} = print_lxc_network($optdata);
1486         PVE::LXC::write_config($vmid, $conf);
1487         $lxc_setup->setup_network($conf);
1488     };
1489
1490     &$change_ip_config(4);
1491     &$change_ip_config(6);
1492
1493 }
1494
1495 # Internal snapshots
1496
1497 # NOTE: Snapshot create/delete involves several non-atomic
1498 # action, and can take a long time.
1499 # So we try to avoid locking the file and use 'lock' variable
1500 # inside the config file instead.
1501
1502 my $snapshot_copy_config = sub {
1503     my ($source, $dest) = @_;
1504
1505     foreach my $k (keys %$source) {
1506         next if $k eq 'snapshots';
1507         next if $k eq 'snapstate';
1508         next if $k eq 'snaptime';
1509         next if $k eq 'vmstate';
1510         next if $k eq 'lock';
1511         next if $k eq 'digest';
1512         next if $k eq 'description';
1513
1514         $dest->{$k} = $source->{$k};
1515     }
1516 };
1517
1518 my $snapshot_prepare = sub {
1519     my ($vmid, $snapname, $comment) = @_;
1520
1521     my $snap;
1522
1523     my $updatefn =  sub {
1524
1525         my $conf = load_config($vmid);
1526
1527         die "you can't take a snapshot if it's a template\n"
1528             if is_template($conf);
1529
1530         check_lock($conf);
1531
1532         $conf->{lock} = 'snapshot';
1533
1534         die "snapshot name '$snapname' already used\n"
1535             if defined($conf->{snapshots}->{$snapname});
1536
1537         my $storecfg = PVE::Storage::config();
1538         die "snapshot feature is not available\n" if !has_feature('snapshot', $conf, $storecfg);
1539
1540         $snap = $conf->{snapshots}->{$snapname} = {};
1541
1542         &$snapshot_copy_config($conf, $snap);
1543
1544         $snap->{'snapstate'} = "prepare";
1545         $snap->{'snaptime'} = time();
1546         $snap->{'description'} = $comment if $comment;
1547         $conf->{snapshots}->{$snapname} = $snap;
1548
1549         PVE::LXC::write_config($vmid, $conf);
1550     };
1551
1552     lock_container($vmid, 10, $updatefn);
1553
1554     return $snap;
1555 };
1556
1557 my $snapshot_commit = sub {
1558     my ($vmid, $snapname) = @_;
1559
1560     my $updatefn = sub {
1561
1562         my $conf = load_config($vmid);
1563
1564         die "missing snapshot lock\n"
1565             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
1566
1567         die "snapshot '$snapname' does not exist\n"
1568             if !defined($conf->{snapshots}->{$snapname});
1569
1570         die "wrong snapshot state\n"
1571             if !($conf->{snapshots}->{$snapname}->{'snapstate'} &&
1572                  $conf->{snapshots}->{$snapname}->{'snapstate'} eq "prepare");
1573
1574         delete $conf->{snapshots}->{$snapname}->{'snapstate'};
1575         delete $conf->{lock};
1576         $conf->{parent} = $snapname;
1577
1578         PVE::LXC::write_config($vmid, $conf);
1579     };
1580
1581     lock_container($vmid, 10 ,$updatefn);
1582 };
1583
1584 sub has_feature {
1585     my ($feature, $conf, $storecfg, $snapname) = @_;
1586
1587     #Fixme add other drives if necessary.
1588     my $err;
1589
1590     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1591     $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $rootinfo->{volume}, $snapname);
1592
1593     return $err ? 0 : 1;
1594 }
1595
1596 sub snapshot_create {
1597     my ($vmid, $snapname, $comment) = @_;
1598
1599     my $snap = &$snapshot_prepare($vmid, $snapname, $comment);
1600
1601     my $conf = load_config($vmid);
1602
1603     my $cmd = "/usr/bin/lxc-freeze -n $vmid";
1604     my $running = check_running($vmid);
1605     eval {
1606         if ($running) {
1607             PVE::Tools::run_command($cmd);
1608         };
1609
1610         my $storecfg = PVE::Storage::config();
1611         my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1612         my $volid = $rootinfo->{volume};
1613
1614         $cmd = "/usr/bin/lxc-unfreeze -n $vmid";
1615         if ($running) {
1616             PVE::Tools::run_command($cmd);
1617         };
1618
1619         PVE::Storage::volume_snapshot($storecfg, $volid, $snapname);
1620         &$snapshot_commit($vmid, $snapname);
1621     };
1622     if(my $err = $@) {
1623         snapshot_delete($vmid, $snapname, 1);
1624         die "$err\n";
1625     }
1626 }
1627
1628 sub snapshot_delete {
1629     my ($vmid, $snapname, $force) = @_;
1630
1631     my $snap;
1632
1633     my $conf;
1634
1635     my $updatefn =  sub {
1636
1637         $conf = load_config($vmid);
1638
1639         die "you can't delete a snapshot if vm is a template\n"
1640             if is_template($conf);
1641
1642         $snap = $conf->{snapshots}->{$snapname};
1643
1644         check_lock($conf);
1645
1646         die "snapshot '$snapname' does not exist\n" if !defined($snap);
1647
1648         $snap->{snapstate} = 'delete';
1649
1650         PVE::LXC::write_config($vmid, $conf);
1651     };
1652
1653     lock_container($vmid, 10, $updatefn);
1654
1655     my $storecfg = PVE::Storage::config();
1656
1657     my $del_snap =  sub {
1658
1659         check_lock($conf);
1660
1661         if ($conf->{parent} eq $snapname) {
1662             if ($conf->{snapshots}->{$snapname}->{snapname}) {
1663                 $conf->{parent} = $conf->{snapshots}->{$snapname}->{parent};
1664             } else {
1665                 delete $conf->{parent};
1666             }
1667         }
1668
1669         delete $conf->{snapshots}->{$snapname};
1670
1671         PVE::LXC::write_config($vmid, $conf);
1672     };
1673
1674     my $rootfs = $conf->{snapshots}->{$snapname}->{rootfs};
1675     my $rootinfo = PVE::LXC::parse_ct_mountpoint($rootfs);
1676     my $volid = $rootinfo->{volume};
1677
1678     eval {
1679         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname);
1680     };
1681     my $err = $@;
1682
1683     if(!$err || ($err && $force)) {
1684         lock_container($vmid, 10, $del_snap);
1685         if ($err) {
1686             die "Can't delete snapshot: $vmid $snapname $err\n";
1687         }
1688     }
1689 }
1690
1691 sub snapshot_rollback {
1692     my ($vmid, $snapname) = @_;
1693
1694     my $storecfg = PVE::Storage::config();
1695
1696     my $conf = load_config($vmid);
1697
1698     die "you can't rollback if vm is a template\n" if is_template($conf);
1699
1700     my $snap = $conf->{snapshots}->{$snapname};
1701
1702     die "snapshot '$snapname' does not exist\n" if !defined($snap);
1703
1704     my $rootfs = $snap->{rootfs};
1705     my $rootinfo = PVE::LXC::parse_ct_mountpoint($rootfs);
1706     my $volid = $rootinfo->{volume};
1707
1708     PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
1709
1710     my $updatefn = sub {
1711
1712         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
1713             if $snap->{snapstate};
1714
1715         check_lock($conf);
1716
1717         system("lxc-stop -n $vmid --kill") if check_running($vmid);
1718
1719         die "unable to rollback vm $vmid: vm is running\n"
1720             if check_running($vmid);
1721
1722         $conf->{lock} = 'rollback';
1723
1724         my $forcemachine;
1725
1726         # copy snapshot config to current config
1727
1728         my $tmp_conf = $conf;
1729         &$snapshot_copy_config($tmp_conf->{snapshots}->{$snapname}, $conf);
1730         $conf->{snapshots} = $tmp_conf->{snapshots};
1731         delete $conf->{snaptime};
1732         delete $conf->{snapname};
1733         $conf->{parent} = $snapname;
1734
1735         PVE::LXC::write_config($vmid, $conf);
1736     };
1737
1738     my $unlockfn = sub {
1739         delete $conf->{lock};
1740         PVE::LXC::write_config($vmid, $conf);
1741     };
1742
1743     lock_container($vmid, 10, $updatefn);
1744
1745     PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
1746
1747     lock_container($vmid, 5, $unlockfn);
1748 }
1749
1750 sub template_create {
1751     my ($vmid, $conf) = @_;
1752
1753     my $storecfg = PVE::Storage::config();
1754
1755     my $rootinfo = PVE::LXC::parse_ct_mountpoint($conf->{rootfs});
1756     my $volid = $rootinfo->{volume};
1757
1758     die "Template feature is not available for '$volid'\n"
1759         if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
1760
1761     PVE::Storage::activate_volumes($storecfg, [$volid]);
1762
1763     my $template_volid = PVE::Storage::vdisk_create_base($storecfg, $volid);
1764     $rootinfo->{volume} = $template_volid;
1765     $conf->{rootfs} = print_ct_mountpoint($rootinfo);
1766
1767     write_config($vmid, $conf);
1768 }
1769
1770 sub is_template {
1771     my ($conf) = @_;
1772
1773     return 1 if defined $conf->{template} && $conf->{template} == 1;
1774 }
1775
1776 sub foreach_mountpoint {
1777     my ($conf, $func) = @_;
1778
1779     my $mountpoint = parse_ct_mountpoint($conf->{rootfs});
1780     $mountpoint->{mp} = '/'; # just to be sure
1781     &$func('rootfs', $mountpoint);
1782
1783     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1784         my $key = "mp$i";
1785         next if !defined($conf->{$key});
1786         $mountpoint = parse_ct_mountpoint($conf->{$key});
1787         &$func($key, $mountpoint);
1788     }
1789 }
1790
1791 sub loopdevices_list {
1792
1793     my $loopdev = {};
1794     my $parser = sub {
1795         my $line = shift;
1796         if ($line =~ m/^(\/dev\/loop\d+)\s+\d\s+\d\s+\d\s+\d\s(\S+)$/) {
1797             $loopdev->{$1} = $2;
1798         }
1799     };
1800
1801     PVE::Tools::run_command(['losetup'], outfunc => $parser);
1802
1803     return $loopdev;
1804 }
1805
1806 sub blockdevices_list {
1807
1808     my $bdevs = {};
1809     dir_glob_foreach("/sys/dev/block/", '(\d+):(\d+)', sub {
1810         my (undef, $major, $minor) = @_;
1811         my $bdev = readlink("/sys/dev/block/$major:$minor");
1812         $bdev =~ s/\.\.\/\.\.\/devices\/virtual\/block\//\/dev\//;
1813         $bdevs->{$bdev}->{major} = $major;
1814         $bdevs->{$bdev}->{minor} = $minor;
1815     });
1816     return $bdevs;
1817 }
1818
1819 sub find_loopdev {
1820     my ($loopdevs, $path) = @_;
1821
1822     foreach my $dev (keys %$loopdevs){
1823         return $dev if $loopdevs->{$dev} eq $path;
1824     }
1825 }
1826 1;