src/PVE/LXC/Config.pm

   1 package PVE::LXC::Config;
   2
   3 use strict;
   4 use warnings;
   5 use Fcntl qw(O_RDONLY);
   6
   7 use PVE::AbstractConfig;
   8 use PVE::Cluster qw(cfs_register_file);
   9 use PVE::DataCenterConfig;
  10 use PVE::GuestHelpers;
  11 use PVE::INotify;
  12 use PVE::JSONSchema qw(get_standard_option);
  13 use PVE::Tools;
  14
  15 use base qw(PVE::AbstractConfig);
  16
  17 use constant {FIFREEZE => 0xc0045877,
  18               FITHAW   => 0xc0045878};
  19
  20 my $nodename = PVE::INotify::nodename();
  21 my $lock_handles =  {};
  22 my $lockdir = "/run/lock/lxc";
  23 mkdir $lockdir;
  24 mkdir "/etc/pve/nodes/$nodename/lxc";
  25 my $MAX_MOUNT_POINTS = 256;
  26 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
  27
  28 # BEGIN implemented abstract methods from PVE::AbstractConfig
  29
  30 sub guest_type {
  31     return "CT";
  32 }
  33
  34 sub __config_max_unused_disks {
  35     my ($class) = @_;
  36
  37     return $MAX_UNUSED_DISKS;
  38 }
  39
  40 sub config_file_lock {
  41     my ($class, $vmid) = @_;
  42
  43     return "$lockdir/pve-config-${vmid}.lock";
  44 }
  45
  46 sub cfs_config_path {
  47     my ($class, $vmid, $node) = @_;
  48
  49     $node = $nodename if !$node;
  50     return "nodes/$node/lxc/$vmid.conf";
  51 }
  52
  53 sub mountpoint_backup_enabled {
  54     my ($class, $mp_key, $mountpoint) = @_;
  55
  56     my $enabled;
  57     my $reason;
  58
  59     if ($mp_key eq 'rootfs') {
  60         $enabled = 1;
  61         $reason = 'rootfs';
  62     } elsif ($mountpoint->{type} ne 'volume') {
  63         $enabled = 0;
  64         $reason = 'not a volume';
  65     } elsif ($mountpoint->{backup}) {
  66         $enabled = 1;
  67         $reason = 'enabled';
  68     } else {
  69         $enabled = 0;
  70         $reason = 'disabled';
  71     }
  72     return wantarray ? ($enabled, $reason) : $enabled;
  73 }
  74
  75 sub has_feature {
  76     my ($class, $feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
  77     my $err;
  78
  79     my $opts;
  80     if ($feature eq 'copy' || $feature eq 'clone') {
  81         $opts = {'valid_target_formats' => ['raw', 'subvol']};
  82     }
  83
  84     $class->foreach_volume($conf, sub {
  85         my ($ms, $mountpoint) = @_;
  86
  87         return if $err; # skip further test
  88         return if $backup_only && !$class->mountpoint_backup_enabled($ms, $mountpoint);
  89
  90         $err = 1
  91             if !PVE::Storage::volume_has_feature($storecfg, $feature,
  92                                                  $mountpoint->{volume},
  93                                                  $snapname, $running, $opts);
  94     });
  95
  96     return $err ? 0 : 1;
  97 }
  98
  99 sub __snapshot_save_vmstate {
 100     my ($class, $vmid, $conf, $snapname, $storecfg) = @_;
 101     die "implement me - snapshot_save_vmstate\n";
 102 }
 103
 104 sub __snapshot_activate_storages {
 105     my ($class, $conf, $include_vmstate) = @_;
 106
 107     my $storecfg = PVE::Storage::config();
 108     my $opts = $include_vmstate ? { 'extra_keys' => ['vmstate'] } : {};
 109     my $storage_hash = {};
 110
 111     $class->foreach_volume_full($conf, $opts, sub {
 112         my ($vs, $mountpoint) = @_;
 113
 114         return if $mountpoint->{type} ne 'volume';
 115
 116         my ($storeid) = PVE::Storage::parse_volume_id($mountpoint->{volume});
 117         $storage_hash->{$storeid} = 1;
 118     });
 119
 120     PVE::Storage::activate_storage_list($storecfg, [ sort keys $storage_hash->%* ]);
 121 }
 122
 123 sub __snapshot_check_running {
 124     my ($class, $vmid) = @_;
 125     return PVE::LXC::check_running($vmid);
 126 }
 127
 128 sub __snapshot_check_freeze_needed {
 129     my ($class, $vmid, $config, $save_vmstate) = @_;
 130
 131     my $ret = $class->__snapshot_check_running($vmid);
 132     return ($ret, $ret);
 133 }
 134
 135 # implements similar functionality to fsfreeze(8)
 136 sub fsfreeze_mountpoint {
 137     my ($path, $thaw) = @_;
 138
 139     my $op = $thaw ? 'thaw' : 'freeze';
 140     my $ioctl = $thaw ? FITHAW : FIFREEZE;
 141
 142     sysopen my $fd, $path, O_RDONLY or die "failed to open $path: $!\n";
 143     my $ioctl_err;
 144     if (!ioctl($fd, $ioctl, 0)) {
 145         $ioctl_err = "$!";
 146     }
 147     close($fd);
 148     die "fs$op '$path' failed - $ioctl_err\n" if defined $ioctl_err;
 149 }
 150
 151 sub __snapshot_freeze {
 152     my ($class, $vmid, $unfreeze) = @_;
 153
 154     my $conf = $class->load_config($vmid);
 155     my $storagecfg = PVE::Storage::config();
 156
 157     my $freeze_mps = [];
 158     $class->foreach_volume($conf, sub {
 159         my ($ms, $mountpoint) = @_;
 160
 161         return if $mountpoint->{type} ne 'volume';
 162
 163         if (PVE::Storage::volume_snapshot_needs_fsfreeze($storagecfg, $mountpoint->{volume})) {
 164             push @$freeze_mps, $mountpoint->{mp};
 165         }
 166     });
 167
 168     my $freeze_mountpoints = sub {
 169         my ($thaw) = @_;
 170
 171         return if scalar(@$freeze_mps) == 0;
 172
 173         my $pid = PVE::LXC::find_lxc_pid($vmid);
 174
 175         for my $mp (@$freeze_mps) {
 176             eval{ fsfreeze_mountpoint("/proc/${pid}/root/${mp}", $thaw); };
 177             warn $@ if $@;
 178         }
 179     };
 180
 181     if ($unfreeze) {
 182         eval { PVE::LXC::thaw($vmid); };
 183         warn $@ if $@;
 184         $freeze_mountpoints->(1);
 185     } else {
 186         PVE::LXC::freeze($vmid);
 187         PVE::LXC::sync_container_namespace($vmid);
 188         $freeze_mountpoints->(0);
 189     }
 190 }
 191
 192 sub __snapshot_create_vol_snapshot {
 193     my ($class, $vmid, $ms, $mountpoint, $snapname) = @_;
 194
 195     my $storecfg = PVE::Storage::config();
 196
 197     return if $snapname eq 'vzdump' &&
 198         !$class->mountpoint_backup_enabled($ms, $mountpoint);
 199
 200     PVE::Storage::volume_snapshot($storecfg, $mountpoint->{volume}, $snapname);
 201 }
 202
 203 sub __snapshot_delete_remove_drive {
 204     my ($class, $snap, $remove_drive) = @_;
 205
 206     if ($remove_drive eq 'vmstate') {
 207         die "implement me - saving vmstate\n";
 208     } else {
 209         my $value = $snap->{$remove_drive};
 210         my $mountpoint = $class->parse_volume($remove_drive, $value, 1);
 211         delete $snap->{$remove_drive};
 212
 213         $class->add_unused_volume($snap, $mountpoint->{volume})
 214             if $mountpoint && ($mountpoint->{type} eq 'volume');
 215     }
 216 }
 217
 218 sub __snapshot_delete_vmstate_file {
 219     my ($class, $snap, $force) = @_;
 220
 221     die "implement me - saving vmstate\n";
 222 }
 223
 224 sub __snapshot_delete_vol_snapshot {
 225     my ($class, $vmid, $ms, $mountpoint, $snapname, $unused) = @_;
 226
 227     return if $snapname eq 'vzdump' &&
 228         !$class->mountpoint_backup_enabled($ms, $mountpoint);
 229
 230     my $storecfg = PVE::Storage::config();
 231     PVE::Storage::volume_snapshot_delete($storecfg, $mountpoint->{volume}, $snapname);
 232     push @$unused, $mountpoint->{volume};
 233 }
 234
 235 sub __snapshot_rollback_vol_possible {
 236     my ($class, $mountpoint, $snapname, $blockers) = @_;
 237
 238     my $storecfg = PVE::Storage::config();
 239     PVE::Storage::volume_rollback_is_possible(
 240         $storecfg,
 241         $mountpoint->{volume},
 242         $snapname,
 243         $blockers,
 244     );
 245 }
 246
 247 sub __snapshot_rollback_vol_rollback {
 248     my ($class, $mountpoint, $snapname) = @_;
 249
 250     my $storecfg = PVE::Storage::config();
 251     PVE::Storage::volume_snapshot_rollback($storecfg, $mountpoint->{volume}, $snapname);
 252 }
 253
 254 sub __snapshot_rollback_vm_stop {
 255     my ($class, $vmid) = @_;
 256
 257     PVE::LXC::vm_stop($vmid, 1)
 258         if $class->__snapshot_check_running($vmid);
 259 }
 260
 261 sub __snapshot_rollback_vm_start {
 262     my ($class, $vmid, $vmstate, $data);
 263
 264     die "implement me - save vmstate\n";
 265 }
 266
 267 sub __snapshot_rollback_get_unused {
 268     my ($class, $conf, $snap) = @_;
 269
 270     my $unused = [];
 271
 272     $class->foreach_volume($conf, sub {
 273         my ($vs, $volume) = @_;
 274
 275         return if $volume->{type} ne 'volume';
 276
 277         my $found = 0;
 278         my $volid = $volume->{volume};
 279
 280         $class->foreach_volume($snap, sub {
 281             my ($ms, $mountpoint) = @_;
 282
 283             return if $found;
 284             return if ($mountpoint->{type} ne 'volume');
 285
 286             $found = 1
 287                 if ($mountpoint->{volume} && $mountpoint->{volume} eq $volid);
 288         });
 289
 290         push @$unused, $volid if !$found;
 291     });
 292
 293     return $unused;
 294 }
 295
 296 # END implemented abstract methods from PVE::AbstractConfig
 297
 298 # BEGIN JSON config code
 299
 300 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
 301
 302
 303 my $valid_mount_option_re = qr/(noatime|lazytime|nodev|nosuid|noexec)/;
 304
 305 sub is_valid_mount_option {
 306     my ($option) = @_;
 307     return $option =~ $valid_mount_option_re;
 308 }
 309
 310 my $rootfs_desc = {
 311     volume => {
 312         type => 'string',
 313         default_key => 1,
 314         format => 'pve-lxc-mp-string',
 315         format_description => 'volume',
 316         description => 'Volume, device or directory to mount into the container.',
 317     },
 318     size => {
 319         type => 'string',
 320         format => 'disk-size',
 321         format_description => 'DiskSize',
 322         description => 'Volume size (read only value).',
 323         optional => 1,
 324     },
 325     acl => {
 326         type => 'boolean',
 327         description => 'Explicitly enable or disable ACL support.',
 328         optional => 1,
 329     },
 330     mountoptions => {
 331         optional => 1,
 332         type => 'string',
 333         description => 'Extra mount options for rootfs/mps.',
 334         format_description => 'opt[;opt...]',
 335         pattern => qr/$valid_mount_option_re(;$valid_mount_option_re)*/,
 336     },
 337     ro => {
 338         type => 'boolean',
 339         description => 'Read-only mount point',
 340         optional => 1,
 341     },
 342     quota => {
 343         type => 'boolean',
 344         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
 345         optional => 1,
 346     },
 347     replicate => {
 348         type => 'boolean',
 349         description => 'Will include this volume to a storage replica job.',
 350         optional => 1,
 351         default => 1,
 352     },
 353     shared => {
 354         type => 'boolean',
 355         description => 'Mark this non-volume mount point as available on multiple nodes (see \'nodes\')',
 356         verbose_description => "Mark this non-volume mount point as available on all nodes.\n\nWARNING: This option does not share the mount point automatically, it assumes it is shared already!",
 357         optional => 1,
 358         default => 0,
 359     },
 360 };
 361
 362 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
 363     type => 'string', format => $rootfs_desc,
 364     description => "Use volume as container root.",
 365     optional => 1,
 366 });
 367
 368 # IP address with optional interface suffix for link local ipv6 addresses
 369 PVE::JSONSchema::register_format('lxc-ip-with-ll-iface', \&verify_ip_with_ll_iface);
 370 sub verify_ip_with_ll_iface {
 371     my ($addr, $noerr) = @_;
 372
 373     if (my ($addr, $iface) = ($addr =~ /^(fe80:[^%]+)%(.*)$/)) {
 374         if (PVE::JSONSchema::pve_verify_ip($addr, 1)
 375             && PVE::JSONSchema::pve_verify_iface($iface, 1))
 376         {
 377             return $addr;
 378         }
 379     }
 380
 381     return PVE::JSONSchema::pve_verify_ip($addr, $noerr);
 382 }
 383
 384
 385 my $features_desc = {
 386     mount => {
 387         optional => 1,
 388         type => 'string',
 389         description => "Allow mounting file systems of specific types."
 390             ." This should be a list of file system types as used with the mount command."
 391             ." Note that this can have negative effects on the container's security."
 392             ." With access to a loop device, mounting a file can circumvent the mknod"
 393             ." permission of the devices cgroup, mounting an NFS file system can"
 394             ." block the host's I/O completely and prevent it from rebooting, etc.",
 395         format_description => 'fstype;fstype;...',
 396         pattern => qr/[a-zA-Z0-9_; ]+/,
 397     },
 398     nesting => {
 399         optional => 1,
 400         type => 'boolean',
 401         default => 0,
 402         description => "Allow nesting."
 403             ." Best used with unprivileged containers with additional id mapping."
 404             ." Note that this will expose procfs and sysfs contents of the host"
 405             ." to the guest.",
 406     },
 407     keyctl => {
 408         optional => 1,
 409         type => 'boolean',
 410         default => 0,
 411         description => "For unprivileged containers only: Allow the use of the keyctl() system call."
 412             ." This is required to use docker inside a container."
 413             ." By default unprivileged containers will see this system call as non-existent."
 414             ." This is mostly a workaround for systemd-networkd, as it will treat it as a fatal"
 415             ." error when some keyctl() operations are denied by the kernel due to lacking permissions."
 416             ." Essentially, you can choose between running systemd-networkd or docker.",
 417     },
 418     fuse => {
 419         optional => 1,
 420         type => 'boolean',
 421         default => 0,
 422         description => "Allow using 'fuse' file systems in a container."
 423             ." Note that interactions between fuse and the freezer cgroup can potentially cause I/O deadlocks.",
 424     },
 425     mknod => {
 426         optional => 1,
 427         type => 'boolean',
 428         default => 0,
 429         description => "Allow unprivileged containers to use mknod() to add certain device nodes."
 430             ." This requires a kernel with seccomp trap to user space support (5.3 or newer)."
 431             ." This is experimental.",
 432     },
 433     force_rw_sys => {
 434         optional => 1,
 435         type => 'boolean',
 436         default => 0,
 437         description => "Mount /sys in unprivileged containers as `rw` instead of `mixed`."
 438             ." This can break networking under newer (>= v245) systemd-network use."
 439     },
 440 };
 441
 442 my $confdesc = {
 443     lock => {
 444         optional => 1,
 445         type => 'string',
 446         description => "Lock/unlock the VM.",
 447         enum => [qw(backup create destroyed disk fstrim migrate mounted rollback snapshot snapshot-delete)],
 448     },
 449     onboot => {
 450         optional => 1,
 451         type => 'boolean',
 452         description => "Specifies whether a VM will be started during system bootup.",
 453         default => 0,
 454     },
 455     startup => get_standard_option('pve-startup-order'),
 456     template => {
 457         optional => 1,
 458         type => 'boolean',
 459         description => "Enable/disable Template.",
 460         default => 0,
 461     },
 462     arch => {
 463         optional => 1,
 464         type => 'string',
 465         enum => ['amd64', 'i386', 'arm64', 'armhf'],
 466         description => "OS architecture type.",
 467         default => 'amd64',
 468     },
 469     ostype => {
 470         optional => 1,
 471         type => 'string',
 472         enum => [qw(debian devuan ubuntu centos fedora opensuse archlinux alpine gentoo nixos unmanaged)],
 473         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 474     },
 475     console => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Attach a console device (/dev/console) to the container.",
 479         default => 1,
 480     },
 481     tty => {
 482         optional => 1,
 483         type => 'integer',
 484         description => "Specify the number of tty available to the container",
 485         minimum => 0,
 486         maximum => 6,
 487         default => 2,
 488     },
 489     cores => {
 490         optional => 1,
 491         type => 'integer',
 492         description => "The number of cores assigned to the container. A container can use all available cores by default.",
 493         minimum => 1,
 494         maximum => 8192,
 495     },
 496     cpulimit => {
 497         optional => 1,
 498         type => 'number',
 499         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 500         minimum => 0,
 501         maximum => 8192,
 502         default => 0,
 503     },
 504     cpuunits => {
 505         optional => 1,
 506         type => 'integer',
 507         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 508         minimum => 0,
 509         maximum => 500000,
 510         default => 1024,
 511     },
 512     memory => {
 513         optional => 1,
 514         type => 'integer',
 515         description => "Amount of RAM for the VM in MB.",
 516         minimum => 16,
 517         default => 512,
 518     },
 519     swap => {
 520         optional => 1,
 521         type => 'integer',
 522         description => "Amount of SWAP for the VM in MB.",
 523         minimum => 0,
 524         default => 512,
 525     },
 526     hostname => {
 527         optional => 1,
 528         description => "Set a host name for the container.",
 529         type => 'string', format => 'dns-name',
 530         maxLength => 255,
 531     },
 532     description => {
 533         optional => 1,
 534         type => 'string',
 535         description => "Description for the Container. Shown in the web-interface CT's summary."
 536             ." This is saved as comment inside the configuration file.",
 537         maxLength => 1024 * 8,
 538     },
 539     searchdomain => {
 540         optional => 1,
 541         type => 'string', format => 'dns-name-list',
 542         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 543     },
 544     nameserver => {
 545         optional => 1,
 546         type => 'string', format => 'lxc-ip-with-ll-iface-list',
 547         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 548     },
 549     timezone => {
 550         optional => 1,
 551         type => 'string', format => 'pve-ct-timezone',
 552         description => "Time zone to use in the container. If option isn't set, then nothing will be done. Can be set to 'host' to match the host time zone, or an arbitrary time zone option from /usr/share/zoneinfo/zone.tab",
 553     },
 554     rootfs => get_standard_option('pve-ct-rootfs'),
 555     parent => {
 556         optional => 1,
 557         type => 'string', format => 'pve-configid',
 558         maxLength => 40,
 559         description => "Parent snapshot name. This is used internally, and should not be modified.",
 560     },
 561     snaptime => {
 562         optional => 1,
 563         description => "Timestamp for snapshots.",
 564         type => 'integer',
 565         minimum => 0,
 566     },
 567     cmode => {
 568         optional => 1,
 569         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 570         type => 'string',
 571         enum => ['shell', 'console', 'tty'],
 572         default => 'tty',
 573     },
 574     protection => {
 575         optional => 1,
 576         type => 'boolean',
 577         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 578         default => 0,
 579     },
 580     unprivileged => {
 581         optional => 1,
 582         type => 'boolean',
 583         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 584         default => 0,
 585     },
 586     features => {
 587         optional => 1,
 588         type => 'string',
 589         format => $features_desc,
 590         description => "Allow containers access to advanced features.",
 591     },
 592     hookscript => {
 593         optional => 1,
 594         type => 'string',
 595         format => 'pve-volume-id',
 596         description => 'Script that will be exectued during various steps in the containers lifetime.',
 597     },
 598     tags => {
 599         type => 'string', format => 'pve-tag-list',
 600         description => 'Tags of the Container. This is only meta information.',
 601         optional => 1,
 602     },
 603     debug => {
 604         optional => 1,
 605         type => 'boolean',
 606         description => "Try to be more verbose. For now this only enables debug log-level on start.",
 607         default => 0,
 608     },
 609 };
 610
 611 my $valid_lxc_conf_keys = {
 612     'lxc.apparmor.profile' => 1,
 613     'lxc.apparmor.allow_incomplete' => 1,
 614     'lxc.apparmor.allow_nesting' => 1,
 615     'lxc.apparmor.raw' => 1,
 616     'lxc.selinux.context' => 1,
 617     'lxc.include' => 1,
 618     'lxc.arch' => 1,
 619     'lxc.uts.name' => 1,
 620     'lxc.signal.halt' => 1,
 621     'lxc.signal.reboot' => 1,
 622     'lxc.signal.stop' => 1,
 623     'lxc.init.cmd' => 1,
 624     'lxc.pty.max' => 1,
 625     'lxc.console.logfile' => 1,
 626     'lxc.console.path' => 1,
 627     'lxc.tty.max' => 1,
 628     'lxc.devtty.dir' => 1,
 629     'lxc.hook.autodev' => 1,
 630     'lxc.autodev' => 1,
 631     'lxc.kmsg' => 1,
 632     'lxc.mount.fstab' => 1,
 633     'lxc.mount.entry' => 1,
 634     'lxc.mount.auto' => 1,
 635     'lxc.rootfs.path' => 'lxc.rootfs.path is auto generated from rootfs',
 636     'lxc.rootfs.mount' => 1,
 637     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 638                             ', please use mount point options in the "rootfs" key',
 639     # lxc.cgroup.*
 640     # lxc.prlimit.*
 641     # lxc.net.*
 642     'lxc.cap.drop' => 1,
 643     'lxc.cap.keep' => 1,
 644     'lxc.seccomp.profile' => 1,
 645     'lxc.seccomp.notify.proxy' => 1,
 646     'lxc.seccomp.notify.cookie' => 1,
 647     'lxc.idmap' => 1,
 648     'lxc.hook.pre-start' => 1,
 649     'lxc.hook.pre-mount' => 1,
 650     'lxc.hook.mount' => 1,
 651     'lxc.hook.start' => 1,
 652     'lxc.hook.stop' => 1,
 653     'lxc.hook.post-stop' => 1,
 654     'lxc.hook.clone' => 1,
 655     'lxc.hook.destroy' => 1,
 656     'lxc.hook.version' => 1,
 657     'lxc.log.level' => 1,
 658     'lxc.log.file' => 1,
 659     'lxc.start.auto' => 1,
 660     'lxc.start.delay' => 1,
 661     'lxc.start.order' => 1,
 662     'lxc.group' => 1,
 663     'lxc.environment' => 1,
 664
 665     # All these are namespaced via CLONE_NEWIPC (see namespaces(7)).
 666     'lxc.sysctl.fs.mqueue' => 1,
 667     'lxc.sysctl.kernel.msgmax' => 1,
 668     'lxc.sysctl.kernel.msgmnb' => 1,
 669     'lxc.sysctl.kernel.msgmni' => 1,
 670     'lxc.sysctl.kernel.sem' => 1,
 671     'lxc.sysctl.kernel.shmall' => 1,
 672     'lxc.sysctl.kernel.shmmax' => 1,
 673     'lxc.sysctl.kernel.shmmni' => 1,
 674     'lxc.sysctl.kernel.shm_rmid_forced' => 1,
 675 };
 676
 677 my $deprecated_lxc_conf_keys = {
 678     # Deprecated (removed with lxc 3.0):
 679     'lxc.aa_profile'           => 'lxc.apparmor.profile',
 680     'lxc.aa_allow_incomplete'  => 'lxc.apparmor.allow_incomplete',
 681     'lxc.console'              => 'lxc.console.path',
 682     'lxc.devttydir'            => 'lxc.tty.dir',
 683     'lxc.haltsignal'           => 'lxc.signal.halt',
 684     'lxc.rebootsignal'         => 'lxc.signal.reboot',
 685     'lxc.stopsignal'           => 'lxc.signal.stop',
 686     'lxc.id_map'               => 'lxc.idmap',
 687     'lxc.init_cmd'             => 'lxc.init.cmd',
 688     'lxc.loglevel'             => 'lxc.log.level',
 689     'lxc.logfile'              => 'lxc.log.file',
 690     'lxc.mount'                => 'lxc.mount.fstab',
 691     'lxc.network.type'         => 'lxc.net.INDEX.type',
 692     'lxc.network.flags'        => 'lxc.net.INDEX.flags',
 693     'lxc.network.link'         => 'lxc.net.INDEX.link',
 694     'lxc.network.mtu'          => 'lxc.net.INDEX.mtu',
 695     'lxc.network.name'         => 'lxc.net.INDEX.name',
 696     'lxc.network.hwaddr'       => 'lxc.net.INDEX.hwaddr',
 697     'lxc.network.ipv4'         => 'lxc.net.INDEX.ipv4.address',
 698     'lxc.network.ipv4.gateway' => 'lxc.net.INDEX.ipv4.gateway',
 699     'lxc.network.ipv6'         => 'lxc.net.INDEX.ipv6.address',
 700     'lxc.network.ipv6.gateway' => 'lxc.net.INDEX.ipv6.gateway',
 701     'lxc.network.script.up'    => 'lxc.net.INDEX.script.up',
 702     'lxc.network.script.down'  => 'lxc.net.INDEX.script.down',
 703     'lxc.pts'                  => 'lxc.pty.max',
 704     'lxc.se_context'           => 'lxc.selinux.context',
 705     'lxc.seccomp'              => 'lxc.seccomp.profile',
 706     'lxc.tty'                  => 'lxc.tty.max',
 707     'lxc.utsname'              => 'lxc.uts.name',
 708 };
 709
 710 sub is_valid_lxc_conf_key {
 711     my ($vmid, $key) = @_;
 712     if ($key =~ /^lxc\.limit\./) {
 713         warn "vm $vmid - $key: lxc.limit.* was renamed to lxc.prlimit.*\n";
 714         return 1;
 715     }
 716     if (defined(my $new_name = $deprecated_lxc_conf_keys->{$key})) {
 717         warn "vm $vmid - $key is deprecated and was renamed to $new_name\n";
 718         return 1;
 719     }
 720     my $validity = $valid_lxc_conf_keys->{$key};
 721     return $validity if defined($validity);
 722     return 1 if $key =~ /^lxc\.cgroup2?\./  # allow all cgroup values
 723              || $key =~ /^lxc\.prlimit\./ # allow all prlimits
 724              || $key =~ /^lxc\.net\./;    # allow custom network definitions
 725     return 0;
 726 }
 727
 728 our $netconf_desc = {
 729     type => {
 730         type => 'string',
 731         optional => 1,
 732         description => "Network interface type.",
 733         enum => [qw(veth)],
 734     },
 735     name => {
 736         type => 'string',
 737         format_description => 'string',
 738         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 739         pattern => '[-_.\w\d]+',
 740     },
 741     bridge => {
 742         type => 'string',
 743         format_description => 'bridge',
 744         description => 'Bridge to attach the network device to.',
 745         pattern => '[-_.\w\d]+',
 746         optional => 1,
 747     },
 748     hwaddr => get_standard_option('mac-addr', {
 749         description => 'The interface MAC address. This is dynamically allocated by default, but you can set that statically if needed, for example to always have the same link-local IPv6 address. (lxc.network.hwaddr)',
 750         }),
 751     mtu => {
 752         type => 'integer',
 753         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 754         minimum => 64, # minimum ethernet frame is 64 bytes
 755         optional => 1,
 756     },
 757     ip => {
 758         type => 'string',
 759         format => 'pve-ipv4-config',
 760         format_description => '(IPv4/CIDR|dhcp|manual)',
 761         description => 'IPv4 address in CIDR format.',
 762         optional => 1,
 763     },
 764     gw => {
 765         type => 'string',
 766         format => 'ipv4',
 767         format_description => 'GatewayIPv4',
 768         description => 'Default gateway for IPv4 traffic.',
 769         optional => 1,
 770     },
 771     ip6 => {
 772         type => 'string',
 773         format => 'pve-ipv6-config',
 774         format_description => '(IPv6/CIDR|auto|dhcp|manual)',
 775         description => 'IPv6 address in CIDR format.',
 776         optional => 1,
 777     },
 778     gw6 => {
 779         type => 'string',
 780         format => 'ipv6',
 781         format_description => 'GatewayIPv6',
 782         description => 'Default gateway for IPv6 traffic.',
 783         optional => 1,
 784     },
 785     firewall => {
 786         type => 'boolean',
 787         description => "Controls whether this interface's firewall rules should be used.",
 788         optional => 1,
 789     },
 790     tag => {
 791         type => 'integer',
 792         minimum => 1,
 793         maximum => 4094,
 794         description => "VLAN tag for this interface.",
 795         optional => 1,
 796     },
 797     trunks => {
 798         type => 'string',
 799         pattern => qr/\d+(?:;\d+)*/,
 800         format_description => 'vlanid[;vlanid...]',
 801         description => "VLAN ids to pass through the interface",
 802         optional => 1,
 803     },
 804     rate => {
 805         type => 'number',
 806         format_description => 'mbps',
 807         description => "Apply rate limiting to the interface",
 808         optional => 1,
 809     },
 810 };
 811 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 812
 813 my $MAX_LXC_NETWORKS = 32;
 814 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 815     $confdesc->{"net$i"} = {
 816         optional => 1,
 817         type => 'string', format => $netconf_desc,
 818         description => "Specifies network interfaces for the container.",
 819     };
 820 }
 821
 822 PVE::JSONSchema::register_format('pve-ct-timezone', \&verify_ct_timezone);
 823 sub verify_ct_timezone {
 824     my ($timezone, $noerr) = @_;
 825
 826     return if $timezone eq 'host'; # using host settings
 827
 828     PVE::JSONSchema::pve_verify_timezone($timezone);
 829 }
 830
 831 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 832 sub verify_lxc_mp_string {
 833     my ($mp, $noerr) = @_;
 834
 835     # do not allow:
 836     # /./ or /../
 837     # /. or /.. at the end
 838     # ../ at the beginning
 839
 840     if($mp =~ m@/\.\.?/@ ||
 841        $mp =~ m@/\.\.?$@ ||
 842        $mp =~ m@^\.\./@) {
 843         return undef if $noerr;
 844         die "$mp contains illegal character sequences\n";
 845     }
 846     return $mp;
 847 }
 848
 849 my $mp_desc = {
 850     %$rootfs_desc,
 851     backup => {
 852         type => 'boolean',
 853         description => 'Whether to include the mount point in backups.',
 854         verbose_description => 'Whether to include the mount point in backups '.
 855                                '(only used for volume mount points).',
 856         optional => 1,
 857     },
 858     mp => {
 859         type => 'string',
 860         format => 'pve-lxc-mp-string',
 861         format_description => 'Path',
 862         description => 'Path to the mount point as seen from inside the container '.
 863                        '(must not contain symlinks).',
 864         verbose_description => "Path to the mount point as seen from inside the container.\n\n".
 865                                "NOTE: Must not contain any symlinks for security reasons."
 866     },
 867 };
 868 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 869
 870 my $unused_desc = {
 871     volume => {
 872         type => 'string',
 873         default_key => 1,
 874         format => 'pve-volume-id',
 875         format_description => 'volume',
 876         description => 'The volume that is not used currently.',
 877     }
 878 };
 879
 880 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 881     $confdesc->{"mp$i"} = {
 882         optional => 1,
 883         type => 'string', format => $mp_desc,
 884         description => "Use volume as container mount point. Use the special " .
 885             "syntax STORAGE_ID:SIZE_IN_GiB to allocate a new volume.",
 886         optional => 1,
 887     };
 888 }
 889
 890 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
 891     $confdesc->{"unused$i"} = {
 892         optional => 1,
 893         type => 'string', format => $unused_desc,
 894         description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
 895     }
 896 }
 897
 898 sub parse_pct_config {
 899     my ($filename, $raw, $strict) = @_;
 900
 901     return undef if !defined($raw);
 902
 903     my $res = {
 904         digest => Digest::SHA::sha1_hex($raw),
 905         snapshots => {},
 906         pending => {},
 907     };
 908
 909     my $handle_error = sub {
 910         my ($msg) = @_;
 911
 912         if ($strict) {
 913             die $msg;
 914         } else {
 915             warn $msg;
 916         }
 917     };
 918
 919     $filename =~ m|/lxc/(\d+).conf$|
 920         || die "got strange filename '$filename'";
 921
 922     my $vmid = $1;
 923
 924     my $conf = $res;
 925     my $descr = '';
 926     my $section = '';
 927
 928     my @lines = split(/\n/, $raw);
 929     foreach my $line (@lines) {
 930         next if $line =~ m/^\s*$/;
 931
 932         if ($line =~ m/^\[pve:pending\]\s*$/i) {
 933             $section = 'pending';
 934             $conf->{description} = $descr if $descr;
 935             $descr = '';
 936             $conf = $res->{$section} = {};
 937             next;
 938         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 939             $section = $1;
 940             $conf->{description} = $descr if $descr;
 941             $descr = '';
 942             $conf = $res->{snapshots}->{$section} = {};
 943             next;
 944         }
 945
 946         if ($line =~ m/^\#(.*)$/) {
 947             $descr .= PVE::Tools::decode_text($1) . "\n";
 948             next;
 949         }
 950
 951         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 952             my $key = $1;
 953             my $value = $3;
 954             my $validity = is_valid_lxc_conf_key($vmid, $key);
 955             if ($validity eq 1) {
 956                 push @{$conf->{lxc}}, [$key, $value];
 957             } elsif (my $errmsg = $validity) {
 958                 $handle_error->("vm $vmid - $key: $errmsg\n");
 959             } else {
 960                 $handle_error->("vm $vmid - unable to parse config: $line\n");
 961             }
 962         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 963             $descr .= PVE::Tools::decode_text($2);
 964         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 965             $conf->{snapstate} = $1;
 966         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
 967             my $value = $1;
 968             if ($section eq 'pending') {
 969                 $conf->{delete} = $value;
 970             } else {
 971                 $handle_error->("vm $vmid - property 'delete' is only allowed in [pve:pending]\n");
 972             }
 973         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
 974             my $key = $1;
 975             my $value = $2;
 976             eval { $value = PVE::LXC::Config->check_type($key, $value); };
 977             $handle_error->("vm $vmid - unable to parse value of '$key' - $@") if $@;
 978             $conf->{$key} = $value;
 979         } else {
 980             $handle_error->("vm $vmid - unable to parse config: $line\n");
 981         }
 982     }
 983
 984     $conf->{description} = $descr if $descr;
 985
 986     delete $res->{snapstate}; # just to be sure
 987
 988     return $res;
 989 }
 990
 991 sub write_pct_config {
 992     my ($filename, $conf) = @_;
 993
 994     delete $conf->{snapstate}; # just to be sure
 995
 996     my $volidlist = PVE::LXC::Config->get_vm_volumes($conf);
 997     my $used_volids = {};
 998     foreach my $vid (@$volidlist) {
 999         $used_volids->{$vid} = 1;
1000     }
1001
1002     # remove 'unusedX' settings if the volume is still used
1003     foreach my $key (keys %$conf) {
1004         my $value = $conf->{$key};
1005         if ($key =~ m/^unused/ && $used_volids->{$value}) {
1006             delete $conf->{$key};
1007         }
1008     }
1009
1010     my $generate_raw_config = sub {
1011         my ($conf) = @_;
1012
1013         my $raw = '';
1014
1015         # add description as comment to top of file
1016         my $descr = $conf->{description} || '';
1017         foreach my $cl (split(/\n/, $descr)) {
1018             $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
1019         }
1020
1021         foreach my $key (sort keys %$conf) {
1022             next if $key eq 'digest' || $key eq 'description' ||
1023                     $key eq 'pending' || $key eq 'snapshots' ||
1024                     $key eq 'snapname' || $key eq 'lxc';
1025             my $value = $conf->{$key};
1026             die "detected invalid newline inside property '$key'\n"
1027                 if $value =~ m/\n/;
1028             $raw .= "$key: $value\n";
1029         }
1030
1031         if (my $lxcconf = $conf->{lxc}) {
1032             foreach my $entry (@$lxcconf) {
1033                 my ($k, $v) = @$entry;
1034                 $raw .= "$k: $v\n";
1035             }
1036         }
1037
1038         return $raw;
1039     };
1040
1041     my $raw = &$generate_raw_config($conf);
1042
1043     if (scalar(keys %{$conf->{pending}})){
1044         $raw .= "\n[pve:pending]\n";
1045         $raw .= &$generate_raw_config($conf->{pending});
1046     }
1047
1048     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
1049         $raw .= "\n[$snapname]\n";
1050         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1051     }
1052
1053     return $raw;
1054 }
1055
1056 sub update_pct_config {
1057     my ($class, $vmid, $conf, $running, $param, $delete, $revert) = @_;
1058
1059     my $storage_cfg = PVE::Storage::config();
1060
1061     foreach my $opt (@$revert) {
1062         delete $conf->{pending}->{$opt};
1063         $class->remove_from_pending_delete($conf, $opt); # also remove from deletion queue
1064     }
1065
1066     # write updates to pending section
1067     my $modified = {}; # record modified options
1068
1069     foreach my $opt (@$delete) {
1070         if (!defined($conf->{$opt}) && !defined($conf->{pending}->{$opt})) {
1071             warn "cannot delete '$opt' - not set in current configuration!\n";
1072             next;
1073         }
1074         $modified->{$opt} = 1;
1075         if ($opt eq 'memory' || $opt eq 'rootfs' || $opt eq 'ostype') {
1076             die "unable to delete required option '$opt'\n";
1077         } elsif ($opt =~ m/^unused(\d+)$/) {
1078             $class->check_protection($conf, "can't remove CT $vmid drive '$opt'");
1079         } elsif ($opt =~ m/^mp(\d+)$/) {
1080             $class->check_protection($conf, "can't remove CT $vmid drive '$opt'");
1081         } elsif ($opt eq 'unprivileged') {
1082             die "unable to delete read-only option: '$opt'\n";
1083         }
1084         $class->add_to_pending_delete($conf, $opt);
1085     }
1086
1087     my $check_content_type = sub {
1088         my ($mp) = @_;
1089         my $sid = PVE::Storage::parse_volume_id($mp->{volume});
1090         my $storage_config = PVE::Storage::storage_config($storage_cfg, $sid);
1091         die "storage '$sid' does not allow content type 'rootdir' (Container)\n"
1092             if !$storage_config->{content}->{rootdir};
1093     };
1094
1095     foreach my $opt (sort keys %$param) { # add/change
1096         $modified->{$opt} = 1;
1097         my $value = $param->{$opt};
1098         if ($opt =~ m/^mp(\d+)$/ || $opt eq 'rootfs') {
1099             $class->check_protection($conf, "can't update CT $vmid drive '$opt'");
1100             my $mp = $class->parse_volume($opt, $value);
1101             $check_content_type->($mp) if ($mp->{type} eq 'volume');
1102         } elsif ($opt eq 'hookscript') {
1103             PVE::GuestHelpers::check_hookscript($value);
1104         } elsif ($opt eq 'nameserver') {
1105             $value = PVE::LXC::verify_nameserver_list($value);
1106         } elsif ($opt eq 'searchdomain') {
1107             $value = PVE::LXC::verify_searchdomain_list($value);
1108         } elsif ($opt eq 'unprivileged') {
1109             die "unable to modify read-only option: '$opt'\n";
1110         }
1111         $conf->{pending}->{$opt} = $value;
1112         $class->remove_from_pending_delete($conf, $opt);
1113     }
1114
1115     my $changes = $class->cleanup_pending($conf);
1116
1117     my $errors = {};
1118     if ($running) {
1119         $class->vmconfig_hotplug_pending($vmid, $conf, $storage_cfg, $modified, $errors);
1120     } else {
1121         $class->vmconfig_apply_pending($vmid, $conf, $storage_cfg, $modified, $errors);
1122     }
1123
1124     return $errors;
1125 }
1126
1127 sub check_type {
1128     my ($class, $key, $value) = @_;
1129
1130     die "unknown setting '$key'\n" if !$confdesc->{$key};
1131
1132     my $type = $confdesc->{$key}->{type};
1133
1134     if (!defined($value)) {
1135         die "got undefined value\n";
1136     }
1137
1138     if ($value =~ m/[\n\r]/) {
1139         die "property contains a line feed\n";
1140     }
1141
1142     if ($type eq 'boolean') {
1143         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1144         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1145         die "type check ('boolean') failed - got '$value'\n";
1146     } elsif ($type eq 'integer') {
1147         return int($1) if $value =~ m/^(\d+)$/;
1148         die "type check ('integer') failed - got '$value'\n";
1149     } elsif ($type eq 'number') {
1150         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1151         die "type check ('number') failed - got '$value'\n";
1152     } elsif ($type eq 'string') {
1153         if (my $fmt = $confdesc->{$key}->{format}) {
1154             PVE::JSONSchema::check_format($fmt, $value);
1155             return $value;
1156         }
1157         return $value;
1158     } else {
1159         die "internal error"
1160     }
1161 }
1162
1163
1164 # add JSON properties for create and set function
1165 sub json_config_properties {
1166     my ($class, $prop) = @_;
1167
1168     foreach my $opt (keys %$confdesc) {
1169         next if $opt eq 'parent' || $opt eq 'snaptime';
1170         next if $prop->{$opt};
1171         $prop->{$opt} = $confdesc->{$opt};
1172     }
1173
1174     return $prop;
1175 }
1176
1177 my $parse_ct_mountpoint_full = sub {
1178     my ($class, $desc, $data, $noerr) = @_;
1179
1180     $data //= '';
1181
1182     my $res;
1183     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
1184     if ($@) {
1185         return undef if $noerr;
1186         die $@;
1187     }
1188
1189     if (defined(my $size = $res->{size})) {
1190         $size = PVE::JSONSchema::parse_size($size);
1191         if (!defined($size)) {
1192             return undef if $noerr;
1193             die "invalid size: $size\n";
1194         }
1195         $res->{size} = $size;
1196     }
1197
1198     $res->{type} = $class->classify_mountpoint($res->{volume});
1199
1200     return $res;
1201 };
1202
1203 sub print_ct_mountpoint {
1204     my ($class, $info, $nomp) = @_;
1205     my $skip = [ 'type' ];
1206     push @$skip, 'mp' if $nomp;
1207     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
1208 }
1209
1210 sub print_ct_unused {
1211     my ($class, $info) = @_;
1212
1213     my $skip = [ 'type' ];
1214     return PVE::JSONSchema::print_property_string($info, $unused_desc, $skip);
1215 }
1216
1217 sub parse_volume {
1218     my ($class, $key, $volume_string, $noerr) = @_;
1219
1220     if ($key eq 'rootfs') {
1221         my $res =  $parse_ct_mountpoint_full->($class, $rootfs_desc, $volume_string, $noerr);
1222         $res->{mp} = '/' if defined($res);
1223         return $res;
1224     } elsif ($key =~ m/^mp\d+$/) {
1225         return $parse_ct_mountpoint_full->($class, $mp_desc, $volume_string, $noerr);
1226     } elsif ($key =~ m/^unused\d+$/) {
1227         return $parse_ct_mountpoint_full->($class, $unused_desc, $volume_string, $noerr);
1228     }
1229
1230     die "parse_volume - unknown type: $key\n" if !$noerr;
1231
1232     return;
1233 }
1234
1235 sub print_volume {
1236     my ($class, $key, $volume) = @_;
1237
1238     return $class->print_ct_unused($volume) if $key =~ m/^unused(\d+)$/;
1239
1240     return $class->print_ct_mountpoint($volume, $key eq 'rootfs');
1241 }
1242
1243 sub volid_key {
1244     my ($class) = @_;
1245
1246     return 'volume';
1247 }
1248
1249 sub print_lxc_network {
1250     my ($class, $net) = @_;
1251     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
1252 }
1253
1254 sub parse_lxc_network {
1255     my ($class, $data) = @_;
1256
1257     my $res = {};
1258
1259     return $res if !$data;
1260
1261     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
1262
1263     $res->{type} = 'veth';
1264     if (!$res->{hwaddr}) {
1265         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1266         $res->{hwaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1267     }
1268
1269     return $res;
1270 }
1271
1272 sub parse_features {
1273     my ($class, $data) = @_;
1274     return {} if !$data;
1275     return PVE::JSONSchema::parse_property_string($features_desc, $data);
1276 }
1277
1278 sub option_exists {
1279     my ($class, $name) = @_;
1280
1281     return defined($confdesc->{$name});
1282 }
1283 # END JSON config code
1284
1285 my $LXC_FASTPLUG_OPTIONS= {
1286     'description' => 1,
1287     'onboot' => 1,
1288     'startup' => 1,
1289     'protection' => 1,
1290     'hostname' => 1,
1291     'hookscript' => 1,
1292     'cores' => 1,
1293     'tags' => 1,
1294     'lock' => 1,
1295 };
1296
1297 sub vmconfig_hotplug_pending {
1298     my ($class, $vmid, $conf, $storecfg, $selection, $errors) = @_;
1299
1300     my $pid = PVE::LXC::find_lxc_pid($vmid);
1301     my $rootdir = "/proc/$pid/root";
1302
1303     my $add_hotplug_error = sub {
1304         my ($opt, $msg) = @_;
1305         $errors->{$opt} = "unable to hotplug $opt: $msg";
1306     };
1307
1308     foreach my $opt (sort keys %{$conf->{pending}}) { # add/change
1309         next if $selection && !$selection->{$opt};
1310         if ($LXC_FASTPLUG_OPTIONS->{$opt}) {
1311             $conf->{$opt} = delete $conf->{pending}->{$opt};
1312         }
1313     }
1314
1315     my $cgroup = PVE::LXC::CGroup->new($vmid);
1316
1317     # There's no separate swap size to configure, there's memory and "total"
1318     # memory (iow. memory+swap). This means we have to change them together.
1319     my $hotplug_memory_done;
1320     my $hotplug_memory = sub {
1321         my ($wanted_memory, $wanted_swap) = @_;
1322
1323         $wanted_memory = int($wanted_memory * 1024 * 1024) if defined($wanted_memory);
1324         $wanted_swap = int($wanted_swap * 1024 * 1024) if defined($wanted_swap);
1325         $cgroup->change_memory_limit($wanted_memory, $wanted_swap);
1326
1327         $hotplug_memory_done = 1;
1328     };
1329
1330     my $pending_delete_hash = $class->parse_pending_delete($conf->{pending}->{delete});
1331     # FIXME: $force deletion is not implemented for CTs
1332     foreach my $opt (sort keys %$pending_delete_hash) {
1333         next if $selection && !$selection->{$opt};
1334         eval {
1335             if ($LXC_FASTPLUG_OPTIONS->{$opt}) {
1336                 # pass
1337             } elsif ($opt =~ m/^unused(\d+)$/) {
1338                 PVE::LXC::delete_mountpoint_volume($storecfg, $vmid, $conf->{$opt})
1339                     if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1340             } elsif ($opt eq 'swap') {
1341                 $hotplug_memory->(undef, 0);
1342             } elsif ($opt eq 'cpulimit') {
1343                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
1344             } elsif ($opt eq 'cpuunits') {
1345                 $cgroup->change_cpu_shares(undef, $confdesc->{cpuunits}->{default});
1346             } elsif ($opt =~ m/^net(\d)$/) {
1347                 my $netid = $1;
1348                 PVE::Network::veth_delete("veth${vmid}i$netid");
1349             } else {
1350                 die "skip\n"; # skip non-hotpluggable opts
1351             }
1352         };
1353         if (my $err = $@) {
1354             $add_hotplug_error->($opt, $err) if $err ne "skip\n";
1355         } else {
1356             delete $conf->{$opt};
1357             $class->remove_from_pending_delete($conf, $opt);
1358         }
1359     }
1360
1361     foreach my $opt (sort keys %{$conf->{pending}}) {
1362         next if $opt eq 'delete'; # just to be sure
1363         next if $selection && !$selection->{$opt};
1364         my $value = $conf->{pending}->{$opt};
1365         eval {
1366             if ($opt eq 'cpulimit') {
1367                 my $quota = 100000 * $value;
1368                 $cgroup->change_cpu_quota(int(100000 * $value), 100000);
1369             } elsif ($opt eq 'cpuunits') {
1370                 $cgroup->change_cpu_shares($value, $confdesc->{cpuunits}->{default});
1371             } elsif ($opt =~ m/^net(\d+)$/) {
1372                 my $netid = $1;
1373                 my $net = $class->parse_lxc_network($value);
1374                 $value = $class->print_lxc_network($net);
1375                 PVE::LXC::update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1376             } elsif ($opt eq 'memory' || $opt eq 'swap') {
1377                 if (!$hotplug_memory_done) { # don't call twice if both opts are passed
1378                     $hotplug_memory->($conf->{pending}->{memory}, $conf->{pending}->{swap});
1379                 }
1380             } elsif ($opt =~ m/^mp(\d+)$/) {
1381                 if (!PVE::LXC::Tools::can_use_new_mount_api()) {
1382                     die "skip\n";
1383                 }
1384
1385                 if (exists($conf->{$opt})) {
1386                     die "skip\n"; # don't try to hotplug over existing mp
1387                 }
1388
1389                 $class->apply_pending_mountpoint($vmid, $conf, $opt, $storecfg, 1);
1390                 # apply_pending_mountpoint modifies the value if it creates a new disk
1391                 $value = $conf->{pending}->{$opt};
1392             } else {
1393                 die "skip\n"; # skip non-hotpluggable
1394             }
1395         };
1396         if (my $err = $@) {
1397             $add_hotplug_error->($opt, $err) if $err ne "skip\n";
1398         } else {
1399             $conf->{$opt} = $value;
1400             delete $conf->{pending}->{$opt};
1401         }
1402     }
1403 }
1404
1405 sub vmconfig_apply_pending {
1406     my ($class, $vmid, $conf, $storecfg, $selection, $errors) = @_;
1407
1408     my $add_apply_error = sub {
1409         my ($opt, $msg) = @_;
1410         my $err_msg = "unable to apply pending change $opt : $msg";
1411         $errors->{$opt} = $err_msg;
1412         warn $err_msg;
1413     };
1414
1415     my $pending_delete_hash = $class->parse_pending_delete($conf->{pending}->{delete});
1416     # FIXME: $force deletion is not implemented for CTs
1417     foreach my $opt (sort keys %$pending_delete_hash) {
1418         next if $selection && !$selection->{$opt};
1419         eval {
1420             if ($opt =~ m/^mp(\d+)$/) {
1421                 my $mp = $class->parse_volume($opt, $conf->{$opt});
1422                 if ($mp->{type} eq 'volume') {
1423                     $class->add_unused_volume($conf, $mp->{volume})
1424                         if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1425                 }
1426             } elsif ($opt =~ m/^unused(\d+)$/) {
1427                 PVE::LXC::delete_mountpoint_volume($storecfg, $vmid, $conf->{$opt})
1428                     if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1429             }
1430         };
1431         if (my $err = $@) {
1432             $add_apply_error->($opt, $err);
1433         } else {
1434             delete $conf->{$opt};
1435             $class->remove_from_pending_delete($conf, $opt);
1436         }
1437     }
1438
1439     $class->cleanup_pending($conf);
1440
1441     foreach my $opt (sort keys %{$conf->{pending}}) { # add/change
1442         next if $opt eq 'delete'; # just to be sure
1443         next if $selection && !$selection->{$opt};
1444         eval {
1445             if ($opt =~ m/^mp(\d+)$/) {
1446                 $class->apply_pending_mountpoint($vmid, $conf, $opt, $storecfg, 0);
1447             } elsif ($opt =~ m/^net(\d+)$/) {
1448                 my $netid = $1;
1449                 my $net = $class->parse_lxc_network($conf->{pending}->{$opt});
1450                 $conf->{pending}->{$opt} = $class->print_lxc_network($net);
1451             }
1452         };
1453         if (my $err = $@) {
1454             $add_apply_error->($opt, $err);
1455         } else {
1456             $conf->{$opt} = delete $conf->{pending}->{$opt};
1457         }
1458     }
1459 }
1460
1461 my $rescan_volume = sub {
1462     my ($storecfg, $mp) = @_;
1463     eval {
1464         $mp->{size} = PVE::Storage::volume_size_info($storecfg, $mp->{volume}, 5);
1465     };
1466     warn "Could not rescan volume size - $@\n" if $@;
1467 };
1468
1469 sub apply_pending_mountpoint {
1470     my ($class, $vmid, $conf, $opt, $storecfg, $running) = @_;
1471
1472     my $mp = $class->parse_volume($opt, $conf->{pending}->{$opt});
1473     my $old = $conf->{$opt};
1474     if ($mp->{type} eq 'volume' && $mp->{volume} =~ $PVE::LXC::NEW_DISK_RE) {
1475         my $original_value = $conf->{pending}->{$opt};
1476         my $vollist = PVE::LXC::create_disks(
1477             $storecfg,
1478             $vmid,
1479             { $opt => $original_value },
1480             $conf,
1481             1,
1482         );
1483         if ($running) {
1484             # Re-parse mount point:
1485             my $mp = $class->parse_volume($opt, $conf->{pending}->{$opt});
1486             eval {
1487                 PVE::LXC::mountpoint_hotplug($vmid, $conf, $opt, $mp, $storecfg);
1488             };
1489             my $err = $@;
1490             if ($err) {
1491                 PVE::LXC::destroy_disks($storecfg, $vollist);
1492                 # The pending-changes code collects errors but keeps on looping through further
1493                 # pending changes, so unroll the change in $conf as well if destroy_disks()
1494                 # didn't die().
1495                 $conf->{pending}->{$opt} = $original_value;
1496                 die $err;
1497             }
1498         }
1499     } else {
1500         die "skip\n" if $running && defined($old); # TODO: "changing" mount points?
1501         $rescan_volume->($storecfg, $mp) if $mp->{type} eq 'volume';
1502         if ($running) {
1503             PVE::LXC::mountpoint_hotplug($vmid, $conf, $opt, $mp, $storecfg);
1504         }
1505         $conf->{pending}->{$opt} = $class->print_ct_mountpoint($mp);
1506     }
1507
1508     if (defined($old)) {
1509         my $mp = $class->parse_volume($opt, $old);
1510         if ($mp->{type} eq 'volume') {
1511             $class->add_unused_volume($conf, $mp->{volume})
1512                 if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1513         }
1514     }
1515 }
1516
1517 sub classify_mountpoint {
1518     my ($class, $vol) = @_;
1519     if ($vol =~ m!^/!) {
1520         return 'device' if $vol =~ m!^/dev/!;
1521         return 'bind';
1522     }
1523     return 'volume';
1524 }
1525
1526 my $__is_volume_in_use = sub {
1527     my ($class, $config, $volid) = @_;
1528     my $used = 0;
1529
1530     $class->foreach_volume($config, sub {
1531         my ($ms, $mountpoint) = @_;
1532         return if $used;
1533         $used = $mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid;
1534     });
1535
1536     return $used;
1537 };
1538
1539 sub is_volume_in_use_by_snapshots {
1540     my ($class, $config, $volid) = @_;
1541
1542     if (my $snapshots = $config->{snapshots}) {
1543         foreach my $snap (keys %$snapshots) {
1544             return 1 if $__is_volume_in_use->($class, $snapshots->{$snap}, $volid);
1545         }
1546     }
1547
1548     return 0;
1549 }
1550
1551 sub is_volume_in_use {
1552     my ($class, $config, $volid, $include_snapshots, $include_pending) = @_;
1553     return 1 if $__is_volume_in_use->($class, $config, $volid);
1554     return 1 if $include_snapshots && $class->is_volume_in_use_by_snapshots($config, $volid);
1555     return 1 if $include_pending && $__is_volume_in_use->($class, $config->{pending}, $volid);
1556     return 0;
1557 }
1558
1559 sub has_dev_console {
1560     my ($class, $conf) = @_;
1561
1562     return !(defined($conf->{console}) && !$conf->{console});
1563 }
1564
1565 sub has_lxc_entry {
1566     my ($class, $conf, $keyname) = @_;
1567
1568     if (my $lxcconf = $conf->{lxc}) {
1569         foreach my $entry (@$lxcconf) {
1570             my ($key, undef) = @$entry;
1571             return 1 if $key eq $keyname;
1572         }
1573     }
1574
1575     return 0;
1576 }
1577
1578 sub get_tty_count {
1579     my ($class, $conf) = @_;
1580
1581     return $conf->{tty} // $confdesc->{tty}->{default};
1582 }
1583
1584 sub get_cmode {
1585     my ($class, $conf) = @_;
1586
1587     return $conf->{cmode} // $confdesc->{cmode}->{default};
1588 }
1589
1590 sub valid_volume_keys {
1591     my ($class, $reverse) = @_;
1592
1593     my @names = ('rootfs');
1594
1595     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1596         push @names, "mp$i";
1597     }
1598
1599     return $reverse ? reverse @names : @names;
1600 }
1601
1602 sub valid_volume_keys_with_unused {
1603     my ($class, $reverse) = @_;
1604     my @names = $class->valid_volume_keys();
1605     for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1606         push @names, "unused$i";
1607     }
1608     return $reverse ? reverse @names : @names;
1609 }
1610
1611 sub get_vm_volumes {
1612     my ($class, $conf, $excludes) = @_;
1613
1614     my $vollist = [];
1615
1616     $class->foreach_volume($conf, sub {
1617         my ($ms, $mountpoint) = @_;
1618
1619         return if $excludes && $ms eq $excludes;
1620
1621         my $volid = $mountpoint->{volume};
1622         return if !$volid || $mountpoint->{type} ne 'volume';
1623
1624         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1625         return if !$sid;
1626
1627         push @$vollist, $volid;
1628     });
1629
1630     return $vollist;
1631 }
1632
1633 sub get_replicatable_volumes {
1634     my ($class, $storecfg, $vmid, $conf, $cleanup, $noerr) = @_;
1635
1636     my $volhash = {};
1637
1638     my $test_volid = sub {
1639         my ($volid, $mountpoint) = @_;
1640
1641         return if !$volid;
1642
1643         my $mptype = $mountpoint->{type};
1644         my $replicate = $mountpoint->{replicate} // 1;
1645
1646         if ($mptype ne 'volume') {
1647             # skip bindmounts if replicate = 0 even for cleanup,
1648             # since bind mounts could not have been replicated ever
1649             return if !$replicate;
1650             die "unable to replicate mountpoint type '$mptype'\n";
1651         }
1652
1653         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, $noerr);
1654         return if !$storeid;
1655
1656         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1657         return if $scfg->{shared};
1658
1659         my ($path, $owner, $vtype) = PVE::Storage::path($storecfg, $volid);
1660         return if !$owner || ($owner != $vmid);
1661
1662         die "unable to replicate volume '$volid', type '$vtype'\n" if $vtype ne 'images';
1663
1664         return if !$cleanup && !$replicate;
1665
1666         if (!PVE::Storage::volume_has_feature($storecfg, 'replicate', $volid)) {
1667             return if $cleanup || $noerr;
1668             die "missing replicate feature on volume '$volid'\n";
1669         }
1670
1671         $volhash->{$volid} = 1;
1672     };
1673
1674     $class->foreach_volume($conf, sub {
1675         my ($ms, $mountpoint) = @_;
1676         $test_volid->($mountpoint->{volume}, $mountpoint);
1677     });
1678
1679     foreach my $snapname (keys %{$conf->{snapshots}}) {
1680         my $snap = $conf->{snapshots}->{$snapname};
1681         $class->foreach_volume($snap, sub {
1682             my ($ms, $mountpoint) = @_;
1683             $test_volid->($mountpoint->{volume}, $mountpoint);
1684         });
1685     }
1686
1687     # add 'unusedX' volumes to volhash
1688     foreach my $key (keys %$conf) {
1689         if ($key =~ m/^unused/) {
1690             $test_volid->($conf->{$key}, { type => 'volume', replicate => 1 });
1691         }
1692     }
1693
1694     return $volhash;
1695 }
1696
1697 sub get_backup_volumes {
1698     my ($class, $conf) = @_;
1699
1700     my $return_volumes = [];
1701
1702     my $test_mountpoint = sub {
1703         my ($key, $volume) = @_;
1704
1705         my ($included, $reason) = $class->mountpoint_backup_enabled($key, $volume);
1706
1707         push @$return_volumes, {
1708             key => $key,
1709             included => $included,
1710             reason => $reason,
1711             volume_config => $volume,
1712         };
1713     };
1714
1715     PVE::LXC::Config->foreach_volume($conf, $test_mountpoint);
1716
1717     return $return_volumes;
1718 }
1719
1720 1;