src/PVE/LXC/Config.pm

   1 package PVE::LXC::Config;
   2
   3 use strict;
   4 use warnings;
   5 use Fcntl qw(O_RDONLY);
   6
   7 use PVE::AbstractConfig;
   8 use PVE::Cluster qw(cfs_register_file);
   9 use PVE::DataCenterConfig;
  10 use PVE::GuestHelpers;
  11 use PVE::INotify;
  12 use PVE::JSONSchema qw(get_standard_option);
  13 use PVE::Tools;
  14
  15 use base qw(PVE::AbstractConfig);
  16
  17 use constant {FIFREEZE => 0xc0045877,
  18               FITHAW   => 0xc0045878};
  19
  20 my $nodename = PVE::INotify::nodename();
  21 my $lock_handles =  {};
  22 my $lockdir = "/run/lock/lxc";
  23 mkdir $lockdir;
  24 mkdir "/etc/pve/nodes/$nodename/lxc";
  25 my $MAX_MOUNT_POINTS = 256;
  26 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
  27
  28 # BEGIN implemented abstract methods from PVE::AbstractConfig
  29
  30 sub guest_type {
  31     return "CT";
  32 }
  33
  34 sub __config_max_unused_disks {
  35     my ($class) = @_;
  36
  37     return $MAX_UNUSED_DISKS;
  38 }
  39
  40 sub config_file_lock {
  41     my ($class, $vmid) = @_;
  42
  43     return "$lockdir/pve-config-${vmid}.lock";
  44 }
  45
  46 sub cfs_config_path {
  47     my ($class, $vmid, $node) = @_;
  48
  49     $node = $nodename if !$node;
  50     return "nodes/$node/lxc/$vmid.conf";
  51 }
  52
  53 sub mountpoint_backup_enabled {
  54     my ($class, $mp_key, $mountpoint) = @_;
  55
  56     my $enabled;
  57     my $reason;
  58
  59     if ($mp_key eq 'rootfs') {
  60         $enabled = 1;
  61         $reason = 'rootfs';
  62     } elsif ($mountpoint->{type} ne 'volume') {
  63         $enabled = 0;
  64         $reason = 'not a volume';
  65     } elsif ($mountpoint->{backup}) {
  66         $enabled = 1;
  67         $reason = 'enabled';
  68     } else {
  69         $enabled = 0;
  70         $reason = 'disabled';
  71     }
  72     return wantarray ? ($enabled, $reason) : $enabled;
  73 }
  74
  75 sub has_feature {
  76     my ($class, $feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
  77     my $err;
  78
  79     my $opts;
  80     if ($feature eq 'copy' || $feature eq 'clone') {
  81         $opts = {'valid_target_formats' => ['raw', 'subvol']};
  82     }
  83
  84     $class->foreach_volume($conf, sub {
  85         my ($ms, $mountpoint) = @_;
  86
  87         return if $err; # skip further test
  88         return if $backup_only && !$class->mountpoint_backup_enabled($ms, $mountpoint);
  89
  90         $err = 1
  91             if !PVE::Storage::volume_has_feature($storecfg, $feature,
  92                                                  $mountpoint->{volume},
  93                                                  $snapname, $running, $opts);
  94     });
  95
  96     return $err ? 0 : 1;
  97 }
  98
  99 sub __snapshot_save_vmstate {
 100     my ($class, $vmid, $conf, $snapname, $storecfg) = @_;
 101     die "implement me - snapshot_save_vmstate\n";
 102 }
 103
 104 sub __snapshot_activate_storages {
 105     my ($class, $conf, $include_vmstate) = @_;
 106
 107     my $storecfg = PVE::Storage::config();
 108     my $opts = $include_vmstate ? { 'extra_keys' => ['vmstate'] } : {};
 109     my $storage_hash = {};
 110
 111     $class->foreach_volume_full($conf, $opts, sub {
 112         my ($vs, $mountpoint) = @_;
 113
 114         return if $mountpoint->{type} ne 'volume';
 115
 116         my ($storeid) = PVE::Storage::parse_volume_id($mountpoint->{volume});
 117         $storage_hash->{$storeid} = 1;
 118     });
 119
 120     PVE::Storage::activate_storage_list($storecfg, [ sort keys $storage_hash->%* ]);
 121 }
 122
 123 sub __snapshot_check_running {
 124     my ($class, $vmid) = @_;
 125     return PVE::LXC::check_running($vmid);
 126 }
 127
 128 sub __snapshot_check_freeze_needed {
 129     my ($class, $vmid, $config, $save_vmstate) = @_;
 130
 131     my $ret = $class->__snapshot_check_running($vmid);
 132     return ($ret, $ret);
 133 }
 134
 135 # implements similar functionality to fsfreeze(8)
 136 sub fsfreeze_mountpoint {
 137     my ($path, $thaw) = @_;
 138
 139     my $op = $thaw ? 'thaw' : 'freeze';
 140     my $ioctl = $thaw ? FITHAW : FIFREEZE;
 141
 142     sysopen my $fd, $path, O_RDONLY or die "failed to open $path: $!\n";
 143     my $ioctl_err;
 144     if (!ioctl($fd, $ioctl, 0)) {
 145         $ioctl_err = "$!";
 146     }
 147     close($fd);
 148     die "fs$op '$path' failed - $ioctl_err\n" if defined $ioctl_err;
 149 }
 150
 151 sub __snapshot_freeze {
 152     my ($class, $vmid, $unfreeze) = @_;
 153
 154     my $conf = $class->load_config($vmid);
 155     my $storagecfg = PVE::Storage::config();
 156
 157     my $freeze_mps = [];
 158     $class->foreach_volume($conf, sub {
 159         my ($ms, $mountpoint) = @_;
 160
 161         return if $mountpoint->{type} ne 'volume';
 162
 163         if (PVE::Storage::volume_snapshot_needs_fsfreeze($storagecfg, $mountpoint->{volume})) {
 164             push @$freeze_mps, $mountpoint->{mp};
 165         }
 166     });
 167
 168     my $freeze_mountpoints = sub {
 169         my ($thaw) = @_;
 170
 171         return if scalar(@$freeze_mps) == 0;
 172
 173         my $pid = PVE::LXC::find_lxc_pid($vmid);
 174
 175         for my $mp (@$freeze_mps) {
 176             eval{ fsfreeze_mountpoint("/proc/${pid}/root/${mp}", $thaw); };
 177             warn $@ if $@;
 178         }
 179     };
 180
 181     if ($unfreeze) {
 182         eval { PVE::LXC::thaw($vmid); };
 183         warn $@ if $@;
 184         $freeze_mountpoints->(1);
 185     } else {
 186         PVE::LXC::freeze($vmid);
 187         PVE::LXC::sync_container_namespace($vmid);
 188         $freeze_mountpoints->(0);
 189     }
 190 }
 191
 192 sub __snapshot_create_vol_snapshot {
 193     my ($class, $vmid, $ms, $mountpoint, $snapname) = @_;
 194
 195     my $storecfg = PVE::Storage::config();
 196
 197     return if $snapname eq 'vzdump' &&
 198         !$class->mountpoint_backup_enabled($ms, $mountpoint);
 199
 200     PVE::Storage::volume_snapshot($storecfg, $mountpoint->{volume}, $snapname);
 201 }
 202
 203 sub __snapshot_delete_remove_drive {
 204     my ($class, $snap, $remove_drive) = @_;
 205
 206     if ($remove_drive eq 'vmstate') {
 207         die "implement me - saving vmstate\n";
 208     } else {
 209         my $value = $snap->{$remove_drive};
 210         my $mountpoint = $class->parse_volume($remove_drive, $value, 1);
 211         delete $snap->{$remove_drive};
 212
 213         $class->add_unused_volume($snap, $mountpoint->{volume})
 214             if $mountpoint && ($mountpoint->{type} eq 'volume');
 215     }
 216 }
 217
 218 sub __snapshot_delete_vmstate_file {
 219     my ($class, $snap, $force) = @_;
 220
 221     die "implement me - saving vmstate\n";
 222 }
 223
 224 sub __snapshot_delete_vol_snapshot {
 225     my ($class, $vmid, $ms, $mountpoint, $snapname, $unused) = @_;
 226
 227     return if $snapname eq 'vzdump' &&
 228         !$class->mountpoint_backup_enabled($ms, $mountpoint);
 229
 230     my $storecfg = PVE::Storage::config();
 231     PVE::Storage::volume_snapshot_delete($storecfg, $mountpoint->{volume}, $snapname);
 232     push @$unused, $mountpoint->{volume};
 233 }
 234
 235 sub __snapshot_rollback_vol_possible {
 236     my ($class, $mountpoint, $snapname, $blockers) = @_;
 237
 238     my $storecfg = PVE::Storage::config();
 239     PVE::Storage::volume_rollback_is_possible(
 240         $storecfg,
 241         $mountpoint->{volume},
 242         $snapname,
 243         $blockers,
 244     );
 245 }
 246
 247 sub __snapshot_rollback_vol_rollback {
 248     my ($class, $mountpoint, $snapname) = @_;
 249
 250     my $storecfg = PVE::Storage::config();
 251     PVE::Storage::volume_snapshot_rollback($storecfg, $mountpoint->{volume}, $snapname);
 252 }
 253
 254 sub __snapshot_rollback_vm_stop {
 255     my ($class, $vmid) = @_;
 256
 257     PVE::LXC::vm_stop($vmid, 1)
 258         if $class->__snapshot_check_running($vmid);
 259 }
 260
 261 sub __snapshot_rollback_vm_start {
 262     my ($class, $vmid, $vmstate, $data);
 263
 264     die "implement me - save vmstate\n";
 265 }
 266
 267 sub __snapshot_rollback_get_unused {
 268     my ($class, $conf, $snap) = @_;
 269
 270     my $unused = [];
 271
 272     $class->foreach_volume($conf, sub {
 273         my ($vs, $volume) = @_;
 274
 275         return if $volume->{type} ne 'volume';
 276
 277         my $found = 0;
 278         my $volid = $volume->{volume};
 279
 280         $class->foreach_volume($snap, sub {
 281             my ($ms, $mountpoint) = @_;
 282
 283             return if $found;
 284             return if ($mountpoint->{type} ne 'volume');
 285
 286             $found = 1
 287                 if ($mountpoint->{volume} && $mountpoint->{volume} eq $volid);
 288         });
 289
 290         push @$unused, $volid if !$found;
 291     });
 292
 293     return $unused;
 294 }
 295
 296 # END implemented abstract methods from PVE::AbstractConfig
 297
 298 # BEGIN JSON config code
 299
 300 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
 301
 302
 303 my $valid_mount_option_re = qr/(noatime|lazytime|nodev|nosuid|noexec)/;
 304
 305 sub is_valid_mount_option {
 306     my ($option) = @_;
 307     return $option =~ $valid_mount_option_re;
 308 }
 309
 310 my $rootfs_desc = {
 311     volume => {
 312         type => 'string',
 313         default_key => 1,
 314         format => 'pve-lxc-mp-string',
 315         format_description => 'volume',
 316         description => 'Volume, device or directory to mount into the container.',
 317     },
 318     size => {
 319         type => 'string',
 320         format => 'disk-size',
 321         format_description => 'DiskSize',
 322         description => 'Volume size (read only value).',
 323         optional => 1,
 324     },
 325     acl => {
 326         type => 'boolean',
 327         description => 'Explicitly enable or disable ACL support.',
 328         optional => 1,
 329     },
 330     mountoptions => {
 331         optional => 1,
 332         type => 'string',
 333         description => 'Extra mount options for rootfs/mps.',
 334         format_description => 'opt[;opt...]',
 335         pattern => qr/$valid_mount_option_re(;$valid_mount_option_re)*/,
 336     },
 337     ro => {
 338         type => 'boolean',
 339         description => 'Read-only mount point',
 340         optional => 1,
 341     },
 342     quota => {
 343         type => 'boolean',
 344         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
 345         optional => 1,
 346     },
 347     replicate => {
 348         type => 'boolean',
 349         description => 'Will include this volume to a storage replica job.',
 350         optional => 1,
 351         default => 1,
 352     },
 353     shared => {
 354         type => 'boolean',
 355         description => 'Mark this non-volume mount point as available on multiple nodes (see \'nodes\')',
 356         verbose_description => "Mark this non-volume mount point as available on all nodes.\n\nWARNING: This option does not share the mount point automatically, it assumes it is shared already!",
 357         optional => 1,
 358         default => 0,
 359     },
 360 };
 361
 362 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
 363     type => 'string', format => $rootfs_desc,
 364     description => "Use volume as container root.",
 365     optional => 1,
 366 });
 367
 368 # IP address with optional interface suffix for link local ipv6 addresses
 369 PVE::JSONSchema::register_format('lxc-ip-with-ll-iface', \&verify_ip_with_ll_iface);
 370 sub verify_ip_with_ll_iface {
 371     my ($addr, $noerr) = @_;
 372
 373     if (my ($addr, $iface) = ($addr =~ /^(fe80:[^%]+)%(.*)$/)) {
 374         if (PVE::JSONSchema::pve_verify_ip($addr, 1)
 375             && PVE::JSONSchema::pve_verify_iface($iface, 1))
 376         {
 377             return $addr;
 378         }
 379     }
 380
 381     return PVE::JSONSchema::pve_verify_ip($addr, $noerr);
 382 }
 383
 384
 385 my $features_desc = {
 386     mount => {
 387         optional => 1,
 388         type => 'string',
 389         description => "Allow mounting file systems of specific types."
 390             ." This should be a list of file system types as used with the mount command."
 391             ." Note that this can have negative effects on the container's security."
 392             ." With access to a loop device, mounting a file can circumvent the mknod"
 393             ." permission of the devices cgroup, mounting an NFS file system can"
 394             ." block the host's I/O completely and prevent it from rebooting, etc.",
 395         format_description => 'fstype;fstype;...',
 396         pattern => qr/[a-zA-Z0-9_; ]+/,
 397     },
 398     nesting => {
 399         optional => 1,
 400         type => 'boolean',
 401         default => 0,
 402         description => "Allow nesting."
 403             ." Best used with unprivileged containers with additional id mapping."
 404             ." Note that this will expose procfs and sysfs contents of the host"
 405             ." to the guest.",
 406     },
 407     keyctl => {
 408         optional => 1,
 409         type => 'boolean',
 410         default => 0,
 411         description => "For unprivileged containers only: Allow the use of the keyctl() system call."
 412             ." This is required to use docker inside a container."
 413             ." By default unprivileged containers will see this system call as non-existent."
 414             ." This is mostly a workaround for systemd-networkd, as it will treat it as a fatal"
 415             ." error when some keyctl() operations are denied by the kernel due to lacking permissions."
 416             ." Essentially, you can choose between running systemd-networkd or docker.",
 417     },
 418     fuse => {
 419         optional => 1,
 420         type => 'boolean',
 421         default => 0,
 422         description => "Allow using 'fuse' file systems in a container."
 423             ." Note that interactions between fuse and the freezer cgroup can potentially cause I/O deadlocks.",
 424     },
 425     mknod => {
 426         optional => 1,
 427         type => 'boolean',
 428         default => 0,
 429         description => "Allow unprivileged containers to use mknod() to add certain device nodes."
 430             ." This requires a kernel with seccomp trap to user space support (5.3 or newer)."
 431             ." This is experimental.",
 432     },
 433     force_rw_sys => {
 434         optional => 1,
 435         type => 'boolean',
 436         default => 0,
 437         description => "Mount /sys in unprivileged containers as `rw` instead of `mixed`."
 438             ." This can break networking under newer (>= v245) systemd-network use."
 439     },
 440 };
 441
 442 my $confdesc = {
 443     lock => {
 444         optional => 1,
 445         type => 'string',
 446         description => "Lock/unlock the VM.",
 447         enum => [qw(backup create destroyed disk fstrim migrate mounted rollback snapshot snapshot-delete)],
 448     },
 449     onboot => {
 450         optional => 1,
 451         type => 'boolean',
 452         description => "Specifies whether a VM will be started during system bootup.",
 453         default => 0,
 454     },
 455     startup => get_standard_option('pve-startup-order'),
 456     template => {
 457         optional => 1,
 458         type => 'boolean',
 459         description => "Enable/disable Template.",
 460         default => 0,
 461     },
 462     arch => {
 463         optional => 1,
 464         type => 'string',
 465         enum => ['amd64', 'i386', 'arm64', 'armhf'],
 466         description => "OS architecture type.",
 467         default => 'amd64',
 468     },
 469     ostype => {
 470         optional => 1,
 471         type => 'string',
 472         enum => [qw(debian devuan ubuntu centos fedora opensuse archlinux alpine gentoo unmanaged)],
 473         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 474     },
 475     console => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Attach a console device (/dev/console) to the container.",
 479         default => 1,
 480     },
 481     tty => {
 482         optional => 1,
 483         type => 'integer',
 484         description => "Specify the number of tty available to the container",
 485         minimum => 0,
 486         maximum => 6,
 487         default => 2,
 488     },
 489     cores => {
 490         optional => 1,
 491         type => 'integer',
 492         description => "The number of cores assigned to the container. A container can use all available cores by default.",
 493         minimum => 1,
 494         maximum => 8192,
 495     },
 496     cpulimit => {
 497         optional => 1,
 498         type => 'number',
 499         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 500         minimum => 0,
 501         maximum => 8192,
 502         default => 0,
 503     },
 504     cpuunits => {
 505         optional => 1,
 506         type => 'integer',
 507         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 508         minimum => 0,
 509         maximum => 500000,
 510         default => 1024,
 511     },
 512     memory => {
 513         optional => 1,
 514         type => 'integer',
 515         description => "Amount of RAM for the VM in MB.",
 516         minimum => 16,
 517         default => 512,
 518     },
 519     swap => {
 520         optional => 1,
 521         type => 'integer',
 522         description => "Amount of SWAP for the VM in MB.",
 523         minimum => 0,
 524         default => 512,
 525     },
 526     hostname => {
 527         optional => 1,
 528         description => "Set a host name for the container.",
 529         type => 'string', format => 'dns-name',
 530         maxLength => 255,
 531     },
 532     description => {
 533         optional => 1,
 534         type => 'string',
 535         description => "Description for the Container. Shown in the web-interface CT's summary."
 536             ." This is saved as comment inside the configuration file.",
 537         maxLength => 1024 * 8,
 538     },
 539     searchdomain => {
 540         optional => 1,
 541         type => 'string', format => 'dns-name-list',
 542         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 543     },
 544     nameserver => {
 545         optional => 1,
 546         type => 'string', format => 'lxc-ip-with-ll-iface-list',
 547         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 548     },
 549     timezone => {
 550         optional => 1,
 551         type => 'string', format => 'pve-ct-timezone',
 552         description => "Time zone to use in the container. If option isn't set, then nothing will be done. Can be set to 'host' to match the host time zone, or an arbitrary time zone option from /usr/share/zoneinfo/zone.tab",
 553     },
 554     rootfs => get_standard_option('pve-ct-rootfs'),
 555     parent => {
 556         optional => 1,
 557         type => 'string', format => 'pve-configid',
 558         maxLength => 40,
 559         description => "Parent snapshot name. This is used internally, and should not be modified.",
 560     },
 561     snaptime => {
 562         optional => 1,
 563         description => "Timestamp for snapshots.",
 564         type => 'integer',
 565         minimum => 0,
 566     },
 567     cmode => {
 568         optional => 1,
 569         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 570         type => 'string',
 571         enum => ['shell', 'console', 'tty'],
 572         default => 'tty',
 573     },
 574     protection => {
 575         optional => 1,
 576         type => 'boolean',
 577         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 578         default => 0,
 579     },
 580     unprivileged => {
 581         optional => 1,
 582         type => 'boolean',
 583         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 584         default => 0,
 585     },
 586     features => {
 587         optional => 1,
 588         type => 'string',
 589         format => $features_desc,
 590         description => "Allow containers access to advanced features.",
 591     },
 592     hookscript => {
 593         optional => 1,
 594         type => 'string',
 595         format => 'pve-volume-id',
 596         description => 'Script that will be exectued during various steps in the containers lifetime.',
 597     },
 598     tags => {
 599         type => 'string', format => 'pve-tag-list',
 600         description => 'Tags of the Container. This is only meta information.',
 601         optional => 1,
 602     },
 603     debug => {
 604         optional => 1,
 605         type => 'boolean',
 606         description => "Try to be more verbose. For now this only enables debug log-level on start.",
 607         default => 0,
 608     },
 609 };
 610
 611 my $valid_lxc_conf_keys = {
 612     'lxc.apparmor.profile' => 1,
 613     'lxc.apparmor.allow_incomplete' => 1,
 614     'lxc.apparmor.allow_nesting' => 1,
 615     'lxc.apparmor.raw' => 1,
 616     'lxc.selinux.context' => 1,
 617     'lxc.include' => 1,
 618     'lxc.arch' => 1,
 619     'lxc.uts.name' => 1,
 620     'lxc.signal.halt' => 1,
 621     'lxc.signal.reboot' => 1,
 622     'lxc.signal.stop' => 1,
 623     'lxc.init.cmd' => 1,
 624     'lxc.pty.max' => 1,
 625     'lxc.console.logfile' => 1,
 626     'lxc.console.path' => 1,
 627     'lxc.tty.max' => 1,
 628     'lxc.devtty.dir' => 1,
 629     'lxc.hook.autodev' => 1,
 630     'lxc.autodev' => 1,
 631     'lxc.kmsg' => 1,
 632     'lxc.mount.fstab' => 1,
 633     'lxc.mount.entry' => 1,
 634     'lxc.mount.auto' => 1,
 635     'lxc.rootfs.path' => 'lxc.rootfs.path is auto generated from rootfs',
 636     'lxc.rootfs.mount' => 1,
 637     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 638                             ', please use mount point options in the "rootfs" key',
 639     # lxc.cgroup.*
 640     # lxc.prlimit.*
 641     # lxc.net.*
 642     'lxc.cap.drop' => 1,
 643     'lxc.cap.keep' => 1,
 644     'lxc.seccomp.profile' => 1,
 645     'lxc.seccomp.notify.proxy' => 1,
 646     'lxc.seccomp.notify.cookie' => 1,
 647     'lxc.idmap' => 1,
 648     'lxc.hook.pre-start' => 1,
 649     'lxc.hook.pre-mount' => 1,
 650     'lxc.hook.mount' => 1,
 651     'lxc.hook.start' => 1,
 652     'lxc.hook.stop' => 1,
 653     'lxc.hook.post-stop' => 1,
 654     'lxc.hook.clone' => 1,
 655     'lxc.hook.destroy' => 1,
 656     'lxc.hook.version' => 1,
 657     'lxc.log.level' => 1,
 658     'lxc.log.file' => 1,
 659     'lxc.start.auto' => 1,
 660     'lxc.start.delay' => 1,
 661     'lxc.start.order' => 1,
 662     'lxc.group' => 1,
 663     'lxc.environment' => 1,
 664
 665     # All these are namespaced via CLONE_NEWIPC (see namespaces(7)).
 666     'lxc.sysctl.fs.mqueue' => 1,
 667     'lxc.sysctl.kernel.msgmax' => 1,
 668     'lxc.sysctl.kernel.msgmnb' => 1,
 669     'lxc.sysctl.kernel.msgmni' => 1,
 670     'lxc.sysctl.kernel.sem' => 1,
 671     'lxc.sysctl.kernel.shmall' => 1,
 672     'lxc.sysctl.kernel.shmmax' => 1,
 673     'lxc.sysctl.kernel.shmmni' => 1,
 674     'lxc.sysctl.kernel.shm_rmid_forced' => 1,
 675 };
 676
 677 my $deprecated_lxc_conf_keys = {
 678     # Deprecated (removed with lxc 3.0):
 679     'lxc.aa_profile'           => 'lxc.apparmor.profile',
 680     'lxc.aa_allow_incomplete'  => 'lxc.apparmor.allow_incomplete',
 681     'lxc.console'              => 'lxc.console.path',
 682     'lxc.devttydir'            => 'lxc.tty.dir',
 683     'lxc.haltsignal'           => 'lxc.signal.halt',
 684     'lxc.rebootsignal'         => 'lxc.signal.reboot',
 685     'lxc.stopsignal'           => 'lxc.signal.stop',
 686     'lxc.id_map'               => 'lxc.idmap',
 687     'lxc.init_cmd'             => 'lxc.init.cmd',
 688     'lxc.loglevel'             => 'lxc.log.level',
 689     'lxc.logfile'              => 'lxc.log.file',
 690     'lxc.mount'                => 'lxc.mount.fstab',
 691     'lxc.network.type'         => 'lxc.net.INDEX.type',
 692     'lxc.network.flags'        => 'lxc.net.INDEX.flags',
 693     'lxc.network.link'         => 'lxc.net.INDEX.link',
 694     'lxc.network.mtu'          => 'lxc.net.INDEX.mtu',
 695     'lxc.network.name'         => 'lxc.net.INDEX.name',
 696     'lxc.network.hwaddr'       => 'lxc.net.INDEX.hwaddr',
 697     'lxc.network.ipv4'         => 'lxc.net.INDEX.ipv4.address',
 698     'lxc.network.ipv4.gateway' => 'lxc.net.INDEX.ipv4.gateway',
 699     'lxc.network.ipv6'         => 'lxc.net.INDEX.ipv6.address',
 700     'lxc.network.ipv6.gateway' => 'lxc.net.INDEX.ipv6.gateway',
 701     'lxc.network.script.up'    => 'lxc.net.INDEX.script.up',
 702     'lxc.network.script.down'  => 'lxc.net.INDEX.script.down',
 703     'lxc.pts'                  => 'lxc.pty.max',
 704     'lxc.se_context'           => 'lxc.selinux.context',
 705     'lxc.seccomp'              => 'lxc.seccomp.profile',
 706     'lxc.tty'                  => 'lxc.tty.max',
 707     'lxc.utsname'              => 'lxc.uts.name',
 708 };
 709
 710 sub is_valid_lxc_conf_key {
 711     my ($vmid, $key) = @_;
 712     if ($key =~ /^lxc\.limit\./) {
 713         warn "vm $vmid - $key: lxc.limit.* was renamed to lxc.prlimit.*\n";
 714         return 1;
 715     }
 716     if (defined(my $new_name = $deprecated_lxc_conf_keys->{$key})) {
 717         warn "vm $vmid - $key is deprecated and was renamed to $new_name\n";
 718         return 1;
 719     }
 720     my $validity = $valid_lxc_conf_keys->{$key};
 721     return $validity if defined($validity);
 722     return 1 if $key =~ /^lxc\.cgroup2?\./  # allow all cgroup values
 723              || $key =~ /^lxc\.prlimit\./ # allow all prlimits
 724              || $key =~ /^lxc\.net\./;    # allow custom network definitions
 725     return 0;
 726 }
 727
 728 our $netconf_desc = {
 729     type => {
 730         type => 'string',
 731         optional => 1,
 732         description => "Network interface type.",
 733         enum => [qw(veth)],
 734     },
 735     name => {
 736         type => 'string',
 737         format_description => 'string',
 738         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 739         pattern => '[-_.\w\d]+',
 740     },
 741     bridge => {
 742         type => 'string',
 743         format_description => 'bridge',
 744         description => 'Bridge to attach the network device to.',
 745         pattern => '[-_.\w\d]+',
 746         optional => 1,
 747     },
 748     hwaddr => get_standard_option('mac-addr', {
 749         description => 'The interface MAC address. This is dynamically allocated by default, but you can set that statically if needed, for example to always have the same link-local IPv6 address. (lxc.network.hwaddr)',
 750         }),
 751     mtu => {
 752         type => 'integer',
 753         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 754         minimum => 64, # minimum ethernet frame is 64 bytes
 755         optional => 1,
 756     },
 757     ip => {
 758         type => 'string',
 759         format => 'pve-ipv4-config',
 760         format_description => '(IPv4/CIDR|dhcp|manual)',
 761         description => 'IPv4 address in CIDR format.',
 762         optional => 1,
 763     },
 764     gw => {
 765         type => 'string',
 766         format => 'ipv4',
 767         format_description => 'GatewayIPv4',
 768         description => 'Default gateway for IPv4 traffic.',
 769         optional => 1,
 770     },
 771     ip6 => {
 772         type => 'string',
 773         format => 'pve-ipv6-config',
 774         format_description => '(IPv6/CIDR|auto|dhcp|manual)',
 775         description => 'IPv6 address in CIDR format.',
 776         optional => 1,
 777     },
 778     gw6 => {
 779         type => 'string',
 780         format => 'ipv6',
 781         format_description => 'GatewayIPv6',
 782         description => 'Default gateway for IPv6 traffic.',
 783         optional => 1,
 784     },
 785     firewall => {
 786         type => 'boolean',
 787         description => "Controls whether this interface's firewall rules should be used.",
 788         optional => 1,
 789     },
 790     tag => {
 791         type => 'integer',
 792         minimum => 1,
 793         maximum => 4094,
 794         description => "VLAN tag for this interface.",
 795         optional => 1,
 796     },
 797     trunks => {
 798         type => 'string',
 799         pattern => qr/\d+(?:;\d+)*/,
 800         format_description => 'vlanid[;vlanid...]',
 801         description => "VLAN ids to pass through the interface",
 802         optional => 1,
 803     },
 804     rate => {
 805         type => 'number',
 806         format_description => 'mbps',
 807         description => "Apply rate limiting to the interface",
 808         optional => 1,
 809     },
 810 };
 811 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 812
 813 my $MAX_LXC_NETWORKS = 32;
 814 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 815     $confdesc->{"net$i"} = {
 816         optional => 1,
 817         type => 'string', format => $netconf_desc,
 818         description => "Specifies network interfaces for the container.",
 819     };
 820 }
 821
 822 PVE::JSONSchema::register_format('pve-ct-timezone', \&verify_ct_timezone);
 823 sub verify_ct_timezone {
 824     my ($timezone, $noerr) = @_;
 825
 826     return if $timezone eq 'host'; # using host settings
 827
 828     PVE::JSONSchema::pve_verify_timezone($timezone);
 829 }
 830
 831 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 832 sub verify_lxc_mp_string {
 833     my ($mp, $noerr) = @_;
 834
 835     # do not allow:
 836     # /./ or /../
 837     # /. or /.. at the end
 838     # ../ at the beginning
 839
 840     if($mp =~ m@/\.\.?/@ ||
 841        $mp =~ m@/\.\.?$@ ||
 842        $mp =~ m@^\.\./@) {
 843         return undef if $noerr;
 844         die "$mp contains illegal character sequences\n";
 845     }
 846     return $mp;
 847 }
 848
 849 my $mp_desc = {
 850     %$rootfs_desc,
 851     backup => {
 852         type => 'boolean',
 853         description => 'Whether to include the mount point in backups.',
 854         verbose_description => 'Whether to include the mount point in backups '.
 855                                '(only used for volume mount points).',
 856         optional => 1,
 857     },
 858     mp => {
 859         type => 'string',
 860         format => 'pve-lxc-mp-string',
 861         format_description => 'Path',
 862         description => 'Path to the mount point as seen from inside the container '.
 863                        '(must not contain symlinks).',
 864         verbose_description => "Path to the mount point as seen from inside the container.\n\n".
 865                                "NOTE: Must not contain any symlinks for security reasons."
 866     },
 867 };
 868 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 869
 870 my $unused_desc = {
 871     volume => {
 872         type => 'string',
 873         default_key => 1,
 874         format => 'pve-volume-id',
 875         format_description => 'volume',
 876         description => 'The volume that is not used currently.',
 877     }
 878 };
 879
 880 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 881     $confdesc->{"mp$i"} = {
 882         optional => 1,
 883         type => 'string', format => $mp_desc,
 884         description => "Use volume as container mount point. Use the special " .
 885             "syntax STORAGE_ID:SIZE_IN_GiB to allocate a new volume.",
 886         optional => 1,
 887     };
 888 }
 889
 890 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
 891     $confdesc->{"unused$i"} = {
 892         optional => 1,
 893         type => 'string', format => $unused_desc,
 894         description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
 895     }
 896 }
 897
 898 sub parse_pct_config {
 899     my ($filename, $raw) = @_;
 900
 901     return undef if !defined($raw);
 902
 903     my $res = {
 904         digest => Digest::SHA::sha1_hex($raw),
 905         snapshots => {},
 906         pending => {},
 907     };
 908
 909     $filename =~ m|/lxc/(\d+).conf$|
 910         || die "got strange filename '$filename'";
 911
 912     my $vmid = $1;
 913
 914     my $conf = $res;
 915     my $descr = '';
 916     my $section = '';
 917
 918     my @lines = split(/\n/, $raw);
 919     foreach my $line (@lines) {
 920         next if $line =~ m/^\s*$/;
 921
 922         if ($line =~ m/^\[pve:pending\]\s*$/i) {
 923             $section = 'pending';
 924             $conf->{description} = $descr if $descr;
 925             $descr = '';
 926             $conf = $res->{$section} = {};
 927             next;
 928         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 929             $section = $1;
 930             $conf->{description} = $descr if $descr;
 931             $descr = '';
 932             $conf = $res->{snapshots}->{$section} = {};
 933             next;
 934         }
 935
 936         if ($line =~ m/^\#(.*)\s*$/) {
 937             $descr .= PVE::Tools::decode_text($1) . "\n";
 938             next;
 939         }
 940
 941         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 942             my $key = $1;
 943             my $value = $3;
 944             my $validity = is_valid_lxc_conf_key($vmid, $key);
 945             if ($validity eq 1) {
 946                 push @{$conf->{lxc}}, [$key, $value];
 947             } elsif (my $errmsg = $validity) {
 948                 warn "vm $vmid - $key: $errmsg\n";
 949             } else {
 950                 warn "vm $vmid - unable to parse config: $line\n";
 951             }
 952         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 953             $descr .= PVE::Tools::decode_text($2);
 954         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 955             $conf->{snapstate} = $1;
 956         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
 957             my $value = $1;
 958             if ($section eq 'pending') {
 959                 $conf->{delete} = $value;
 960             } else {
 961                 warn "vm $vmid - property 'delete' is only allowed in [pve:pending]\n";
 962             }
 963         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
 964             my $key = $1;
 965             my $value = $2;
 966             eval { $value = PVE::LXC::Config->check_type($key, $value); };
 967             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 968             $conf->{$key} = $value;
 969         } else {
 970             warn "vm $vmid - unable to parse config: $line\n";
 971         }
 972     }
 973
 974     $conf->{description} = $descr if $descr;
 975
 976     delete $res->{snapstate}; # just to be sure
 977
 978     return $res;
 979 }
 980
 981 sub write_pct_config {
 982     my ($filename, $conf) = @_;
 983
 984     delete $conf->{snapstate}; # just to be sure
 985
 986     my $volidlist = PVE::LXC::Config->get_vm_volumes($conf);
 987     my $used_volids = {};
 988     foreach my $vid (@$volidlist) {
 989         $used_volids->{$vid} = 1;
 990     }
 991
 992     # remove 'unusedX' settings if the volume is still used
 993     foreach my $key (keys %$conf) {
 994         my $value = $conf->{$key};
 995         if ($key =~ m/^unused/ && $used_volids->{$value}) {
 996             delete $conf->{$key};
 997         }
 998     }
 999
1000     my $generate_raw_config = sub {
1001         my ($conf) = @_;
1002
1003         my $raw = '';
1004
1005         # add description as comment to top of file
1006         my $descr = $conf->{description} || '';
1007         foreach my $cl (split(/\n/, $descr)) {
1008             $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
1009         }
1010
1011         foreach my $key (sort keys %$conf) {
1012             next if $key eq 'digest' || $key eq 'description' ||
1013                     $key eq 'pending' || $key eq 'snapshots' ||
1014                     $key eq 'snapname' || $key eq 'lxc';
1015             my $value = $conf->{$key};
1016             die "detected invalid newline inside property '$key'\n"
1017                 if $value =~ m/\n/;
1018             $raw .= "$key: $value\n";
1019         }
1020
1021         if (my $lxcconf = $conf->{lxc}) {
1022             foreach my $entry (@$lxcconf) {
1023                 my ($k, $v) = @$entry;
1024                 $raw .= "$k: $v\n";
1025             }
1026         }
1027
1028         return $raw;
1029     };
1030
1031     my $raw = &$generate_raw_config($conf);
1032
1033     if (scalar(keys %{$conf->{pending}})){
1034         $raw .= "\n[pve:pending]\n";
1035         $raw .= &$generate_raw_config($conf->{pending});
1036     }
1037
1038     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
1039         $raw .= "\n[$snapname]\n";
1040         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1041     }
1042
1043     return $raw;
1044 }
1045
1046 sub update_pct_config {
1047     my ($class, $vmid, $conf, $running, $param, $delete, $revert) = @_;
1048
1049     my $storage_cfg = PVE::Storage::config();
1050
1051     foreach my $opt (@$revert) {
1052         delete $conf->{pending}->{$opt};
1053         $class->remove_from_pending_delete($conf, $opt); # also remove from deletion queue
1054     }
1055
1056     # write updates to pending section
1057     my $modified = {}; # record modified options
1058
1059     foreach my $opt (@$delete) {
1060         if (!defined($conf->{$opt}) && !defined($conf->{pending}->{$opt})) {
1061             warn "cannot delete '$opt' - not set in current configuration!\n";
1062             next;
1063         }
1064         $modified->{$opt} = 1;
1065         if ($opt eq 'memory' || $opt eq 'rootfs' || $opt eq 'ostype') {
1066             die "unable to delete required option '$opt'\n";
1067         } elsif ($opt =~ m/^unused(\d+)$/) {
1068             $class->check_protection($conf, "can't remove CT $vmid drive '$opt'");
1069         } elsif ($opt =~ m/^mp(\d+)$/) {
1070             $class->check_protection($conf, "can't remove CT $vmid drive '$opt'");
1071         } elsif ($opt eq 'unprivileged') {
1072             die "unable to delete read-only option: '$opt'\n";
1073         }
1074         $class->add_to_pending_delete($conf, $opt);
1075     }
1076
1077     my $check_content_type = sub {
1078         my ($mp) = @_;
1079         my $sid = PVE::Storage::parse_volume_id($mp->{volume});
1080         my $storage_config = PVE::Storage::storage_config($storage_cfg, $sid);
1081         die "storage '$sid' does not allow content type 'rootdir' (Container)\n"
1082             if !$storage_config->{content}->{rootdir};
1083     };
1084
1085     foreach my $opt (sort keys %$param) { # add/change
1086         $modified->{$opt} = 1;
1087         my $value = $param->{$opt};
1088         if ($opt =~ m/^mp(\d+)$/ || $opt eq 'rootfs') {
1089             $class->check_protection($conf, "can't update CT $vmid drive '$opt'");
1090             my $mp = $class->parse_volume($opt, $value);
1091             $check_content_type->($mp) if ($mp->{type} eq 'volume');
1092         } elsif ($opt eq 'hookscript') {
1093             PVE::GuestHelpers::check_hookscript($value);
1094         } elsif ($opt eq 'nameserver') {
1095             $value = PVE::LXC::verify_nameserver_list($value);
1096         } elsif ($opt eq 'searchdomain') {
1097             $value = PVE::LXC::verify_searchdomain_list($value);
1098         } elsif ($opt eq 'unprivileged') {
1099             die "unable to modify read-only option: '$opt'\n";
1100         }
1101         $conf->{pending}->{$opt} = $value;
1102         $class->remove_from_pending_delete($conf, $opt);
1103     }
1104
1105     my $changes = $class->cleanup_pending($conf);
1106
1107     my $errors = {};
1108     if ($running) {
1109         $class->vmconfig_hotplug_pending($vmid, $conf, $storage_cfg, $modified, $errors);
1110     } else {
1111         $class->vmconfig_apply_pending($vmid, $conf, $storage_cfg, $modified, $errors);
1112     }
1113
1114     return $errors;
1115 }
1116
1117 sub check_type {
1118     my ($class, $key, $value) = @_;
1119
1120     die "unknown setting '$key'\n" if !$confdesc->{$key};
1121
1122     my $type = $confdesc->{$key}->{type};
1123
1124     if (!defined($value)) {
1125         die "got undefined value\n";
1126     }
1127
1128     if ($value =~ m/[\n\r]/) {
1129         die "property contains a line feed\n";
1130     }
1131
1132     if ($type eq 'boolean') {
1133         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1134         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1135         die "type check ('boolean') failed - got '$value'\n";
1136     } elsif ($type eq 'integer') {
1137         return int($1) if $value =~ m/^(\d+)$/;
1138         die "type check ('integer') failed - got '$value'\n";
1139     } elsif ($type eq 'number') {
1140         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1141         die "type check ('number') failed - got '$value'\n";
1142     } elsif ($type eq 'string') {
1143         if (my $fmt = $confdesc->{$key}->{format}) {
1144             PVE::JSONSchema::check_format($fmt, $value);
1145             return $value;
1146         }
1147         return $value;
1148     } else {
1149         die "internal error"
1150     }
1151 }
1152
1153
1154 # add JSON properties for create and set function
1155 sub json_config_properties {
1156     my ($class, $prop) = @_;
1157
1158     foreach my $opt (keys %$confdesc) {
1159         next if $opt eq 'parent' || $opt eq 'snaptime';
1160         next if $prop->{$opt};
1161         $prop->{$opt} = $confdesc->{$opt};
1162     }
1163
1164     return $prop;
1165 }
1166
1167 my $parse_ct_mountpoint_full = sub {
1168     my ($class, $desc, $data, $noerr) = @_;
1169
1170     $data //= '';
1171
1172     my $res;
1173     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
1174     if ($@) {
1175         return undef if $noerr;
1176         die $@;
1177     }
1178
1179     if (defined(my $size = $res->{size})) {
1180         $size = PVE::JSONSchema::parse_size($size);
1181         if (!defined($size)) {
1182             return undef if $noerr;
1183             die "invalid size: $size\n";
1184         }
1185         $res->{size} = $size;
1186     }
1187
1188     $res->{type} = $class->classify_mountpoint($res->{volume});
1189
1190     return $res;
1191 };
1192
1193 sub print_ct_mountpoint {
1194     my ($class, $info, $nomp) = @_;
1195     my $skip = [ 'type' ];
1196     push @$skip, 'mp' if $nomp;
1197     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
1198 }
1199
1200 sub parse_volume {
1201     my ($class, $key, $volume_string, $noerr) = @_;
1202
1203     if ($key eq 'rootfs') {
1204         my $res =  $parse_ct_mountpoint_full->($class, $rootfs_desc, $volume_string, $noerr);
1205         $res->{mp} = '/' if defined($res);
1206         return $res;
1207     } elsif ($key =~ m/^mp\d+$/) {
1208         return $parse_ct_mountpoint_full->($class, $mp_desc, $volume_string, $noerr);
1209     } elsif ($key =~ m/^unused\d+$/) {
1210         return $parse_ct_mountpoint_full->($class, $unused_desc, $volume_string, $noerr);
1211     }
1212
1213     die "parse_volume - unknown type: $key\n" if !$noerr;
1214
1215     return;
1216 }
1217
1218 sub print_volume {
1219     my ($class, $key, $volume) = @_;
1220
1221     return $class->print_ct_mountpoint($volume, $key eq 'rootfs');
1222 }
1223
1224 sub volid_key {
1225     my ($class) = @_;
1226
1227     return 'volume';
1228 }
1229
1230 sub print_lxc_network {
1231     my ($class, $net) = @_;
1232     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
1233 }
1234
1235 sub parse_lxc_network {
1236     my ($class, $data) = @_;
1237
1238     my $res = {};
1239
1240     return $res if !$data;
1241
1242     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
1243
1244     $res->{type} = 'veth';
1245     if (!$res->{hwaddr}) {
1246         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1247         $res->{hwaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1248     }
1249
1250     return $res;
1251 }
1252
1253 sub parse_features {
1254     my ($class, $data) = @_;
1255     return {} if !$data;
1256     return PVE::JSONSchema::parse_property_string($features_desc, $data);
1257 }
1258
1259 sub option_exists {
1260     my ($class, $name) = @_;
1261
1262     return defined($confdesc->{$name});
1263 }
1264 # END JSON config code
1265
1266 my $LXC_FASTPLUG_OPTIONS= {
1267     'description' => 1,
1268     'onboot' => 1,
1269     'startup' => 1,
1270     'protection' => 1,
1271     'hostname' => 1,
1272     'hookscript' => 1,
1273     'cores' => 1,
1274     'tags' => 1,
1275     'lock' => 1,
1276 };
1277
1278 sub vmconfig_hotplug_pending {
1279     my ($class, $vmid, $conf, $storecfg, $selection, $errors) = @_;
1280
1281     my $pid = PVE::LXC::find_lxc_pid($vmid);
1282     my $rootdir = "/proc/$pid/root";
1283
1284     my $add_hotplug_error = sub {
1285         my ($opt, $msg) = @_;
1286         $errors->{$opt} = "unable to hotplug $opt: $msg";
1287     };
1288
1289     foreach my $opt (sort keys %{$conf->{pending}}) { # add/change
1290         next if $selection && !$selection->{$opt};
1291         if ($LXC_FASTPLUG_OPTIONS->{$opt}) {
1292             $conf->{$opt} = delete $conf->{pending}->{$opt};
1293         }
1294     }
1295
1296     my $cgroup = PVE::LXC::CGroup->new($vmid);
1297
1298     # There's no separate swap size to configure, there's memory and "total"
1299     # memory (iow. memory+swap). This means we have to change them together.
1300     my $hotplug_memory_done;
1301     my $hotplug_memory = sub {
1302         my ($wanted_memory, $wanted_swap) = @_;
1303
1304         $wanted_memory = int($wanted_memory * 1024 * 1024) if defined($wanted_memory);
1305         $wanted_swap = int($wanted_swap * 1024 * 1024) if defined($wanted_swap);
1306         $cgroup->change_memory_limit($wanted_memory, $wanted_swap);
1307
1308         $hotplug_memory_done = 1;
1309     };
1310
1311     my $pending_delete_hash = $class->parse_pending_delete($conf->{pending}->{delete});
1312     # FIXME: $force deletion is not implemented for CTs
1313     foreach my $opt (sort keys %$pending_delete_hash) {
1314         next if $selection && !$selection->{$opt};
1315         eval {
1316             if ($LXC_FASTPLUG_OPTIONS->{$opt}) {
1317                 # pass
1318             } elsif ($opt =~ m/^unused(\d+)$/) {
1319                 PVE::LXC::delete_mountpoint_volume($storecfg, $vmid, $conf->{$opt})
1320                     if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1321             } elsif ($opt eq 'swap') {
1322                 $hotplug_memory->(undef, 0);
1323             } elsif ($opt eq 'cpulimit') {
1324                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
1325             } elsif ($opt eq 'cpuunits') {
1326                 $cgroup->change_cpu_shares(undef, $confdesc->{cpuunits}->{default});
1327             } elsif ($opt =~ m/^net(\d)$/) {
1328                 my $netid = $1;
1329                 PVE::Network::veth_delete("veth${vmid}i$netid");
1330             } else {
1331                 die "skip\n"; # skip non-hotpluggable opts
1332             }
1333         };
1334         if (my $err = $@) {
1335             $add_hotplug_error->($opt, $err) if $err ne "skip\n";
1336         } else {
1337             delete $conf->{$opt};
1338             $class->remove_from_pending_delete($conf, $opt);
1339         }
1340     }
1341
1342     foreach my $opt (sort keys %{$conf->{pending}}) {
1343         next if $opt eq 'delete'; # just to be sure
1344         next if $selection && !$selection->{$opt};
1345         my $value = $conf->{pending}->{$opt};
1346         eval {
1347             if ($opt eq 'cpulimit') {
1348                 my $quota = 100000 * $value;
1349                 $cgroup->change_cpu_quota(int(100000 * $value), 100000);
1350             } elsif ($opt eq 'cpuunits') {
1351                 $cgroup->change_cpu_shares($value, $confdesc->{cpuunits}->{default});
1352             } elsif ($opt =~ m/^net(\d+)$/) {
1353                 my $netid = $1;
1354                 my $net = $class->parse_lxc_network($value);
1355                 $value = $class->print_lxc_network($net);
1356                 PVE::LXC::update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1357             } elsif ($opt eq 'memory' || $opt eq 'swap') {
1358                 if (!$hotplug_memory_done) { # don't call twice if both opts are passed
1359                     $hotplug_memory->($conf->{pending}->{memory}, $conf->{pending}->{swap});
1360                 }
1361             } elsif ($opt =~ m/^mp(\d+)$/) {
1362                 if (!PVE::LXC::Tools::can_use_new_mount_api()) {
1363                     die "skip\n";
1364                 }
1365
1366                 if (exists($conf->{$opt})) {
1367                     die "skip\n"; # don't try to hotplug over existing mp
1368                 }
1369
1370                 $class->apply_pending_mountpoint($vmid, $conf, $opt, $storecfg, 1);
1371                 # apply_pending_mountpoint modifies the value if it creates a new disk
1372                 $value = $conf->{pending}->{$opt};
1373             } else {
1374                 die "skip\n"; # skip non-hotpluggable
1375             }
1376         };
1377         if (my $err = $@) {
1378             $add_hotplug_error->($opt, $err) if $err ne "skip\n";
1379         } else {
1380             $conf->{$opt} = $value;
1381             delete $conf->{pending}->{$opt};
1382         }
1383     }
1384 }
1385
1386 sub vmconfig_apply_pending {
1387     my ($class, $vmid, $conf, $storecfg, $selection, $errors) = @_;
1388
1389     my $add_apply_error = sub {
1390         my ($opt, $msg) = @_;
1391         my $err_msg = "unable to apply pending change $opt : $msg";
1392         $errors->{$opt} = $err_msg;
1393         warn $err_msg;
1394     };
1395
1396     my $pending_delete_hash = $class->parse_pending_delete($conf->{pending}->{delete});
1397     # FIXME: $force deletion is not implemented for CTs
1398     foreach my $opt (sort keys %$pending_delete_hash) {
1399         next if $selection && !$selection->{$opt};
1400         eval {
1401             if ($opt =~ m/^mp(\d+)$/) {
1402                 my $mp = $class->parse_volume($opt, $conf->{$opt});
1403                 if ($mp->{type} eq 'volume') {
1404                     $class->add_unused_volume($conf, $mp->{volume})
1405                         if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1406                 }
1407             } elsif ($opt =~ m/^unused(\d+)$/) {
1408                 PVE::LXC::delete_mountpoint_volume($storecfg, $vmid, $conf->{$opt})
1409                     if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1410             }
1411         };
1412         if (my $err = $@) {
1413             $add_apply_error->($opt, $err);
1414         } else {
1415             delete $conf->{$opt};
1416             $class->remove_from_pending_delete($conf, $opt);
1417         }
1418     }
1419
1420     $class->cleanup_pending($conf);
1421
1422     foreach my $opt (sort keys %{$conf->{pending}}) { # add/change
1423         next if $opt eq 'delete'; # just to be sure
1424         next if $selection && !$selection->{$opt};
1425         eval {
1426             if ($opt =~ m/^mp(\d+)$/) {
1427                 $class->apply_pending_mountpoint($vmid, $conf, $opt, $storecfg, 0);
1428             } elsif ($opt =~ m/^net(\d+)$/) {
1429                 my $netid = $1;
1430                 my $net = $class->parse_lxc_network($conf->{pending}->{$opt});
1431                 $conf->{pending}->{$opt} = $class->print_lxc_network($net);
1432             }
1433         };
1434         if (my $err = $@) {
1435             $add_apply_error->($opt, $err);
1436         } else {
1437             $conf->{$opt} = delete $conf->{pending}->{$opt};
1438         }
1439     }
1440 }
1441
1442 my $rescan_volume = sub {
1443     my ($storecfg, $mp) = @_;
1444     eval {
1445         $mp->{size} = PVE::Storage::volume_size_info($storecfg, $mp->{volume}, 5);
1446     };
1447     warn "Could not rescan volume size - $@\n" if $@;
1448 };
1449
1450 sub apply_pending_mountpoint {
1451     my ($class, $vmid, $conf, $opt, $storecfg, $running) = @_;
1452
1453     my $mp = $class->parse_volume($opt, $conf->{pending}->{$opt});
1454     my $old = $conf->{$opt};
1455     if ($mp->{type} eq 'volume') {
1456         if ($mp->{volume} =~ $PVE::LXC::NEW_DISK_RE) {
1457             my $original_value = $conf->{pending}->{$opt};
1458             my $vollist = PVE::LXC::create_disks(
1459                 $storecfg,
1460                 $vmid,
1461                 { $opt => $original_value },
1462                 $conf,
1463                 1,
1464             );
1465             if ($running) {
1466                 # Re-parse mount point:
1467                 my $mp = $class->parse_volume($opt, $conf->{pending}->{$opt});
1468                 eval {
1469                     PVE::LXC::mountpoint_hotplug($vmid, $conf, $opt, $mp, $storecfg);
1470                 };
1471                 my $err = $@;
1472                 if ($err) {
1473                     PVE::LXC::destroy_disks($storecfg, $vollist);
1474                     # The pending-changes code collects errors but keeps on looping through further
1475                     # pending changes, so unroll the change in $conf as well if destroy_disks()
1476                     # didn't die().
1477                     $conf->{pending}->{$opt} = $original_value;
1478                     die $err;
1479                 }
1480             }
1481         } else {
1482             die "skip\n" if $running && defined($old); # TODO: "changing" mount points?
1483             $rescan_volume->($storecfg, $mp);
1484             if ($running) {
1485                 PVE::LXC::mountpoint_hotplug($vmid, $conf, $opt, $mp, $storecfg);
1486             }
1487             $conf->{pending}->{$opt} = $class->print_ct_mountpoint($mp);
1488         }
1489     }
1490
1491     if (defined($old)) {
1492         my $mp = $class->parse_volume($opt, $old);
1493         if ($mp->{type} eq 'volume') {
1494             $class->add_unused_volume($conf, $mp->{volume})
1495                 if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1496         }
1497     }
1498 }
1499
1500 sub classify_mountpoint {
1501     my ($class, $vol) = @_;
1502     if ($vol =~ m!^/!) {
1503         return 'device' if $vol =~ m!^/dev/!;
1504         return 'bind';
1505     }
1506     return 'volume';
1507 }
1508
1509 my $__is_volume_in_use = sub {
1510     my ($class, $config, $volid) = @_;
1511     my $used = 0;
1512
1513     $class->foreach_volume($config, sub {
1514         my ($ms, $mountpoint) = @_;
1515         return if $used;
1516         $used = $mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid;
1517     });
1518
1519     return $used;
1520 };
1521
1522 sub is_volume_in_use_by_snapshots {
1523     my ($class, $config, $volid) = @_;
1524
1525     if (my $snapshots = $config->{snapshots}) {
1526         foreach my $snap (keys %$snapshots) {
1527             return 1 if $__is_volume_in_use->($class, $snapshots->{$snap}, $volid);
1528         }
1529     }
1530
1531     return 0;
1532 }
1533
1534 sub is_volume_in_use {
1535     my ($class, $config, $volid, $include_snapshots, $include_pending) = @_;
1536     return 1 if $__is_volume_in_use->($class, $config, $volid);
1537     return 1 if $include_snapshots && $class->is_volume_in_use_by_snapshots($config, $volid);
1538     return 1 if $include_pending && $__is_volume_in_use->($class, $config->{pending}, $volid);
1539     return 0;
1540 }
1541
1542 sub has_dev_console {
1543     my ($class, $conf) = @_;
1544
1545     return !(defined($conf->{console}) && !$conf->{console});
1546 }
1547
1548 sub has_lxc_entry {
1549     my ($class, $conf, $keyname) = @_;
1550
1551     if (my $lxcconf = $conf->{lxc}) {
1552         foreach my $entry (@$lxcconf) {
1553             my ($key, undef) = @$entry;
1554             return 1 if $key eq $keyname;
1555         }
1556     }
1557
1558     return 0;
1559 }
1560
1561 sub get_tty_count {
1562     my ($class, $conf) = @_;
1563
1564     return $conf->{tty} // $confdesc->{tty}->{default};
1565 }
1566
1567 sub get_cmode {
1568     my ($class, $conf) = @_;
1569
1570     return $conf->{cmode} // $confdesc->{cmode}->{default};
1571 }
1572
1573 sub valid_volume_keys {
1574     my ($class, $reverse) = @_;
1575
1576     my @names = ('rootfs');
1577
1578     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1579         push @names, "mp$i";
1580     }
1581
1582     return $reverse ? reverse @names : @names;
1583 }
1584
1585 sub valid_volume_keys_with_unused {
1586     my ($class, $reverse) = @_;
1587     my @names = $class->valid_volume_keys();
1588     for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1589         push @names, "unused$i";
1590     }
1591     return $reverse ? reverse @names : @names;
1592 }
1593
1594 sub get_vm_volumes {
1595     my ($class, $conf, $excludes) = @_;
1596
1597     my $vollist = [];
1598
1599     $class->foreach_volume($conf, sub {
1600         my ($ms, $mountpoint) = @_;
1601
1602         return if $excludes && $ms eq $excludes;
1603
1604         my $volid = $mountpoint->{volume};
1605         return if !$volid || $mountpoint->{type} ne 'volume';
1606
1607         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1608         return if !$sid;
1609
1610         push @$vollist, $volid;
1611     });
1612
1613     return $vollist;
1614 }
1615
1616 sub get_replicatable_volumes {
1617     my ($class, $storecfg, $vmid, $conf, $cleanup, $noerr) = @_;
1618
1619     my $volhash = {};
1620
1621     my $test_volid = sub {
1622         my ($volid, $mountpoint) = @_;
1623
1624         return if !$volid;
1625
1626         my $mptype = $mountpoint->{type};
1627         my $replicate = $mountpoint->{replicate} // 1;
1628
1629         if ($mptype ne 'volume') {
1630             # skip bindmounts if replicate = 0 even for cleanup,
1631             # since bind mounts could not have been replicated ever
1632             return if !$replicate;
1633             die "unable to replicate mountpoint type '$mptype'\n";
1634         }
1635
1636         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, $noerr);
1637         return if !$storeid;
1638
1639         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1640         return if $scfg->{shared};
1641
1642         my ($path, $owner, $vtype) = PVE::Storage::path($storecfg, $volid);
1643         return if !$owner || ($owner != $vmid);
1644
1645         die "unable to replicate volume '$volid', type '$vtype'\n" if $vtype ne 'images';
1646
1647         return if !$cleanup && !$replicate;
1648
1649         if (!PVE::Storage::volume_has_feature($storecfg, 'replicate', $volid)) {
1650             return if $cleanup || $noerr;
1651             die "missing replicate feature on volume '$volid'\n";
1652         }
1653
1654         $volhash->{$volid} = 1;
1655     };
1656
1657     $class->foreach_volume($conf, sub {
1658         my ($ms, $mountpoint) = @_;
1659         $test_volid->($mountpoint->{volume}, $mountpoint);
1660     });
1661
1662     foreach my $snapname (keys %{$conf->{snapshots}}) {
1663         my $snap = $conf->{snapshots}->{$snapname};
1664         $class->foreach_volume($snap, sub {
1665             my ($ms, $mountpoint) = @_;
1666             $test_volid->($mountpoint->{volume}, $mountpoint);
1667         });
1668     }
1669
1670     # add 'unusedX' volumes to volhash
1671     foreach my $key (keys %$conf) {
1672         if ($key =~ m/^unused/) {
1673             $test_volid->($conf->{$key}, { type => 'volume', replicate => 1 });
1674         }
1675     }
1676
1677     return $volhash;
1678 }
1679
1680 sub get_backup_volumes {
1681     my ($class, $conf) = @_;
1682
1683     my $return_volumes = [];
1684
1685     my $test_mountpoint = sub {
1686         my ($key, $volume) = @_;
1687
1688         my ($included, $reason) = $class->mountpoint_backup_enabled($key, $volume);
1689
1690         push @$return_volumes, {
1691             key => $key,
1692             included => $included,
1693             reason => $reason,
1694             volume_config => $volume,
1695         };
1696     };
1697
1698     PVE::LXC::Config->foreach_volume($conf, $test_mountpoint);
1699
1700     return $return_volumes;
1701 }
1702
1703 1;