src/PVE/LXC/Config.pm

   1 package PVE::LXC::Config;
   2
   3 use strict;
   4 use warnings;
   5 use Fcntl qw(O_RDONLY);
   6
   7 use PVE::AbstractConfig;
   8 use PVE::Cluster qw(cfs_register_file);
   9 use PVE::DataCenterConfig;
  10 use PVE::GuestHelpers;
  11 use PVE::INotify;
  12 use PVE::JSONSchema qw(get_standard_option);
  13 use PVE::Tools;
  14
  15 use base qw(PVE::AbstractConfig);
  16
  17 use constant {FIFREEZE => 0xc0045877,
  18               FITHAW   => 0xc0045878};
  19
  20 my $nodename = PVE::INotify::nodename();
  21 my $lock_handles =  {};
  22 my $lockdir = "/run/lock/lxc";
  23 mkdir $lockdir;
  24 mkdir "/etc/pve/nodes/$nodename/lxc";
  25 my $MAX_MOUNT_POINTS = 256;
  26 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
  27
  28 # BEGIN implemented abstract methods from PVE::AbstractConfig
  29
  30 sub guest_type {
  31     return "CT";
  32 }
  33
  34 sub __config_max_unused_disks {
  35     my ($class) = @_;
  36
  37     return $MAX_UNUSED_DISKS;
  38 }
  39
  40 sub config_file_lock {
  41     my ($class, $vmid) = @_;
  42
  43     return "$lockdir/pve-config-${vmid}.lock";
  44 }
  45
  46 sub cfs_config_path {
  47     my ($class, $vmid, $node) = @_;
  48
  49     $node = $nodename if !$node;
  50     return "nodes/$node/lxc/$vmid.conf";
  51 }
  52
  53 sub mountpoint_backup_enabled {
  54     my ($class, $mp_key, $mountpoint) = @_;
  55
  56     my $enabled;
  57     my $reason;
  58
  59     if ($mp_key eq 'rootfs') {
  60         $enabled = 1;
  61         $reason = 'rootfs';
  62     } elsif ($mountpoint->{type} ne 'volume') {
  63         $enabled = 0;
  64         $reason = 'not a volume';
  65     } elsif ($mountpoint->{backup}) {
  66         $enabled = 1;
  67         $reason = 'enabled';
  68     } else {
  69         $enabled = 0;
  70         $reason = 'disabled';
  71     }
  72     return wantarray ? ($enabled, $reason) : $enabled;
  73 }
  74
  75 sub has_feature {
  76     my ($class, $feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
  77     my $err;
  78
  79     my $opts;
  80     if ($feature eq 'copy' || $feature eq 'clone') {
  81         $opts = {'valid_target_formats' => ['raw', 'subvol']};
  82     }
  83
  84     $class->foreach_volume($conf, sub {
  85         my ($ms, $mountpoint) = @_;
  86
  87         return if $err; # skip further test
  88         return if $backup_only && !$class->mountpoint_backup_enabled($ms, $mountpoint);
  89
  90         $err = 1
  91             if !PVE::Storage::volume_has_feature($storecfg, $feature,
  92                                                  $mountpoint->{volume},
  93                                                  $snapname, $running, $opts);
  94     });
  95
  96     return $err ? 0 : 1;
  97 }
  98
  99 sub __snapshot_save_vmstate {
 100     my ($class, $vmid, $conf, $snapname, $storecfg) = @_;
 101     die "implement me - snapshot_save_vmstate\n";
 102 }
 103
 104 sub __snapshot_check_running {
 105     my ($class, $vmid) = @_;
 106     return PVE::LXC::check_running($vmid);
 107 }
 108
 109 sub __snapshot_check_freeze_needed {
 110     my ($class, $vmid, $config, $save_vmstate) = @_;
 111
 112     my $ret = $class->__snapshot_check_running($vmid);
 113     return ($ret, $ret);
 114 }
 115
 116 # implements similar functionality to fsfreeze(8)
 117 sub fsfreeze_mountpoint {
 118     my ($path, $thaw) = @_;
 119
 120     my $op = $thaw ? 'thaw' : 'freeze';
 121     my $ioctl = $thaw ? FITHAW : FIFREEZE;
 122
 123     sysopen my $fd, $path, O_RDONLY or die "failed to open $path: $!\n";
 124     my $ioctl_err;
 125     if (!ioctl($fd, $ioctl, 0)) {
 126         $ioctl_err = "$!";
 127     }
 128     close($fd);
 129     die "fs$op '$path' failed - $ioctl_err\n" if defined $ioctl_err;
 130 }
 131
 132 sub __snapshot_freeze {
 133     my ($class, $vmid, $unfreeze) = @_;
 134
 135     my $conf = $class->load_config($vmid);
 136     my $storagecfg = PVE::Storage::config();
 137
 138     my $freeze_mps = [];
 139     $class->foreach_volume($conf, sub {
 140         my ($ms, $mountpoint) = @_;
 141
 142         return if $mountpoint->{type} ne 'volume';
 143
 144         if (PVE::Storage::volume_snapshot_needs_fsfreeze($storagecfg, $mountpoint->{volume})) {
 145             push @$freeze_mps, $mountpoint->{mp};
 146         }
 147     });
 148
 149     my $freeze_mountpoints = sub {
 150         my ($thaw) = @_;
 151
 152         return if scalar(@$freeze_mps) == 0;
 153
 154         my $pid = PVE::LXC::find_lxc_pid($vmid);
 155
 156         for my $mp (@$freeze_mps) {
 157             eval{ fsfreeze_mountpoint("/proc/${pid}/root/${mp}", $thaw); };
 158             warn $@ if $@;
 159         }
 160     };
 161
 162     if ($unfreeze) {
 163         eval { PVE::LXC::thaw($vmid); };
 164         warn $@ if $@;
 165         $freeze_mountpoints->(1);
 166     } else {
 167         PVE::LXC::freeze($vmid);
 168         PVE::LXC::sync_container_namespace($vmid);
 169         $freeze_mountpoints->(0);
 170     }
 171 }
 172
 173 sub __snapshot_create_vol_snapshot {
 174     my ($class, $vmid, $ms, $mountpoint, $snapname) = @_;
 175
 176     my $storecfg = PVE::Storage::config();
 177
 178     return if $snapname eq 'vzdump' &&
 179         !$class->mountpoint_backup_enabled($ms, $mountpoint);
 180
 181     PVE::Storage::volume_snapshot($storecfg, $mountpoint->{volume}, $snapname);
 182 }
 183
 184 sub __snapshot_delete_remove_drive {
 185     my ($class, $snap, $remove_drive) = @_;
 186
 187     if ($remove_drive eq 'vmstate') {
 188         die "implement me - saving vmstate\n";
 189     } else {
 190         my $value = $snap->{$remove_drive};
 191         my $mountpoint = $class->parse_volume($remove_drive, $value, 1);
 192         delete $snap->{$remove_drive};
 193
 194         $class->add_unused_volume($snap, $mountpoint->{volume})
 195             if ($mountpoint->{type} eq 'volume');
 196     }
 197 }
 198
 199 sub __snapshot_delete_vmstate_file {
 200     my ($class, $snap, $force) = @_;
 201
 202     die "implement me - saving vmstate\n";
 203 }
 204
 205 sub __snapshot_delete_vol_snapshot {
 206     my ($class, $vmid, $ms, $mountpoint, $snapname, $unused) = @_;
 207
 208     return if $snapname eq 'vzdump' &&
 209         !$class->mountpoint_backup_enabled($ms, $mountpoint);
 210
 211     my $storecfg = PVE::Storage::config();
 212     PVE::Storage::volume_snapshot_delete($storecfg, $mountpoint->{volume}, $snapname);
 213     push @$unused, $mountpoint->{volume};
 214 }
 215
 216 sub __snapshot_rollback_vol_possible {
 217     my ($class, $mountpoint, $snapname) = @_;
 218
 219     my $storecfg = PVE::Storage::config();
 220     PVE::Storage::volume_rollback_is_possible($storecfg, $mountpoint->{volume}, $snapname);
 221 }
 222
 223 sub __snapshot_rollback_vol_rollback {
 224     my ($class, $mountpoint, $snapname) = @_;
 225
 226     my $storecfg = PVE::Storage::config();
 227     PVE::Storage::volume_snapshot_rollback($storecfg, $mountpoint->{volume}, $snapname);
 228 }
 229
 230 sub __snapshot_rollback_vm_stop {
 231     my ($class, $vmid) = @_;
 232
 233     PVE::LXC::vm_stop($vmid, 1)
 234         if $class->__snapshot_check_running($vmid);
 235 }
 236
 237 sub __snapshot_rollback_vm_start {
 238     my ($class, $vmid, $vmstate, $data);
 239
 240     die "implement me - save vmstate\n";
 241 }
 242
 243 sub __snapshot_rollback_get_unused {
 244     my ($class, $conf, $snap) = @_;
 245
 246     my $unused = [];
 247
 248     $class->foreach_volume($conf, sub {
 249         my ($vs, $volume) = @_;
 250
 251         return if $volume->{type} ne 'volume';
 252
 253         my $found = 0;
 254         my $volid = $volume->{volume};
 255
 256         $class->foreach_volume($snap, sub {
 257             my ($ms, $mountpoint) = @_;
 258
 259             return if $found;
 260             return if ($mountpoint->{type} ne 'volume');
 261
 262             $found = 1
 263                 if ($mountpoint->{volume} && $mountpoint->{volume} eq $volid);
 264         });
 265
 266         push @$unused, $volid if !$found;
 267     });
 268
 269     return $unused;
 270 }
 271
 272 # END implemented abstract methods from PVE::AbstractConfig
 273
 274 # BEGIN JSON config code
 275
 276 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
 277
 278
 279 my $valid_mount_option_re = qr/(noatime|nodev|nosuid|noexec)/;
 280
 281 sub is_valid_mount_option {
 282     my ($option) = @_;
 283     return $option =~ $valid_mount_option_re;
 284 }
 285
 286 my $rootfs_desc = {
 287     volume => {
 288         type => 'string',
 289         default_key => 1,
 290         format => 'pve-lxc-mp-string',
 291         format_description => 'volume',
 292         description => 'Volume, device or directory to mount into the container.',
 293     },
 294     size => {
 295         type => 'string',
 296         format => 'disk-size',
 297         format_description => 'DiskSize',
 298         description => 'Volume size (read only value).',
 299         optional => 1,
 300     },
 301     acl => {
 302         type => 'boolean',
 303         description => 'Explicitly enable or disable ACL support.',
 304         optional => 1,
 305     },
 306     mountoptions => {
 307         optional => 1,
 308         type => 'string',
 309         description => 'Extra mount options for rootfs/mps.',
 310         format_description => 'opt[;opt...]',
 311         pattern => qr/$valid_mount_option_re(;$valid_mount_option_re)*/,
 312     },
 313     ro => {
 314         type => 'boolean',
 315         description => 'Read-only mount point',
 316         optional => 1,
 317     },
 318     quota => {
 319         type => 'boolean',
 320         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
 321         optional => 1,
 322     },
 323     replicate => {
 324         type => 'boolean',
 325         description => 'Will include this volume to a storage replica job.',
 326         optional => 1,
 327         default => 1,
 328     },
 329     shared => {
 330         type => 'boolean',
 331         description => 'Mark this non-volume mount point as available on multiple nodes (see \'nodes\')',
 332         verbose_description => "Mark this non-volume mount point as available on all nodes.\n\nWARNING: This option does not share the mount point automatically, it assumes it is shared already!",
 333         optional => 1,
 334         default => 0,
 335     },
 336 };
 337
 338 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
 339     type => 'string', format => $rootfs_desc,
 340     description => "Use volume as container root.",
 341     optional => 1,
 342 });
 343
 344 # IP address with optional interface suffix for link local ipv6 addresses
 345 PVE::JSONSchema::register_format('lxc-ip-with-ll-iface', \&verify_ip_with_ll_iface);
 346 sub verify_ip_with_ll_iface {
 347     my ($addr, $noerr) = @_;
 348
 349     if (my ($addr, $iface) = ($addr =~ /^(fe80:[^%]+)%(.*)$/)) {
 350         if (PVE::JSONSchema::pve_verify_ip($addr, 1)
 351             && PVE::JSONSchema::pve_verify_iface($iface, 1))
 352         {
 353             return $addr;
 354         }
 355     }
 356
 357     return PVE::JSONSchema::pve_verify_ip($addr, $noerr);
 358 }
 359
 360
 361 my $features_desc = {
 362     mount => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Allow mounting file systems of specific types."
 366             ." This should be a list of file system types as used with the mount command."
 367             ." Note that this can have negative effects on the container's security."
 368             ." With access to a loop device, mounting a file can circumvent the mknod"
 369             ." permission of the devices cgroup, mounting an NFS file system can"
 370             ." block the host's I/O completely and prevent it from rebooting, etc.",
 371         format_description => 'fstype;fstype;...',
 372         pattern => qr/[a-zA-Z0-9_; ]+/,
 373     },
 374     nesting => {
 375         optional => 1,
 376         type => 'boolean',
 377         default => 0,
 378         description => "Allow nesting."
 379             ." Best used with unprivileged containers with additional id mapping."
 380             ." Note that this will expose procfs and sysfs contents of the host"
 381             ." to the guest.",
 382     },
 383     keyctl => {
 384         optional => 1,
 385         type => 'boolean',
 386         default => 0,
 387         description => "For unprivileged containers only: Allow the use of the keyctl() system call."
 388             ." This is required to use docker inside a container."
 389             ." By default unprivileged containers will see this system call as non-existent."
 390             ." This is mostly a workaround for systemd-networkd, as it will treat it as a fatal"
 391             ." error when some keyctl() operations are denied by the kernel due to lacking permissions."
 392             ." Essentially, you can choose between running systemd-networkd or docker.",
 393     },
 394     fuse => {
 395         optional => 1,
 396         type => 'boolean',
 397         default => 0,
 398         description => "Allow using 'fuse' file systems in a container."
 399             ." Note that interactions between fuse and the freezer cgroup can potentially cause I/O deadlocks.",
 400     },
 401     mknod => {
 402         optional => 1,
 403         type => 'boolean',
 404         default => 0,
 405         description => "Allow unprivileged containers to use mknod() to add certain device nodes."
 406             ." This requires a kernel with seccomp trap to user space support (5.3 or newer)."
 407             ." This is experimental.",
 408     },
 409     force_rw_sys => {
 410         optional => 1,
 411         type => 'boolean',
 412         default => 0,
 413         description => "Mount /sys in unprivileged containers as `rw` instead of `mixed`."
 414             ." This can break networking under newer (>= v245) systemd-network use."
 415     },
 416 };
 417
 418 my $confdesc = {
 419     lock => {
 420         optional => 1,
 421         type => 'string',
 422         description => "Lock/unlock the VM.",
 423         enum => [qw(backup create destroyed disk fstrim migrate mounted rollback snapshot snapshot-delete)],
 424     },
 425     onboot => {
 426         optional => 1,
 427         type => 'boolean',
 428         description => "Specifies whether a VM will be started during system bootup.",
 429         default => 0,
 430     },
 431     startup => get_standard_option('pve-startup-order'),
 432     template => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable Template.",
 436         default => 0,
 437     },
 438     arch => {
 439         optional => 1,
 440         type => 'string',
 441         enum => ['amd64', 'i386', 'arm64', 'armhf'],
 442         description => "OS architecture type.",
 443         default => 'amd64',
 444     },
 445     ostype => {
 446         optional => 1,
 447         type => 'string',
 448         enum => [qw(debian devuan ubuntu centos fedora opensuse archlinux alpine gentoo unmanaged)],
 449         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 450     },
 451     console => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Attach a console device (/dev/console) to the container.",
 455         default => 1,
 456     },
 457     tty => {
 458         optional => 1,
 459         type => 'integer',
 460         description => "Specify the number of tty available to the container",
 461         minimum => 0,
 462         maximum => 6,
 463         default => 2,
 464     },
 465     cores => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "The number of cores assigned to the container. A container can use all available cores by default.",
 469         minimum => 1,
 470         maximum => 8192,
 471     },
 472     cpulimit => {
 473         optional => 1,
 474         type => 'number',
 475         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 476         minimum => 0,
 477         maximum => 8192,
 478         default => 0,
 479     },
 480     cpuunits => {
 481         optional => 1,
 482         type => 'integer',
 483         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to the weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 484         minimum => 0,
 485         maximum => 500000,
 486         default => 1024,
 487     },
 488     memory => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Amount of RAM for the VM in MB.",
 492         minimum => 16,
 493         default => 512,
 494     },
 495     swap => {
 496         optional => 1,
 497         type => 'integer',
 498         description => "Amount of SWAP for the VM in MB.",
 499         minimum => 0,
 500         default => 512,
 501     },
 502     hostname => {
 503         optional => 1,
 504         description => "Set a host name for the container.",
 505         type => 'string', format => 'dns-name',
 506         maxLength => 255,
 507     },
 508     description => {
 509         optional => 1,
 510         type => 'string',
 511         description => "Container description. Only used on the configuration web interface.",
 512     },
 513     searchdomain => {
 514         optional => 1,
 515         type => 'string', format => 'dns-name-list',
 516         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 517     },
 518     nameserver => {
 519         optional => 1,
 520         type => 'string', format => 'lxc-ip-with-ll-iface-list',
 521         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 522     },
 523     timezone => {
 524         optional => 1,
 525         type => 'string', format => 'pve-ct-timezone',
 526         description => "Time zone to use in the container. If option isn't set, then nothing will be done. Can be set to 'host' to match the host time zone, or an arbitrary time zone option from /usr/share/zoneinfo/zone.tab",
 527     },
 528     rootfs => get_standard_option('pve-ct-rootfs'),
 529     parent => {
 530         optional => 1,
 531         type => 'string', format => 'pve-configid',
 532         maxLength => 40,
 533         description => "Parent snapshot name. This is used internally, and should not be modified.",
 534     },
 535     snaptime => {
 536         optional => 1,
 537         description => "Timestamp for snapshots.",
 538         type => 'integer',
 539         minimum => 0,
 540     },
 541     cmode => {
 542         optional => 1,
 543         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 544         type => 'string',
 545         enum => ['shell', 'console', 'tty'],
 546         default => 'tty',
 547     },
 548     protection => {
 549         optional => 1,
 550         type => 'boolean',
 551         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 552         default => 0,
 553     },
 554     unprivileged => {
 555         optional => 1,
 556         type => 'boolean',
 557         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 558         default => 0,
 559     },
 560     features => {
 561         optional => 1,
 562         type => 'string',
 563         format => $features_desc,
 564         description => "Allow containers access to advanced features.",
 565     },
 566     hookscript => {
 567         optional => 1,
 568         type => 'string',
 569         format => 'pve-volume-id',
 570         description => 'Script that will be exectued during various steps in the containers lifetime.',
 571     },
 572     tags => {
 573         type => 'string', format => 'pve-tag-list',
 574         description => 'Tags of the Container. This is only meta information.',
 575         optional => 1,
 576     },
 577     debug => {
 578         optional => 1,
 579         type => 'boolean',
 580         description => "Try to be more verbose. For now this only enables debug log-level on start.",
 581         default => 0,
 582     },
 583 };
 584
 585 my $valid_lxc_conf_keys = {
 586     'lxc.apparmor.profile' => 1,
 587     'lxc.apparmor.allow_incomplete' => 1,
 588     'lxc.apparmor.allow_nesting' => 1,
 589     'lxc.apparmor.raw' => 1,
 590     'lxc.selinux.context' => 1,
 591     'lxc.include' => 1,
 592     'lxc.arch' => 1,
 593     'lxc.uts.name' => 1,
 594     'lxc.signal.halt' => 1,
 595     'lxc.signal.reboot' => 1,
 596     'lxc.signal.stop' => 1,
 597     'lxc.init.cmd' => 1,
 598     'lxc.pty.max' => 1,
 599     'lxc.console.logfile' => 1,
 600     'lxc.console.path' => 1,
 601     'lxc.tty.max' => 1,
 602     'lxc.devtty.dir' => 1,
 603     'lxc.hook.autodev' => 1,
 604     'lxc.autodev' => 1,
 605     'lxc.kmsg' => 1,
 606     'lxc.mount.fstab' => 1,
 607     'lxc.mount.entry' => 1,
 608     'lxc.mount.auto' => 1,
 609     'lxc.rootfs.path' => 'lxc.rootfs.path is auto generated from rootfs',
 610     'lxc.rootfs.mount' => 1,
 611     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 612                             ', please use mount point options in the "rootfs" key',
 613     # lxc.cgroup.*
 614     # lxc.prlimit.*
 615     # lxc.net.*
 616     'lxc.cap.drop' => 1,
 617     'lxc.cap.keep' => 1,
 618     'lxc.seccomp.profile' => 1,
 619     'lxc.seccomp.notify.proxy' => 1,
 620     'lxc.seccomp.notify.cookie' => 1,
 621     'lxc.idmap' => 1,
 622     'lxc.hook.pre-start' => 1,
 623     'lxc.hook.pre-mount' => 1,
 624     'lxc.hook.mount' => 1,
 625     'lxc.hook.start' => 1,
 626     'lxc.hook.stop' => 1,
 627     'lxc.hook.post-stop' => 1,
 628     'lxc.hook.clone' => 1,
 629     'lxc.hook.destroy' => 1,
 630     'lxc.hook.version' => 1,
 631     'lxc.log.level' => 1,
 632     'lxc.log.file' => 1,
 633     'lxc.start.auto' => 1,
 634     'lxc.start.delay' => 1,
 635     'lxc.start.order' => 1,
 636     'lxc.group' => 1,
 637     'lxc.environment' => 1,
 638
 639     # All these are namespaced via CLONE_NEWIPC (see namespaces(7)).
 640     'lxc.sysctl.fs.mqueue' => 1,
 641     'lxc.sysctl.kernel.msgmax' => 1,
 642     'lxc.sysctl.kernel.msgmnb' => 1,
 643     'lxc.sysctl.kernel.msgmni' => 1,
 644     'lxc.sysctl.kernel.sem' => 1,
 645     'lxc.sysctl.kernel.shmall' => 1,
 646     'lxc.sysctl.kernel.shmmax' => 1,
 647     'lxc.sysctl.kernel.shmmni' => 1,
 648     'lxc.sysctl.kernel.shm_rmid_forced' => 1,
 649 };
 650
 651 my $deprecated_lxc_conf_keys = {
 652     # Deprecated (removed with lxc 3.0):
 653     'lxc.aa_profile'           => 'lxc.apparmor.profile',
 654     'lxc.aa_allow_incomplete'  => 'lxc.apparmor.allow_incomplete',
 655     'lxc.console'              => 'lxc.console.path',
 656     'lxc.devttydir'            => 'lxc.tty.dir',
 657     'lxc.haltsignal'           => 'lxc.signal.halt',
 658     'lxc.rebootsignal'         => 'lxc.signal.reboot',
 659     'lxc.stopsignal'           => 'lxc.signal.stop',
 660     'lxc.id_map'               => 'lxc.idmap',
 661     'lxc.init_cmd'             => 'lxc.init.cmd',
 662     'lxc.loglevel'             => 'lxc.log.level',
 663     'lxc.logfile'              => 'lxc.log.file',
 664     'lxc.mount'                => 'lxc.mount.fstab',
 665     'lxc.network.type'         => 'lxc.net.INDEX.type',
 666     'lxc.network.flags'        => 'lxc.net.INDEX.flags',
 667     'lxc.network.link'         => 'lxc.net.INDEX.link',
 668     'lxc.network.mtu'          => 'lxc.net.INDEX.mtu',
 669     'lxc.network.name'         => 'lxc.net.INDEX.name',
 670     'lxc.network.hwaddr'       => 'lxc.net.INDEX.hwaddr',
 671     'lxc.network.ipv4'         => 'lxc.net.INDEX.ipv4.address',
 672     'lxc.network.ipv4.gateway' => 'lxc.net.INDEX.ipv4.gateway',
 673     'lxc.network.ipv6'         => 'lxc.net.INDEX.ipv6.address',
 674     'lxc.network.ipv6.gateway' => 'lxc.net.INDEX.ipv6.gateway',
 675     'lxc.network.script.up'    => 'lxc.net.INDEX.script.up',
 676     'lxc.network.script.down'  => 'lxc.net.INDEX.script.down',
 677     'lxc.pts'                  => 'lxc.pty.max',
 678     'lxc.se_context'           => 'lxc.selinux.context',
 679     'lxc.seccomp'              => 'lxc.seccomp.profile',
 680     'lxc.tty'                  => 'lxc.tty.max',
 681     'lxc.utsname'              => 'lxc.uts.name',
 682 };
 683
 684 sub is_valid_lxc_conf_key {
 685     my ($vmid, $key) = @_;
 686     if ($key =~ /^lxc\.limit\./) {
 687         warn "vm $vmid - $key: lxc.limit.* was renamed to lxc.prlimit.*\n";
 688         return 1;
 689     }
 690     if (defined(my $new_name = $deprecated_lxc_conf_keys->{$key})) {
 691         warn "vm $vmid - $key is deprecated and was renamed to $new_name\n";
 692         return 1;
 693     }
 694     my $validity = $valid_lxc_conf_keys->{$key};
 695     return $validity if defined($validity);
 696     return 1 if $key =~ /^lxc\.cgroup2?\./  # allow all cgroup values
 697              || $key =~ /^lxc\.prlimit\./ # allow all prlimits
 698              || $key =~ /^lxc\.net\./;    # allow custom network definitions
 699     return 0;
 700 }
 701
 702 our $netconf_desc = {
 703     type => {
 704         type => 'string',
 705         optional => 1,
 706         description => "Network interface type.",
 707         enum => [qw(veth)],
 708     },
 709     name => {
 710         type => 'string',
 711         format_description => 'string',
 712         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 713         pattern => '[-_.\w\d]+',
 714     },
 715     bridge => {
 716         type => 'string',
 717         format_description => 'bridge',
 718         description => 'Bridge to attach the network device to.',
 719         pattern => '[-_.\w\d]+',
 720         optional => 1,
 721     },
 722     hwaddr => get_standard_option('mac-addr', {
 723         description => 'The interface MAC address. This is dynamically allocated by default, but you can set that statically if needed, for example to always have the same link-local IPv6 address. (lxc.network.hwaddr)',
 724         }),
 725     mtu => {
 726         type => 'integer',
 727         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 728         minimum => 64, # minimum ethernet frame is 64 bytes
 729         optional => 1,
 730     },
 731     ip => {
 732         type => 'string',
 733         format => 'pve-ipv4-config',
 734         format_description => '(IPv4/CIDR|dhcp|manual)',
 735         description => 'IPv4 address in CIDR format.',
 736         optional => 1,
 737     },
 738     gw => {
 739         type => 'string',
 740         format => 'ipv4',
 741         format_description => 'GatewayIPv4',
 742         description => 'Default gateway for IPv4 traffic.',
 743         optional => 1,
 744     },
 745     ip6 => {
 746         type => 'string',
 747         format => 'pve-ipv6-config',
 748         format_description => '(IPv6/CIDR|auto|dhcp|manual)',
 749         description => 'IPv6 address in CIDR format.',
 750         optional => 1,
 751     },
 752     gw6 => {
 753         type => 'string',
 754         format => 'ipv6',
 755         format_description => 'GatewayIPv6',
 756         description => 'Default gateway for IPv6 traffic.',
 757         optional => 1,
 758     },
 759     firewall => {
 760         type => 'boolean',
 761         description => "Controls whether this interface's firewall rules should be used.",
 762         optional => 1,
 763     },
 764     tag => {
 765         type => 'integer',
 766         minimum => 1,
 767         maximum => 4094,
 768         description => "VLAN tag for this interface.",
 769         optional => 1,
 770     },
 771     trunks => {
 772         type => 'string',
 773         pattern => qr/\d+(?:;\d+)*/,
 774         format_description => 'vlanid[;vlanid...]',
 775         description => "VLAN ids to pass through the interface",
 776         optional => 1,
 777     },
 778     rate => {
 779         type => 'number',
 780         format_description => 'mbps',
 781         description => "Apply rate limiting to the interface",
 782         optional => 1,
 783     },
 784 };
 785 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 786
 787 my $MAX_LXC_NETWORKS = 32;
 788 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 789     $confdesc->{"net$i"} = {
 790         optional => 1,
 791         type => 'string', format => $netconf_desc,
 792         description => "Specifies network interfaces for the container.",
 793     };
 794 }
 795
 796 PVE::JSONSchema::register_format('pve-ct-timezone', \&verify_ct_timezone);
 797 sub verify_ct_timezone {
 798     my ($timezone, $noerr) = @_;
 799
 800     return if $timezone eq 'host'; # using host settings
 801
 802     PVE::JSONSchema::pve_verify_timezone($timezone);
 803 }
 804
 805 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 806 sub verify_lxc_mp_string {
 807     my ($mp, $noerr) = @_;
 808
 809     # do not allow:
 810     # /./ or /../
 811     # /. or /.. at the end
 812     # ../ at the beginning
 813
 814     if($mp =~ m@/\.\.?/@ ||
 815        $mp =~ m@/\.\.?$@ ||
 816        $mp =~ m@^\.\./@) {
 817         return undef if $noerr;
 818         die "$mp contains illegal character sequences\n";
 819     }
 820     return $mp;
 821 }
 822
 823 my $mp_desc = {
 824     %$rootfs_desc,
 825     backup => {
 826         type => 'boolean',
 827         description => 'Whether to include the mount point in backups.',
 828         verbose_description => 'Whether to include the mount point in backups '.
 829                                '(only used for volume mount points).',
 830         optional => 1,
 831     },
 832     mp => {
 833         type => 'string',
 834         format => 'pve-lxc-mp-string',
 835         format_description => 'Path',
 836         description => 'Path to the mount point as seen from inside the container '.
 837                        '(must not contain symlinks).',
 838         verbose_description => "Path to the mount point as seen from inside the container.\n\n".
 839                                "NOTE: Must not contain any symlinks for security reasons."
 840     },
 841 };
 842 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 843
 844 my $unused_desc = {
 845     volume => {
 846         type => 'string',
 847         default_key => 1,
 848         format => 'pve-volume-id',
 849         format_description => 'volume',
 850         description => 'The volume that is not used currently.',
 851     }
 852 };
 853
 854 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 855     $confdesc->{"mp$i"} = {
 856         optional => 1,
 857         type => 'string', format => $mp_desc,
 858         description => "Use volume as container mount point.",
 859         optional => 1,
 860     };
 861 }
 862
 863 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
 864     $confdesc->{"unused$i"} = {
 865         optional => 1,
 866         type => 'string', format => $unused_desc,
 867         description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
 868     }
 869 }
 870
 871 sub parse_pct_config {
 872     my ($filename, $raw) = @_;
 873
 874     return undef if !defined($raw);
 875
 876     my $res = {
 877         digest => Digest::SHA::sha1_hex($raw),
 878         snapshots => {},
 879         pending => {},
 880     };
 881
 882     $filename =~ m|/lxc/(\d+).conf$|
 883         || die "got strange filename '$filename'";
 884
 885     my $vmid = $1;
 886
 887     my $conf = $res;
 888     my $descr = '';
 889     my $section = '';
 890
 891     my @lines = split(/\n/, $raw);
 892     foreach my $line (@lines) {
 893         next if $line =~ m/^\s*$/;
 894
 895         if ($line =~ m/^\[pve:pending\]\s*$/i) {
 896             $section = 'pending';
 897             $conf->{description} = $descr if $descr;
 898             $descr = '';
 899             $conf = $res->{$section} = {};
 900             next;
 901         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 902             $section = $1;
 903             $conf->{description} = $descr if $descr;
 904             $descr = '';
 905             $conf = $res->{snapshots}->{$section} = {};
 906             next;
 907         }
 908
 909         if ($line =~ m/^\#(.*)\s*$/) {
 910             $descr .= PVE::Tools::decode_text($1) . "\n";
 911             next;
 912         }
 913
 914         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
 915             my $key = $1;
 916             my $value = $3;
 917             my $validity = is_valid_lxc_conf_key($vmid, $key);
 918             if ($validity eq 1) {
 919                 push @{$conf->{lxc}}, [$key, $value];
 920             } elsif (my $errmsg = $validity) {
 921                 warn "vm $vmid - $key: $errmsg\n";
 922             } else {
 923                 warn "vm $vmid - unable to parse config: $line\n";
 924             }
 925         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
 926             $descr .= PVE::Tools::decode_text($2);
 927         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 928             $conf->{snapstate} = $1;
 929         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
 930             my $value = $1;
 931             if ($section eq 'pending') {
 932                 $conf->{delete} = $value;
 933             } else {
 934                 warn "vm $vmid - property 'delete' is only allowed in [pve:pending]\n";
 935             }
 936         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S.*)\s*$/) {
 937             my $key = $1;
 938             my $value = $2;
 939             eval { $value = PVE::LXC::Config->check_type($key, $value); };
 940             warn "vm $vmid - unable to parse value of '$key' - $@" if $@;
 941             $conf->{$key} = $value;
 942         } else {
 943             warn "vm $vmid - unable to parse config: $line\n";
 944         }
 945     }
 946
 947     $conf->{description} = $descr if $descr;
 948
 949     delete $res->{snapstate}; # just to be sure
 950
 951     return $res;
 952 }
 953
 954 sub write_pct_config {
 955     my ($filename, $conf) = @_;
 956
 957     delete $conf->{snapstate}; # just to be sure
 958
 959     my $volidlist = PVE::LXC::Config->get_vm_volumes($conf);
 960     my $used_volids = {};
 961     foreach my $vid (@$volidlist) {
 962         $used_volids->{$vid} = 1;
 963     }
 964
 965     # remove 'unusedX' settings if the volume is still used
 966     foreach my $key (keys %$conf) {
 967         my $value = $conf->{$key};
 968         if ($key =~ m/^unused/ && $used_volids->{$value}) {
 969             delete $conf->{$key};
 970         }
 971     }
 972
 973     my $generate_raw_config = sub {
 974         my ($conf) = @_;
 975
 976         my $raw = '';
 977
 978         # add description as comment to top of file
 979         my $descr = $conf->{description} || '';
 980         foreach my $cl (split(/\n/, $descr)) {
 981             $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
 982         }
 983
 984         foreach my $key (sort keys %$conf) {
 985             next if $key eq 'digest' || $key eq 'description' ||
 986                     $key eq 'pending' || $key eq 'snapshots' ||
 987                     $key eq 'snapname' || $key eq 'lxc';
 988             my $value = $conf->{$key};
 989             die "detected invalid newline inside property '$key'\n"
 990                 if $value =~ m/\n/;
 991             $raw .= "$key: $value\n";
 992         }
 993
 994         if (my $lxcconf = $conf->{lxc}) {
 995             foreach my $entry (@$lxcconf) {
 996                 my ($k, $v) = @$entry;
 997                 $raw .= "$k: $v\n";
 998             }
 999         }
1000
1001         return $raw;
1002     };
1003
1004     my $raw = &$generate_raw_config($conf);
1005
1006     if (scalar(keys %{$conf->{pending}})){
1007         $raw .= "\n[pve:pending]\n";
1008         $raw .= &$generate_raw_config($conf->{pending});
1009     }
1010
1011     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
1012         $raw .= "\n[$snapname]\n";
1013         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1014     }
1015
1016     return $raw;
1017 }
1018
1019 sub update_pct_config {
1020     my ($class, $vmid, $conf, $running, $param, $delete, $revert) = @_;
1021
1022     my $storage_cfg = PVE::Storage::config();
1023
1024     foreach my $opt (@$revert) {
1025         delete $conf->{pending}->{$opt};
1026         $class->remove_from_pending_delete($conf, $opt); # also remove from deletion queue
1027     }
1028
1029     # write updates to pending section
1030     my $modified = {}; # record modified options
1031
1032     foreach my $opt (@$delete) {
1033         if (!defined($conf->{$opt}) && !defined($conf->{pending}->{$opt})) {
1034             warn "cannot delete '$opt' - not set in current configuration!\n";
1035             next;
1036         }
1037         $modified->{$opt} = 1;
1038         if ($opt eq 'memory' || $opt eq 'rootfs' || $opt eq 'ostype') {
1039             die "unable to delete required option '$opt'\n";
1040         } elsif ($opt =~ m/^unused(\d+)$/) {
1041             $class->check_protection($conf, "can't remove CT $vmid drive '$opt'");
1042         } elsif ($opt =~ m/^mp(\d+)$/) {
1043             $class->check_protection($conf, "can't remove CT $vmid drive '$opt'");
1044         } elsif ($opt eq 'unprivileged') {
1045             die "unable to delete read-only option: '$opt'\n";
1046         }
1047         $class->add_to_pending_delete($conf, $opt);
1048     }
1049
1050     my $check_content_type = sub {
1051         my ($mp) = @_;
1052         my $sid = PVE::Storage::parse_volume_id($mp->{volume});
1053         my $storage_config = PVE::Storage::storage_config($storage_cfg, $sid);
1054         die "storage '$sid' does not allow content type 'rootdir' (Container)\n"
1055             if !$storage_config->{content}->{rootdir};
1056     };
1057
1058     foreach my $opt (sort keys %$param) { # add/change
1059         $modified->{$opt} = 1;
1060         my $value = $param->{$opt};
1061         if ($opt =~ m/^mp(\d+)$/ || $opt eq 'rootfs') {
1062             $class->check_protection($conf, "can't update CT $vmid drive '$opt'");
1063             my $mp = $class->parse_volume($opt, $value);
1064             $check_content_type->($mp) if ($mp->{type} eq 'volume');
1065         } elsif ($opt eq 'hookscript') {
1066             PVE::GuestHelpers::check_hookscript($value);
1067         } elsif ($opt eq 'nameserver') {
1068             $value = PVE::LXC::verify_nameserver_list($value);
1069         } elsif ($opt eq 'searchdomain') {
1070             $value = PVE::LXC::verify_searchdomain_list($value);
1071         } elsif ($opt eq 'unprivileged') {
1072             die "unable to modify read-only option: '$opt'\n";
1073         }
1074         $conf->{pending}->{$opt} = $value;
1075         $class->remove_from_pending_delete($conf, $opt);
1076     }
1077
1078     my $changes = $class->cleanup_pending($conf);
1079
1080     my $errors = {};
1081     if ($running) {
1082         $class->vmconfig_hotplug_pending($vmid, $conf, $storage_cfg, $modified, $errors);
1083     } else {
1084         $class->vmconfig_apply_pending($vmid, $conf, $storage_cfg, $modified, $errors);
1085     }
1086
1087     return $errors;
1088 }
1089
1090 sub check_type {
1091     my ($class, $key, $value) = @_;
1092
1093     die "unknown setting '$key'\n" if !$confdesc->{$key};
1094
1095     my $type = $confdesc->{$key}->{type};
1096
1097     if (!defined($value)) {
1098         die "got undefined value\n";
1099     }
1100
1101     if ($value =~ m/[\n\r]/) {
1102         die "property contains a line feed\n";
1103     }
1104
1105     if ($type eq 'boolean') {
1106         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1107         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1108         die "type check ('boolean') failed - got '$value'\n";
1109     } elsif ($type eq 'integer') {
1110         return int($1) if $value =~ m/^(\d+)$/;
1111         die "type check ('integer') failed - got '$value'\n";
1112     } elsif ($type eq 'number') {
1113         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1114         die "type check ('number') failed - got '$value'\n";
1115     } elsif ($type eq 'string') {
1116         if (my $fmt = $confdesc->{$key}->{format}) {
1117             PVE::JSONSchema::check_format($fmt, $value);
1118             return $value;
1119         }
1120         return $value;
1121     } else {
1122         die "internal error"
1123     }
1124 }
1125
1126
1127 # add JSON properties for create and set function
1128 sub json_config_properties {
1129     my ($class, $prop) = @_;
1130
1131     foreach my $opt (keys %$confdesc) {
1132         next if $opt eq 'parent' || $opt eq 'snaptime';
1133         next if $prop->{$opt};
1134         $prop->{$opt} = $confdesc->{$opt};
1135     }
1136
1137     return $prop;
1138 }
1139
1140 my $parse_ct_mountpoint_full = sub {
1141     my ($class, $desc, $data, $noerr) = @_;
1142
1143     $data //= '';
1144
1145     my $res;
1146     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
1147     if ($@) {
1148         return undef if $noerr;
1149         die $@;
1150     }
1151
1152     if (defined(my $size = $res->{size})) {
1153         $size = PVE::JSONSchema::parse_size($size);
1154         if (!defined($size)) {
1155             return undef if $noerr;
1156             die "invalid size: $size\n";
1157         }
1158         $res->{size} = $size;
1159     }
1160
1161     $res->{type} = $class->classify_mountpoint($res->{volume});
1162
1163     return $res;
1164 };
1165
1166 sub print_ct_mountpoint {
1167     my ($class, $info, $nomp) = @_;
1168     my $skip = [ 'type' ];
1169     push @$skip, 'mp' if $nomp;
1170     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
1171 }
1172
1173 sub parse_volume {
1174     my ($class, $key, $volume_string, $noerr) = @_;
1175
1176     if ($key eq 'rootfs') {
1177         my $res =  $parse_ct_mountpoint_full->($class, $rootfs_desc, $volume_string, $noerr);
1178         $res->{mp} = '/' if defined($res);
1179         return $res;
1180     } elsif ($key =~ m/^mp\d+$/) {
1181         return $parse_ct_mountpoint_full->($class, $mp_desc, $volume_string, $noerr);
1182     } elsif ($key =~ m/^unused\d+$/) {
1183         return $parse_ct_mountpoint_full->($class, $unused_desc, $volume_string, $noerr);
1184     }
1185
1186     die "parse_volume - unknown type: $key\n";
1187 }
1188
1189 sub print_volume {
1190     my ($class, $key, $volume) = @_;
1191
1192     return $class->print_ct_mountpoint($volume, $key eq 'rootfs');
1193 }
1194
1195 sub volid_key {
1196     my ($class) = @_;
1197
1198     return 'volume';
1199 }
1200
1201 sub print_lxc_network {
1202     my ($class, $net) = @_;
1203     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
1204 }
1205
1206 sub parse_lxc_network {
1207     my ($class, $data) = @_;
1208
1209     my $res = {};
1210
1211     return $res if !$data;
1212
1213     $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
1214
1215     $res->{type} = 'veth';
1216     if (!$res->{hwaddr}) {
1217         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1218         $res->{hwaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1219     }
1220
1221     return $res;
1222 }
1223
1224 sub parse_features {
1225     my ($class, $data) = @_;
1226     return {} if !$data;
1227     return PVE::JSONSchema::parse_property_string($features_desc, $data);
1228 }
1229
1230 sub option_exists {
1231     my ($class, $name) = @_;
1232
1233     return defined($confdesc->{$name});
1234 }
1235 # END JSON config code
1236
1237 my $LXC_FASTPLUG_OPTIONS= {
1238     'description' => 1,
1239     'onboot' => 1,
1240     'startup' => 1,
1241     'protection' => 1,
1242     'hostname' => 1,
1243     'hookscript' => 1,
1244     'cores' => 1,
1245     'tags' => 1,
1246     'lock' => 1,
1247 };
1248
1249 sub vmconfig_hotplug_pending {
1250     my ($class, $vmid, $conf, $storecfg, $selection, $errors) = @_;
1251
1252     my $pid = PVE::LXC::find_lxc_pid($vmid);
1253     my $rootdir = "/proc/$pid/root";
1254
1255     my $add_hotplug_error = sub {
1256         my ($opt, $msg) = @_;
1257         $errors->{$opt} = "unable to hotplug $opt: $msg";
1258     };
1259
1260     foreach my $opt (sort keys %{$conf->{pending}}) { # add/change
1261         next if $selection && !$selection->{$opt};
1262         if ($LXC_FASTPLUG_OPTIONS->{$opt}) {
1263             $conf->{$opt} = delete $conf->{pending}->{$opt};
1264         }
1265     }
1266
1267     my $cgroup = PVE::LXC::CGroup->new($vmid);
1268
1269     # There's no separate swap size to configure, there's memory and "total"
1270     # memory (iow. memory+swap). This means we have to change them together.
1271     my $hotplug_memory_done;
1272     my $hotplug_memory = sub {
1273         my ($wanted_memory, $wanted_swap) = @_;
1274
1275         $wanted_memory = int($wanted_memory * 1024 * 1024) if defined($wanted_memory);
1276         $wanted_swap = int($wanted_swap * 1024 * 1024) if defined($wanted_swap);
1277         $cgroup->change_memory_limit($wanted_memory, $wanted_swap);
1278
1279         $hotplug_memory_done = 1;
1280     };
1281
1282     my $pending_delete_hash = $class->parse_pending_delete($conf->{pending}->{delete});
1283     # FIXME: $force deletion is not implemented for CTs
1284     foreach my $opt (sort keys %$pending_delete_hash) {
1285         next if $selection && !$selection->{$opt};
1286         eval {
1287             if ($LXC_FASTPLUG_OPTIONS->{$opt}) {
1288                 # pass
1289             } elsif ($opt =~ m/^unused(\d+)$/) {
1290                 PVE::LXC::delete_mountpoint_volume($storecfg, $vmid, $conf->{$opt})
1291                     if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1292             } elsif ($opt eq 'swap') {
1293                 $hotplug_memory->(undef, 0);
1294             } elsif ($opt eq 'cpulimit') {
1295                 $cgroup->change_cpu_quota(-1, 100000);
1296             } elsif ($opt eq 'cpuunits') {
1297                 $cgroup->change_cpu_shares(undef, $confdesc->{cpuunits}->{default});
1298             } elsif ($opt =~ m/^net(\d)$/) {
1299                 my $netid = $1;
1300                 PVE::Network::veth_delete("veth${vmid}i$netid");
1301             } else {
1302                 die "skip\n"; # skip non-hotpluggable opts
1303             }
1304         };
1305         if (my $err = $@) {
1306             $add_hotplug_error->($opt, $err) if $err ne "skip\n";
1307         } else {
1308             delete $conf->{$opt};
1309             $class->remove_from_pending_delete($conf, $opt);
1310         }
1311     }
1312
1313     foreach my $opt (sort keys %{$conf->{pending}}) {
1314         next if $opt eq 'delete'; # just to be sure
1315         next if $selection && !$selection->{$opt};
1316         my $value = $conf->{pending}->{$opt};
1317         eval {
1318             if ($opt eq 'cpulimit') {
1319                 my $quota = 100000 * $value;
1320                 $cgroup->change_cpu_quota(int(100000 * $value), 100000);
1321             } elsif ($opt eq 'cpuunits') {
1322                 $cgroup->change_cpu_shares($value, $confdesc->{cpuunits}->{default});
1323             } elsif ($opt =~ m/^net(\d+)$/) {
1324                 my $netid = $1;
1325                 my $net = $class->parse_lxc_network($value);
1326                 $value = $class->print_lxc_network($net);
1327                 PVE::LXC::update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1328             } elsif ($opt eq 'memory' || $opt eq 'swap') {
1329                 if (!$hotplug_memory_done) { # don't call twice if both opts are passed
1330                     $hotplug_memory->($conf->{pending}->{memory}, $conf->{pending}->{swap});
1331                 }
1332             } elsif ($opt =~ m/^mp(\d+)$/) {
1333                 if (!PVE::LXC::Tools::can_use_new_mount_api()) {
1334                     die "skip\n";
1335                 }
1336
1337                 if (exists($conf->{$opt})) {
1338                     die "skip\n"; # don't try to hotplug over existing mp
1339                 }
1340
1341                 $class->apply_pending_mountpoint($vmid, $conf, $opt, $storecfg, 1);
1342                 # apply_pending_mountpoint modifies the value if it creates a new disk
1343                 $value = $conf->{pending}->{$opt};
1344             } else {
1345                 die "skip\n"; # skip non-hotpluggable
1346             }
1347         };
1348         if (my $err = $@) {
1349             $add_hotplug_error->($opt, $err) if $err ne "skip\n";
1350         } else {
1351             $conf->{$opt} = $value;
1352             delete $conf->{pending}->{$opt};
1353         }
1354     }
1355 }
1356
1357 sub vmconfig_apply_pending {
1358     my ($class, $vmid, $conf, $storecfg, $selection, $errors) = @_;
1359
1360     my $add_apply_error = sub {
1361         my ($opt, $msg) = @_;
1362         my $err_msg = "unable to apply pending change $opt : $msg";
1363         $errors->{$opt} = $err_msg;
1364         warn $err_msg;
1365     };
1366
1367     my $pending_delete_hash = $class->parse_pending_delete($conf->{pending}->{delete});
1368     # FIXME: $force deletion is not implemented for CTs
1369     foreach my $opt (sort keys %$pending_delete_hash) {
1370         next if $selection && !$selection->{$opt};
1371         eval {
1372             if ($opt =~ m/^mp(\d+)$/) {
1373                 my $mp = $class->parse_volume($opt, $conf->{$opt});
1374                 if ($mp->{type} eq 'volume') {
1375                     $class->add_unused_volume($conf, $mp->{volume})
1376                         if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1377                 }
1378             } elsif ($opt =~ m/^unused(\d+)$/) {
1379                 PVE::LXC::delete_mountpoint_volume($storecfg, $vmid, $conf->{$opt})
1380                     if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1381             }
1382         };
1383         if (my $err = $@) {
1384             $add_apply_error->($opt, $err);
1385         } else {
1386             delete $conf->{$opt};
1387             $class->remove_from_pending_delete($conf, $opt);
1388         }
1389     }
1390
1391     $class->cleanup_pending($conf);
1392
1393     foreach my $opt (sort keys %{$conf->{pending}}) { # add/change
1394         next if $opt eq 'delete'; # just to be sure
1395         next if $selection && !$selection->{$opt};
1396         eval {
1397             if ($opt =~ m/^mp(\d+)$/) {
1398                 $class->apply_pending_mountpoint($vmid, $conf, $opt, $storecfg, 0);
1399             } elsif ($opt =~ m/^net(\d+)$/) {
1400                 my $netid = $1;
1401                 my $net = $class->parse_lxc_network($conf->{pending}->{$opt});
1402                 $conf->{pending}->{$opt} = $class->print_lxc_network($net);
1403             }
1404         };
1405         if (my $err = $@) {
1406             $add_apply_error->($opt, $err);
1407         } else {
1408             $conf->{$opt} = delete $conf->{pending}->{$opt};
1409         }
1410     }
1411 }
1412
1413 my $rescan_volume = sub {
1414     my ($storecfg, $mp) = @_;
1415     eval {
1416         $mp->{size} = PVE::Storage::volume_size_info($storecfg, $mp->{volume}, 5);
1417     };
1418     warn "Could not rescan volume size - $@\n" if $@;
1419 };
1420
1421 sub apply_pending_mountpoint {
1422     my ($class, $vmid, $conf, $opt, $storecfg, $running) = @_;
1423
1424     my $mp = $class->parse_volume($opt, $conf->{pending}->{$opt});
1425     my $old = $conf->{$opt};
1426     if ($mp->{type} eq 'volume') {
1427         if ($mp->{volume} =~ $PVE::LXC::NEW_DISK_RE) {
1428             my $original_value = $conf->{pending}->{$opt};
1429             my $vollist = PVE::LXC::create_disks(
1430                 $storecfg,
1431                 $vmid,
1432                 { $opt => $original_value },
1433                 $conf,
1434                 1,
1435             );
1436             if ($running) {
1437                 # Re-parse mount point:
1438                 my $mp = $class->parse_volume($opt, $conf->{pending}->{$opt});
1439                 eval {
1440                     PVE::LXC::mountpoint_hotplug($vmid, $conf, $opt, $mp, $storecfg);
1441                 };
1442                 my $err = $@;
1443                 if ($err) {
1444                     PVE::LXC::destroy_disks($storecfg, $vollist);
1445                     # The pending-changes code collects errors but keeps on looping through further
1446                     # pending changes, so unroll the change in $conf as well if destroy_disks()
1447                     # didn't die().
1448                     $conf->{pending}->{$opt} = $original_value;
1449                     die $err;
1450                 }
1451             }
1452         } else {
1453             die "skip\n" if $running && defined($old); # TODO: "changing" mount points?
1454             $rescan_volume->($storecfg, $mp);
1455             if ($running) {
1456                 PVE::LXC::mountpoint_hotplug($vmid, $conf, $opt, $mp, $storecfg);
1457             }
1458             $conf->{pending}->{$opt} = $class->print_ct_mountpoint($mp);
1459         }
1460     }
1461
1462     if (defined($old)) {
1463         my $mp = $class->parse_volume($opt, $old);
1464         if ($mp->{type} eq 'volume') {
1465             $class->add_unused_volume($conf, $mp->{volume})
1466                 if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1467         }
1468     }
1469 }
1470
1471 sub classify_mountpoint {
1472     my ($class, $vol) = @_;
1473     if ($vol =~ m!^/!) {
1474         return 'device' if $vol =~ m!^/dev/!;
1475         return 'bind';
1476     }
1477     return 'volume';
1478 }
1479
1480 my $__is_volume_in_use = sub {
1481     my ($class, $config, $volid) = @_;
1482     my $used = 0;
1483
1484     $class->foreach_volume($config, sub {
1485         my ($ms, $mountpoint) = @_;
1486         return if $used;
1487         $used = $mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid;
1488     });
1489
1490     return $used;
1491 };
1492
1493 sub is_volume_in_use_by_snapshots {
1494     my ($class, $config, $volid) = @_;
1495
1496     if (my $snapshots = $config->{snapshots}) {
1497         foreach my $snap (keys %$snapshots) {
1498             return 1 if $__is_volume_in_use->($class, $snapshots->{$snap}, $volid);
1499         }
1500     }
1501
1502     return 0;
1503 }
1504
1505 sub is_volume_in_use {
1506     my ($class, $config, $volid, $include_snapshots, $include_pending) = @_;
1507     return 1 if $__is_volume_in_use->($class, $config, $volid);
1508     return 1 if $include_snapshots && $class->is_volume_in_use_by_snapshots($config, $volid);
1509     return 1 if $include_pending && $__is_volume_in_use->($class, $config->{pending}, $volid);
1510     return 0;
1511 }
1512
1513 sub has_dev_console {
1514     my ($class, $conf) = @_;
1515
1516     return !(defined($conf->{console}) && !$conf->{console});
1517 }
1518
1519 sub has_lxc_entry {
1520     my ($class, $conf, $keyname) = @_;
1521
1522     if (my $lxcconf = $conf->{lxc}) {
1523         foreach my $entry (@$lxcconf) {
1524             my ($key, undef) = @$entry;
1525             return 1 if $key eq $keyname;
1526         }
1527     }
1528
1529     return 0;
1530 }
1531
1532 sub get_tty_count {
1533     my ($class, $conf) = @_;
1534
1535     return $conf->{tty} // $confdesc->{tty}->{default};
1536 }
1537
1538 sub get_cmode {
1539     my ($class, $conf) = @_;
1540
1541     return $conf->{cmode} // $confdesc->{cmode}->{default};
1542 }
1543
1544 sub valid_volume_keys {
1545     my ($class, $reverse) = @_;
1546
1547     my @names = ('rootfs');
1548
1549     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1550         push @names, "mp$i";
1551     }
1552
1553     return $reverse ? reverse @names : @names;
1554 }
1555
1556 sub get_vm_volumes {
1557     my ($class, $conf, $excludes) = @_;
1558
1559     my $vollist = [];
1560
1561     $class->foreach_volume($conf, sub {
1562         my ($ms, $mountpoint) = @_;
1563
1564         return if $excludes && $ms eq $excludes;
1565
1566         my $volid = $mountpoint->{volume};
1567         return if !$volid || $mountpoint->{type} ne 'volume';
1568
1569         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1570         return if !$sid;
1571
1572         push @$vollist, $volid;
1573     });
1574
1575     return $vollist;
1576 }
1577
1578 sub get_replicatable_volumes {
1579     my ($class, $storecfg, $vmid, $conf, $cleanup, $noerr) = @_;
1580
1581     my $volhash = {};
1582
1583     my $test_volid = sub {
1584         my ($volid, $mountpoint) = @_;
1585
1586         return if !$volid;
1587
1588         my $mptype = $mountpoint->{type};
1589         my $replicate = $mountpoint->{replicate} // 1;
1590
1591         if ($mptype ne 'volume') {
1592             # skip bindmounts if replicate = 0 even for cleanup,
1593             # since bind mounts could not have been replicated ever
1594             return if !$replicate;
1595             die "unable to replicate mountpoint type '$mptype'\n";
1596         }
1597
1598         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, $noerr);
1599         return if !$storeid;
1600
1601         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1602         return if $scfg->{shared};
1603
1604         my ($path, $owner, $vtype) = PVE::Storage::path($storecfg, $volid);
1605         return if !$owner || ($owner != $vmid);
1606
1607         die "unable to replicate volume '$volid', type '$vtype'\n" if $vtype ne 'images';
1608
1609         return if !$cleanup && !$replicate;
1610
1611         if (!PVE::Storage::volume_has_feature($storecfg, 'replicate', $volid)) {
1612             return if $cleanup || $noerr;
1613             die "missing replicate feature on volume '$volid'\n";
1614         }
1615
1616         $volhash->{$volid} = 1;
1617     };
1618
1619     $class->foreach_volume($conf, sub {
1620         my ($ms, $mountpoint) = @_;
1621         $test_volid->($mountpoint->{volume}, $mountpoint);
1622     });
1623
1624     foreach my $snapname (keys %{$conf->{snapshots}}) {
1625         my $snap = $conf->{snapshots}->{$snapname};
1626         $class->foreach_volume($snap, sub {
1627             my ($ms, $mountpoint) = @_;
1628             $test_volid->($mountpoint->{volume}, $mountpoint);
1629         });
1630     }
1631
1632     # add 'unusedX' volumes to volhash
1633     foreach my $key (keys %$conf) {
1634         if ($key =~ m/^unused/) {
1635             $test_volid->($conf->{$key}, { type => 'volume', replicate => 1 });
1636         }
1637     }
1638
1639     return $volhash;
1640 }
1641
1642 sub get_backup_volumes {
1643     my ($class, $conf) = @_;
1644
1645     my $return_volumes = [];
1646
1647     my $test_mountpoint = sub {
1648         my ($key, $volume) = @_;
1649
1650         my ($included, $reason) = $class->mountpoint_backup_enabled($key, $volume);
1651
1652         push @$return_volumes, {
1653             key => $key,
1654             included => $included,
1655             reason => $reason,
1656             volume_config => $volume,
1657         };
1658     };
1659
1660     PVE::LXC::Config->foreach_volume($conf, $test_mountpoint);
1661
1662     return $return_volumes;
1663 }
1664
1665 1;