src/PVE/LXC/Config.pm

   1 package PVE::LXC::Config;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl qw(O_RDONLY);
   7
   8 use PVE::AbstractConfig;
   9 use PVE::Cluster qw(cfs_register_file);
  10 use PVE::DataCenterConfig;
  11 use PVE::GuestHelpers;
  12 use PVE::INotify;
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Tools;
  15
  16 use PVE::LXC;
  17 use PVE::LXC::Tools;
  18
  19 use base qw(PVE::AbstractConfig);
  20
  21 use constant {
  22     FIFREEZE => 0xc0045877,
  23     FITHAW   => 0xc0045878,
  24 };
  25
  26 my $have_sdn;
  27 eval {
  28     require PVE::Network::SDN::Vnets;
  29     $have_sdn = 1;
  30 };
  31
  32 my $nodename = PVE::INotify::nodename();
  33 my $lock_handles =  {};
  34 my $lockdir = "/run/lock/lxc";
  35 mkdir $lockdir;
  36 mkdir "/etc/pve/nodes/$nodename/lxc";
  37 my $MAX_MOUNT_POINTS = 256;
  38 my $MAX_UNUSED_DISKS = $MAX_MOUNT_POINTS;
  39 my $MAX_DEVICES = 256;
  40
  41 # BEGIN implemented abstract methods from PVE::AbstractConfig
  42
  43 sub guest_type {
  44     return "CT";
  45 }
  46
  47 sub __config_max_unused_disks {
  48     my ($class) = @_;
  49
  50     return $MAX_UNUSED_DISKS;
  51 }
  52
  53 sub config_file_lock {
  54     my ($class, $vmid) = @_;
  55
  56     return "$lockdir/pve-config-${vmid}.lock";
  57 }
  58
  59 sub cfs_config_path {
  60     my ($class, $vmid, $node) = @_;
  61
  62     $node = $nodename if !$node;
  63     return "nodes/$node/lxc/$vmid.conf";
  64 }
  65
  66 sub mountpoint_backup_enabled {
  67     my ($class, $mp_key, $mountpoint) = @_;
  68
  69     my $enabled;
  70     my $reason;
  71
  72     if ($mp_key eq 'rootfs') {
  73         $enabled = 1;
  74         $reason = 'rootfs';
  75     } elsif ($mountpoint->{type} ne 'volume') {
  76         $enabled = 0;
  77         $reason = 'not a volume';
  78     } elsif ($mountpoint->{backup}) {
  79         $enabled = 1;
  80         $reason = 'enabled';
  81     } else {
  82         $enabled = 0;
  83         $reason = 'disabled';
  84     }
  85     return wantarray ? ($enabled, $reason) : $enabled;
  86 }
  87
  88 sub has_feature {
  89     my ($class, $feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
  90     my $err;
  91
  92     my $opts;
  93     if ($feature eq 'copy' || $feature eq 'clone') {
  94         $opts = {'valid_target_formats' => ['raw', 'subvol']};
  95     }
  96
  97     $class->foreach_volume($conf, sub {
  98         my ($ms, $mountpoint) = @_;
  99
 100         return if $err; # skip further test
 101         return if $backup_only && !$class->mountpoint_backup_enabled($ms, $mountpoint);
 102
 103         $err = 1 if !PVE::Storage::volume_has_feature(
 104             $storecfg, $feature, $mountpoint->{volume}, $snapname, $running, $opts);
 105     });
 106
 107     return $err ? 0 : 1;
 108 }
 109
 110 sub __snapshot_save_vmstate {
 111     my ($class, $vmid, $conf, $snapname, $storecfg) = @_;
 112     die "implement me - snapshot_save_vmstate\n";
 113 }
 114
 115 sub __snapshot_activate_storages {
 116     my ($class, $conf, $include_vmstate) = @_;
 117
 118     my $storecfg = PVE::Storage::config();
 119     my $opts = $include_vmstate ? { 'extra_keys' => ['vmstate'] } : {};
 120     my $storage_hash = {};
 121
 122     $class->foreach_volume_full($conf, $opts, sub {
 123         my ($vs, $mountpoint) = @_;
 124
 125         return if $mountpoint->{type} ne 'volume';
 126
 127         my ($storeid) = PVE::Storage::parse_volume_id($mountpoint->{volume});
 128         $storage_hash->{$storeid} = 1;
 129     });
 130
 131     PVE::Storage::activate_storage_list($storecfg, [ sort keys $storage_hash->%* ]);
 132 }
 133
 134 sub __snapshot_check_running {
 135     my ($class, $vmid) = @_;
 136     return PVE::LXC::check_running($vmid);
 137 }
 138
 139 sub __snapshot_check_freeze_needed {
 140     my ($class, $vmid, $config, $save_vmstate) = @_;
 141
 142     my $ret = $class->__snapshot_check_running($vmid);
 143     return ($ret, $ret);
 144 }
 145
 146 # implements similar functionality to fsfreeze(8)
 147 sub fsfreeze_mountpoint {
 148     my ($path, $thaw) = @_;
 149
 150     my $op = $thaw ? 'thaw' : 'freeze';
 151     my $ioctl = $thaw ? FITHAW : FIFREEZE;
 152
 153     sysopen my $fd, $path, O_RDONLY or die "failed to open $path: $!\n";
 154     my $ioctl_err;
 155     if (!ioctl($fd, $ioctl, 0)) {
 156         $ioctl_err = "$!";
 157     }
 158     close($fd);
 159     die "fs$op '$path' failed - $ioctl_err\n" if defined $ioctl_err;
 160 }
 161
 162 sub __snapshot_freeze {
 163     my ($class, $vmid, $unfreeze) = @_;
 164
 165     my $conf = $class->load_config($vmid);
 166     my $storagecfg = PVE::Storage::config();
 167
 168     my $freeze_mps = [];
 169     $class->foreach_volume($conf, sub {
 170         my ($ms, $mountpoint) = @_;
 171
 172         return if $mountpoint->{type} ne 'volume';
 173
 174         if (PVE::Storage::volume_snapshot_needs_fsfreeze($storagecfg, $mountpoint->{volume})) {
 175             push @$freeze_mps, $mountpoint->{mp};
 176         }
 177     });
 178
 179     my $freeze_mountpoints = sub {
 180         my ($thaw) = @_;
 181
 182         return if scalar(@$freeze_mps) == 0;
 183
 184         my $pid = PVE::LXC::find_lxc_pid($vmid);
 185
 186         for my $mp (@$freeze_mps) {
 187             eval{ fsfreeze_mountpoint("/proc/${pid}/root/${mp}", $thaw); };
 188             warn $@ if $@;
 189         }
 190     };
 191
 192     if ($unfreeze) {
 193         eval { PVE::LXC::thaw($vmid); };
 194         warn $@ if $@;
 195         $freeze_mountpoints->(1);
 196     } else {
 197         PVE::LXC::freeze($vmid);
 198         PVE::LXC::sync_container_namespace($vmid);
 199         $freeze_mountpoints->(0);
 200     }
 201 }
 202
 203 sub __snapshot_create_vol_snapshot {
 204     my ($class, $vmid, $ms, $mountpoint, $snapname) = @_;
 205
 206     my $storecfg = PVE::Storage::config();
 207
 208     return if $snapname eq 'vzdump' &&
 209         !$class->mountpoint_backup_enabled($ms, $mountpoint);
 210
 211     PVE::Storage::volume_snapshot($storecfg, $mountpoint->{volume}, $snapname);
 212 }
 213
 214 sub __snapshot_delete_remove_drive {
 215     my ($class, $snap, $remove_drive) = @_;
 216
 217     if ($remove_drive eq 'vmstate') {
 218         die "implement me - saving vmstate\n";
 219     } else {
 220         my $value = $snap->{$remove_drive};
 221         my $mountpoint = $class->parse_volume($remove_drive, $value, 1);
 222         delete $snap->{$remove_drive};
 223
 224         $class->add_unused_volume($snap, $mountpoint->{volume})
 225             if $mountpoint && ($mountpoint->{type} eq 'volume');
 226     }
 227 }
 228
 229 sub __snapshot_delete_vmstate_file {
 230     my ($class, $snap, $force) = @_;
 231
 232     die "implement me - saving vmstate\n";
 233 }
 234
 235 sub __snapshot_delete_vol_snapshot {
 236     my ($class, $vmid, $ms, $mountpoint, $snapname, $unused) = @_;
 237
 238     return if $snapname eq 'vzdump' &&
 239         !$class->mountpoint_backup_enabled($ms, $mountpoint);
 240
 241     my $storecfg = PVE::Storage::config();
 242     PVE::Storage::volume_snapshot_delete($storecfg, $mountpoint->{volume}, $snapname);
 243     push @$unused, $mountpoint->{volume};
 244 }
 245
 246 sub __snapshot_rollback_vol_possible {
 247     my ($class, $mountpoint, $snapname, $blockers) = @_;
 248
 249     my $storecfg = PVE::Storage::config();
 250     PVE::Storage::volume_rollback_is_possible(
 251         $storecfg,
 252         $mountpoint->{volume},
 253         $snapname,
 254         $blockers,
 255     );
 256 }
 257
 258 sub __snapshot_rollback_vol_rollback {
 259     my ($class, $mountpoint, $snapname) = @_;
 260
 261     my $storecfg = PVE::Storage::config();
 262     PVE::Storage::volume_snapshot_rollback($storecfg, $mountpoint->{volume}, $snapname);
 263 }
 264
 265 sub __snapshot_rollback_vm_stop {
 266     my ($class, $vmid) = @_;
 267
 268     PVE::LXC::vm_stop($vmid, 1)
 269         if $class->__snapshot_check_running($vmid);
 270 }
 271
 272 sub __snapshot_rollback_vm_start {
 273     my ($class, $vmid, $vmstate, $data);
 274
 275     die "implement me - save vmstate\n";
 276 }
 277
 278 sub __snapshot_rollback_get_unused {
 279     my ($class, $conf, $snap) = @_;
 280
 281     my $unused = [];
 282
 283     $class->foreach_volume($conf, sub {
 284         my ($vs, $volume) = @_;
 285
 286         return if $volume->{type} ne 'volume';
 287
 288         my $found = 0;
 289         my $volid = $volume->{volume};
 290
 291         $class->foreach_volume($snap, sub {
 292             my ($ms, $mountpoint) = @_;
 293
 294             return if $found;
 295             return if ($mountpoint->{type} ne 'volume');
 296
 297             $found = 1
 298                 if ($mountpoint->{volume} && $mountpoint->{volume} eq $volid);
 299         });
 300
 301         push @$unused, $volid if !$found;
 302     });
 303
 304     return $unused;
 305 }
 306
 307 # END implemented abstract methods from PVE::AbstractConfig
 308
 309 # BEGIN JSON config code
 310
 311 cfs_register_file('/lxc/', \&parse_pct_config, \&write_pct_config);
 312
 313
 314 my $valid_mount_option_re = qr/(noatime|lazytime|nodev|nosuid|noexec)/;
 315
 316 sub is_valid_mount_option {
 317     my ($option) = @_;
 318     return $option =~ $valid_mount_option_re;
 319 }
 320
 321 my $rootfs_desc = {
 322     volume => {
 323         type => 'string',
 324         default_key => 1,
 325         format => 'pve-lxc-mp-string',
 326         format_description => 'volume',
 327         description => 'Volume, device or directory to mount into the container.',
 328     },
 329     size => {
 330         type => 'string',
 331         format => 'disk-size',
 332         format_description => 'DiskSize',
 333         description => 'Volume size (read only value).',
 334         optional => 1,
 335     },
 336     acl => {
 337         type => 'boolean',
 338         description => 'Explicitly enable or disable ACL support.',
 339         optional => 1,
 340     },
 341     mountoptions => {
 342         optional => 1,
 343         type => 'string',
 344         description => 'Extra mount options for rootfs/mps.',
 345         format_description => 'opt[;opt...]',
 346         pattern => qr/$valid_mount_option_re(;$valid_mount_option_re)*/,
 347     },
 348     ro => {
 349         type => 'boolean',
 350         description => 'Read-only mount point',
 351         optional => 1,
 352     },
 353     quota => {
 354         type => 'boolean',
 355         description => 'Enable user quotas inside the container (not supported with zfs subvolumes)',
 356         optional => 1,
 357     },
 358     replicate => {
 359         type => 'boolean',
 360         description => 'Will include this volume to a storage replica job.',
 361         optional => 1,
 362         default => 1,
 363     },
 364     shared => {
 365         type => 'boolean',
 366         description => 'Mark this non-volume mount point as available on multiple nodes (see \'nodes\')',
 367         verbose_description => "Mark this non-volume mount point as available on all nodes.\n\nWARNING: This option does not share the mount point automatically, it assumes it is shared already!",
 368         optional => 1,
 369         default => 0,
 370     },
 371 };
 372
 373 PVE::JSONSchema::register_standard_option('pve-ct-rootfs', {
 374     type => 'string', format => $rootfs_desc,
 375     description => "Use volume as container root.",
 376     optional => 1,
 377 });
 378
 379 # IP address with optional interface suffix for link local ipv6 addresses
 380 PVE::JSONSchema::register_format('lxc-ip-with-ll-iface', \&verify_ip_with_ll_iface);
 381 sub verify_ip_with_ll_iface {
 382     my ($addr, $noerr) = @_;
 383
 384     if (my ($addr, $iface) = ($addr =~ /^(fe80:[^%]+)%(.*)$/)) {
 385         if (PVE::JSONSchema::pve_verify_ip($addr, 1)
 386             && PVE::JSONSchema::pve_verify_iface($iface, 1))
 387         {
 388             return $addr;
 389         }
 390     }
 391
 392     return PVE::JSONSchema::pve_verify_ip($addr, $noerr);
 393 }
 394
 395
 396 my $features_desc = {
 397     mount => {
 398         optional => 1,
 399         type => 'string',
 400         description => "Allow mounting file systems of specific types."
 401             ." This should be a list of file system types as used with the mount command."
 402             ." Note that this can have negative effects on the container's security."
 403             ." With access to a loop device, mounting a file can circumvent the mknod"
 404             ." permission of the devices cgroup, mounting an NFS file system can"
 405             ." block the host's I/O completely and prevent it from rebooting, etc.",
 406         format_description => 'fstype;fstype;...',
 407         pattern => qr/[a-zA-Z0-9_; ]+/,
 408     },
 409     nesting => {
 410         optional => 1,
 411         type => 'boolean',
 412         default => 0,
 413         description => "Allow nesting."
 414             ." Best used with unprivileged containers with additional id mapping."
 415             ." Note that this will expose procfs and sysfs contents of the host"
 416             ." to the guest.",
 417     },
 418     keyctl => {
 419         optional => 1,
 420         type => 'boolean',
 421         default => 0,
 422         description => "For unprivileged containers only: Allow the use of the keyctl() system call."
 423             ." This is required to use docker inside a container."
 424             ." By default unprivileged containers will see this system call as non-existent."
 425             ." This is mostly a workaround for systemd-networkd, as it will treat it as a fatal"
 426             ." error when some keyctl() operations are denied by the kernel due to lacking permissions."
 427             ." Essentially, you can choose between running systemd-networkd or docker.",
 428     },
 429     fuse => {
 430         optional => 1,
 431         type => 'boolean',
 432         default => 0,
 433         description => "Allow using 'fuse' file systems in a container."
 434             ." Note that interactions between fuse and the freezer cgroup can potentially cause I/O deadlocks.",
 435     },
 436     mknod => {
 437         optional => 1,
 438         type => 'boolean',
 439         default => 0,
 440         description => "Allow unprivileged containers to use mknod() to add certain device nodes."
 441             ." This requires a kernel with seccomp trap to user space support (5.3 or newer)."
 442             ." This is experimental.",
 443     },
 444     force_rw_sys => {
 445         optional => 1,
 446         type => 'boolean',
 447         default => 0,
 448         description => "Mount /sys in unprivileged containers as `rw` instead of `mixed`."
 449             ." This can break networking under newer (>= v245) systemd-network use."
 450     },
 451 };
 452
 453 my $confdesc = {
 454     lock => {
 455         optional => 1,
 456         type => 'string',
 457         description => "Lock/unlock the container.",
 458         enum => [qw(backup create destroyed disk fstrim migrate mounted rollback snapshot snapshot-delete)],
 459     },
 460     onboot => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Specifies whether a container will be started during system bootup.",
 464         default => 0,
 465     },
 466     startup => get_standard_option('pve-startup-order'),
 467     template => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable Template.",
 471         default => 0,
 472     },
 473     arch => {
 474         optional => 1,
 475         type => 'string',
 476         enum => ['amd64', 'i386', 'arm64', 'armhf', 'riscv32', 'riscv64'],
 477         description => "OS architecture type.",
 478         default => 'amd64',
 479     },
 480     ostype => {
 481         optional => 1,
 482         type => 'string',
 483         enum => [qw(debian devuan ubuntu centos fedora opensuse archlinux alpine gentoo nixos unmanaged)],
 484         description => "OS type. This is used to setup configuration inside the container, and corresponds to lxc setup scripts in /usr/share/lxc/config/<ostype>.common.conf. Value 'unmanaged' can be used to skip and OS specific setup.",
 485     },
 486     console => {
 487         optional => 1,
 488         type => 'boolean',
 489         description => "Attach a console device (/dev/console) to the container.",
 490         default => 1,
 491     },
 492     tty => {
 493         optional => 1,
 494         type => 'integer',
 495         description => "Specify the number of tty available to the container",
 496         minimum => 0,
 497         maximum => 6,
 498         default => 2,
 499     },
 500     cores => {
 501         optional => 1,
 502         type => 'integer',
 503         description => "The number of cores assigned to the container. A container can use all available cores by default.",
 504         minimum => 1,
 505         maximum => 8192,
 506     },
 507     cpulimit => {
 508         optional => 1,
 509         type => 'number',
 510         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has a total of '2' CPU time. Value '0' indicates no CPU limit.",
 511         minimum => 0,
 512         maximum => 8192,
 513         default => 0,
 514     },
 515     cpuunits => {
 516         optional => 1,
 517         type => 'integer',
 518         description => "CPU weight for a container, will be clamped to [1, 10000] in cgroup v2.",
 519         verbose_description => "CPU weight for a container. Argument is used in the kernel fair "
 520             ."scheduler. The larger the number is, the more CPU time this container gets. Number "
 521             ."is relative to the weights of all the other running guests.",
 522         minimum => 0,
 523         maximum => 500000,
 524         default => 'cgroup v1: 1024, cgroup v2: 100',
 525     },
 526     memory => {
 527         optional => 1,
 528         type => 'integer',
 529         description => "Amount of RAM for the container in MB.",
 530         minimum => 16,
 531         default => 512,
 532     },
 533     swap => {
 534         optional => 1,
 535         type => 'integer',
 536         description => "Amount of SWAP for the container in MB.",
 537         minimum => 0,
 538         default => 512,
 539     },
 540     hostname => {
 541         optional => 1,
 542         description => "Set a host name for the container.",
 543         type => 'string', format => 'dns-name',
 544         maxLength => 255,
 545     },
 546     description => {
 547         optional => 1,
 548         type => 'string',
 549         description => "Description for the Container. Shown in the web-interface CT's summary."
 550             ." This is saved as comment inside the configuration file.",
 551         maxLength => 1024 * 8,
 552     },
 553     searchdomain => {
 554         optional => 1,
 555         type => 'string', format => 'dns-name-list',
 556         description => "Sets DNS search domains for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 557     },
 558     nameserver => {
 559         optional => 1,
 560         type => 'string', format => 'lxc-ip-with-ll-iface-list',
 561         description => "Sets DNS server IP address for a container. Create will automatically use the setting from the host if you neither set searchdomain nor nameserver.",
 562     },
 563     timezone => {
 564         optional => 1,
 565         type => 'string', format => 'pve-ct-timezone',
 566         description => "Time zone to use in the container. If option isn't set, then nothing will be done. Can be set to 'host' to match the host time zone, or an arbitrary time zone option from /usr/share/zoneinfo/zone.tab",
 567     },
 568     rootfs => get_standard_option('pve-ct-rootfs'),
 569     parent => {
 570         optional => 1,
 571         type => 'string', format => 'pve-configid',
 572         maxLength => 40,
 573         description => "Parent snapshot name. This is used internally, and should not be modified.",
 574     },
 575     snaptime => {
 576         optional => 1,
 577         description => "Timestamp for snapshots.",
 578         type => 'integer',
 579         minimum => 0,
 580     },
 581     cmode => {
 582         optional => 1,
 583         description => "Console mode. By default, the console command tries to open a connection to one of the available tty devices. By setting cmode to 'console' it tries to attach to /dev/console instead. If you set cmode to 'shell', it simply invokes a shell inside the container (no login).",
 584         type => 'string',
 585         enum => ['shell', 'console', 'tty'],
 586         default => 'tty',
 587     },
 588     protection => {
 589         optional => 1,
 590         type => 'boolean',
 591         description => "Sets the protection flag of the container. This will prevent the CT or CT's disk remove/update operation.",
 592         default => 0,
 593     },
 594     unprivileged => {
 595         optional => 1,
 596         type => 'boolean',
 597         description => "Makes the container run as unprivileged user. (Should not be modified manually.)",
 598         default => 0,
 599     },
 600     features => {
 601         optional => 1,
 602         type => 'string',
 603         format => $features_desc,
 604         description => "Allow containers access to advanced features.",
 605     },
 606     hookscript => {
 607         optional => 1,
 608         type => 'string',
 609         format => 'pve-volume-id',
 610         description => 'Script that will be exectued during various steps in the containers lifetime.',
 611     },
 612     tags => {
 613         type => 'string', format => 'pve-tag-list',
 614         description => 'Tags of the Container. This is only meta information.',
 615         optional => 1,
 616     },
 617     debug => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Try to be more verbose. For now this only enables debug log-level on start.",
 621         default => 0,
 622     },
 623 };
 624
 625 my $valid_lxc_conf_keys = {
 626     'lxc.apparmor.profile' => 1,
 627     'lxc.apparmor.allow_incomplete' => 1,
 628     'lxc.apparmor.allow_nesting' => 1,
 629     'lxc.apparmor.raw' => 1,
 630     'lxc.selinux.context' => 1,
 631     'lxc.include' => 1,
 632     'lxc.arch' => 1,
 633     'lxc.uts.name' => 1,
 634     'lxc.signal.halt' => 1,
 635     'lxc.signal.reboot' => 1,
 636     'lxc.signal.stop' => 1,
 637     'lxc.init.cmd' => 1,
 638     'lxc.pty.max' => 1,
 639     'lxc.console.logfile' => 1,
 640     'lxc.console.path' => 1,
 641     'lxc.tty.max' => 1,
 642     'lxc.devtty.dir' => 1,
 643     'lxc.hook.autodev' => 1,
 644     'lxc.autodev' => 1,
 645     'lxc.kmsg' => 1,
 646     'lxc.mount.fstab' => 1,
 647     'lxc.mount.entry' => 1,
 648     'lxc.mount.auto' => 1,
 649     'lxc.rootfs.path' => 'lxc.rootfs.path is auto generated from rootfs',
 650     'lxc.rootfs.mount' => 1,
 651     'lxc.rootfs.options' => 'lxc.rootfs.options is not supported' .
 652                             ', please use mount point options in the "rootfs" key',
 653     # lxc.cgroup.*
 654     # lxc.prlimit.*
 655     # lxc.net.*
 656     'lxc.cap.drop' => 1,
 657     'lxc.cap.keep' => 1,
 658     'lxc.seccomp.profile' => 1,
 659     'lxc.seccomp.notify.proxy' => 1,
 660     'lxc.seccomp.notify.cookie' => 1,
 661     'lxc.idmap' => 1,
 662     'lxc.hook.pre-start' => 1,
 663     'lxc.hook.pre-mount' => 1,
 664     'lxc.hook.mount' => 1,
 665     'lxc.hook.start' => 1,
 666     'lxc.hook.stop' => 1,
 667     'lxc.hook.post-stop' => 1,
 668     'lxc.hook.clone' => 1,
 669     'lxc.hook.destroy' => 1,
 670     'lxc.hook.version' => 1,
 671     'lxc.log.level' => 1,
 672     'lxc.log.file' => 1,
 673     'lxc.start.auto' => 1,
 674     'lxc.start.delay' => 1,
 675     'lxc.start.order' => 1,
 676     'lxc.group' => 1,
 677     'lxc.environment' => 1,
 678
 679     # All these are namespaced via CLONE_NEWIPC (see namespaces(7)).
 680     'lxc.sysctl.fs.mqueue' => 1,
 681     'lxc.sysctl.kernel.msgmax' => 1,
 682     'lxc.sysctl.kernel.msgmnb' => 1,
 683     'lxc.sysctl.kernel.msgmni' => 1,
 684     'lxc.sysctl.kernel.sem' => 1,
 685     'lxc.sysctl.kernel.shmall' => 1,
 686     'lxc.sysctl.kernel.shmmax' => 1,
 687     'lxc.sysctl.kernel.shmmni' => 1,
 688     'lxc.sysctl.kernel.shm_rmid_forced' => 1,
 689 };
 690
 691 my $deprecated_lxc_conf_keys = {
 692     # Deprecated (removed with lxc 3.0):
 693     'lxc.aa_profile'           => 'lxc.apparmor.profile',
 694     'lxc.aa_allow_incomplete'  => 'lxc.apparmor.allow_incomplete',
 695     'lxc.console'              => 'lxc.console.path',
 696     'lxc.devttydir'            => 'lxc.tty.dir',
 697     'lxc.haltsignal'           => 'lxc.signal.halt',
 698     'lxc.rebootsignal'         => 'lxc.signal.reboot',
 699     'lxc.stopsignal'           => 'lxc.signal.stop',
 700     'lxc.id_map'               => 'lxc.idmap',
 701     'lxc.init_cmd'             => 'lxc.init.cmd',
 702     'lxc.loglevel'             => 'lxc.log.level',
 703     'lxc.logfile'              => 'lxc.log.file',
 704     'lxc.mount'                => 'lxc.mount.fstab',
 705     'lxc.network.type'         => 'lxc.net.INDEX.type',
 706     'lxc.network.flags'        => 'lxc.net.INDEX.flags',
 707     'lxc.network.link'         => 'lxc.net.INDEX.link',
 708     'lxc.network.mtu'          => 'lxc.net.INDEX.mtu',
 709     'lxc.network.name'         => 'lxc.net.INDEX.name',
 710     'lxc.network.hwaddr'       => 'lxc.net.INDEX.hwaddr',
 711     'lxc.network.ipv4'         => 'lxc.net.INDEX.ipv4.address',
 712     'lxc.network.ipv4.gateway' => 'lxc.net.INDEX.ipv4.gateway',
 713     'lxc.network.ipv6'         => 'lxc.net.INDEX.ipv6.address',
 714     'lxc.network.ipv6.gateway' => 'lxc.net.INDEX.ipv6.gateway',
 715     'lxc.network.script.up'    => 'lxc.net.INDEX.script.up',
 716     'lxc.network.script.down'  => 'lxc.net.INDEX.script.down',
 717     'lxc.pts'                  => 'lxc.pty.max',
 718     'lxc.se_context'           => 'lxc.selinux.context',
 719     'lxc.seccomp'              => 'lxc.seccomp.profile',
 720     'lxc.tty'                  => 'lxc.tty.max',
 721     'lxc.utsname'              => 'lxc.uts.name',
 722 };
 723
 724 sub is_valid_lxc_conf_key {
 725     my ($vmid, $key) = @_;
 726     if ($key =~ /^lxc\.limit\./) {
 727         warn "vm $vmid - $key: lxc.limit.* was renamed to lxc.prlimit.*\n";
 728         return 1;
 729     }
 730     if (defined(my $new_name = $deprecated_lxc_conf_keys->{$key})) {
 731         warn "vm $vmid - $key is deprecated and was renamed to $new_name\n";
 732         return 1;
 733     }
 734     my $validity = $valid_lxc_conf_keys->{$key};
 735     return $validity if defined($validity);
 736     return 1 if $key =~ /^lxc\.cgroup2?\./  # allow all cgroup values
 737              || $key =~ /^lxc\.prlimit\./ # allow all prlimits
 738              || $key =~ /^lxc\.net\./;    # allow custom network definitions
 739     return 0;
 740 }
 741
 742 our $netconf_desc = {
 743     type => {
 744         type => 'string',
 745         optional => 1,
 746         description => "Network interface type.",
 747         enum => [qw(veth)],
 748     },
 749     name => {
 750         type => 'string',
 751         format_description => 'string',
 752         description => 'Name of the network device as seen from inside the container. (lxc.network.name)',
 753         pattern => '[-_.\w\d]+',
 754     },
 755     bridge => {
 756         type => 'string',
 757         format_description => 'bridge',
 758         description => 'Bridge to attach the network device to.',
 759         pattern => '[-_.\w\d]+',
 760         optional => 1,
 761     },
 762     hwaddr => get_standard_option('mac-addr', {
 763         description => 'The interface MAC address. This is dynamically allocated by default, but you can set that statically if needed, for example to always have the same link-local IPv6 address. (lxc.network.hwaddr)',
 764         }),
 765     mtu => {
 766         type => 'integer',
 767         description => 'Maximum transfer unit of the interface. (lxc.network.mtu)',
 768         minimum => 64, # minimum ethernet frame is 64 bytes
 769         maximum => 65535,
 770         optional => 1,
 771     },
 772     ip => {
 773         type => 'string',
 774         format => 'pve-ipv4-config',
 775         format_description => '(IPv4/CIDR|dhcp|manual)',
 776         description => 'IPv4 address in CIDR format.',
 777         optional => 1,
 778     },
 779     gw => {
 780         type => 'string',
 781         format => 'ipv4',
 782         format_description => 'GatewayIPv4',
 783         description => 'Default gateway for IPv4 traffic.',
 784         optional => 1,
 785     },
 786     ip6 => {
 787         type => 'string',
 788         format => 'pve-ipv6-config',
 789         format_description => '(IPv6/CIDR|auto|dhcp|manual)',
 790         description => 'IPv6 address in CIDR format.',
 791         optional => 1,
 792     },
 793     gw6 => {
 794         type => 'string',
 795         format => 'ipv6',
 796         format_description => 'GatewayIPv6',
 797         description => 'Default gateway for IPv6 traffic.',
 798         optional => 1,
 799     },
 800     firewall => {
 801         type => 'boolean',
 802         description => "Controls whether this interface's firewall rules should be used.",
 803         optional => 1,
 804     },
 805     tag => {
 806         type => 'integer',
 807         minimum => 1,
 808         maximum => 4094,
 809         description => "VLAN tag for this interface.",
 810         optional => 1,
 811     },
 812     trunks => {
 813         type => 'string',
 814         pattern => qr/\d+(?:;\d+)*/,
 815         format_description => 'vlanid[;vlanid...]',
 816         description => "VLAN ids to pass through the interface",
 817         optional => 1,
 818     },
 819     rate => {
 820         type => 'number',
 821         format_description => 'mbps',
 822         description => "Apply rate limiting to the interface",
 823         optional => 1,
 824     },
 825     # TODO: Rename this option and the qemu-server one to `link-down` for PVE 8.0
 826     link_down => {
 827         type => 'boolean',
 828         description => 'Whether this interface should be disconnected (like pulling the plug).',
 829         optional => 1,
 830     },
 831 };
 832 PVE::JSONSchema::register_format('pve-lxc-network', $netconf_desc);
 833
 834 my $MAX_LXC_NETWORKS = 32;
 835 for (my $i = 0; $i < $MAX_LXC_NETWORKS; $i++) {
 836     $confdesc->{"net$i"} = {
 837         optional => 1,
 838         type => 'string', format => $netconf_desc,
 839         description => "Specifies network interfaces for the container.",
 840     };
 841 }
 842
 843 PVE::JSONSchema::register_format('pve-ct-timezone', \&verify_ct_timezone);
 844 sub verify_ct_timezone {
 845     my ($timezone, $noerr) = @_;
 846
 847     return if $timezone eq 'host'; # using host settings
 848
 849     PVE::JSONSchema::pve_verify_timezone($timezone);
 850 }
 851
 852 PVE::JSONSchema::register_format('pve-lxc-mp-string', \&verify_lxc_mp_string);
 853 sub verify_lxc_mp_string {
 854     my ($mp, $noerr) = @_;
 855
 856     # do not allow:
 857     # /./ or /../
 858     # /. or /.. at the end
 859     # ../ at the beginning
 860
 861     if($mp =~ m@/\.\.?/@ ||
 862        $mp =~ m@/\.\.?$@ ||
 863        $mp =~ m@^\.\./@) {
 864         return undef if $noerr;
 865         die "$mp contains illegal character sequences\n";
 866     }
 867     return $mp;
 868 }
 869
 870 my $mp_desc = {
 871     %$rootfs_desc,
 872     backup => {
 873         type => 'boolean',
 874         description => 'Whether to include the mount point in backups.',
 875         verbose_description => 'Whether to include the mount point in backups '.
 876                                '(only used for volume mount points).',
 877         optional => 1,
 878     },
 879     mp => {
 880         type => 'string',
 881         format => 'pve-lxc-mp-string',
 882         format_description => 'Path',
 883         description => 'Path to the mount point as seen from inside the container '.
 884                        '(must not contain symlinks).',
 885         verbose_description => "Path to the mount point as seen from inside the container.\n\n".
 886                                "NOTE: Must not contain any symlinks for security reasons."
 887     },
 888 };
 889 PVE::JSONSchema::register_format('pve-ct-mountpoint', $mp_desc);
 890
 891 my $unused_desc = {
 892     volume => {
 893         type => 'string',
 894         default_key => 1,
 895         format => 'pve-volume-id',
 896         format_description => 'volume',
 897         description => 'The volume that is not used currently.',
 898     }
 899 };
 900
 901 for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
 902     $confdesc->{"mp$i"} = {
 903         optional => 1,
 904         type => 'string', format => $mp_desc,
 905         description => "Use volume as container mount point. Use the special " .
 906             "syntax STORAGE_ID:SIZE_IN_GiB to allocate a new volume.",
 907     };
 908 }
 909
 910 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
 911     $confdesc->{"unused$i"} = {
 912         optional => 1,
 913         type => 'string', format => $unused_desc,
 914         description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
 915     }
 916 }
 917
 918 PVE::JSONSchema::register_format('pve-lxc-dev-string', \&verify_lxc_dev_string);
 919 sub verify_lxc_dev_string {
 920     my ($dev, $noerr) = @_;
 921
 922     # do not allow /./ or /../ or /.$ or /..$
 923     # enforce /dev/ at the beginning
 924
 925     if (
 926         $dev =~ m@/\.\.?(?:/|$)@ ||
 927         $dev !~ m!^/dev/!
 928     ) {
 929         return undef if $noerr;
 930         die "$dev is not a valid device path\n";
 931     }
 932
 933     return $dev;
 934 }
 935
 936 my $dev_desc = {
 937     path => {
 938         optional => 1,
 939         type => 'string',
 940         default_key => 1,
 941         format => 'pve-lxc-dev-string',
 942         format_description => 'Path',
 943         description => 'Device to pass through to the container',
 944         verbose_description => 'Path to the device to pass through to the container',
 945     },
 946     mode => {
 947         optional => 1,
 948         type => 'string',
 949         pattern => '0[0-7]{3}',
 950         format_description => 'Octal access mode',
 951         description => 'Access mode to be set on the device node',
 952     },
 953     uid => {
 954         optional => 1,
 955         type => 'integer',
 956         minimum => 0,
 957         description => 'User ID to be assigned to the device node',
 958     },
 959     gid => {
 960         optional => 1,
 961         type => 'integer',
 962         minimum => 0,
 963         description => 'Group ID to be assigned to the device node',
 964     },
 965 };
 966
 967 for (my $i = 0; $i < $MAX_DEVICES; $i++) {
 968     $confdesc->{"dev$i"} = {
 969         optional => 1,
 970         type => 'string', format => $dev_desc,
 971         description => "Device to pass through to the container",
 972     }
 973 }
 974
 975 sub parse_pct_config {
 976     my ($filename, $raw, $strict) = @_;
 977
 978     return undef if !defined($raw);
 979
 980     my $res = {
 981         digest => Digest::SHA::sha1_hex($raw),
 982         snapshots => {},
 983         pending => {},
 984     };
 985
 986     my $handle_error = sub {
 987         my ($msg) = @_;
 988
 989         if ($strict) {
 990             die $msg;
 991         } else {
 992             warn $msg;
 993         }
 994     };
 995
 996     $filename =~ m|/lxc/(\d+).conf$|
 997         || die "got strange filename '$filename'";
 998
 999     my $vmid = $1;
1000
1001     my $conf = $res;
1002     my $descr = '';
1003     my $section = '';
1004
1005     my @lines = split(/\n/, $raw);
1006     foreach my $line (@lines) {
1007         next if $line =~ m/^\s*$/;
1008
1009         if ($line =~ m/^\[pve:pending\]\s*$/i) {
1010             $section = 'pending';
1011             $conf->{description} = $descr if $descr;
1012             $descr = '';
1013             $conf = $res->{$section} = {};
1014             next;
1015         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1016             $section = $1;
1017             $conf->{description} = $descr if $descr;
1018             $descr = '';
1019             $conf = $res->{snapshots}->{$section} = {};
1020             next;
1021         }
1022
1023         if ($line =~ m/^\#(.*)$/) {
1024             $descr .= PVE::Tools::decode_text($1) . "\n";
1025             next;
1026         }
1027
1028         if ($line =~ m/^(lxc\.[a-z0-9_\-\.]+)(:|\s*=)\s*(.*?)\s*$/) {
1029             my $key = $1;
1030             my $value = $3;
1031             my $validity = is_valid_lxc_conf_key($vmid, $key);
1032             if ($validity eq 1) {
1033                 push @{$conf->{lxc}}, [$key, $value];
1034             } elsif (my $errmsg = $validity) {
1035                 $handle_error->("vm $vmid - $key: $errmsg\n");
1036             } else {
1037                 $handle_error->("vm $vmid - unable to parse config: $line\n");
1038             }
1039         } elsif ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1040             $descr .= PVE::Tools::decode_text($2);
1041         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1042             $conf->{snapstate} = $1;
1043         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1044             my $value = $1;
1045             if ($section eq 'pending') {
1046                 $conf->{delete} = $value;
1047             } else {
1048                 $handle_error->("vm $vmid - property 'delete' is only allowed in [pve:pending]\n");
1049             }
1050         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
1051             my $key = $1;
1052             my $value = $2;
1053             eval { $value = PVE::LXC::Config->check_type($key, $value); };
1054             $handle_error->("vm $vmid - unable to parse value of '$key' - $@") if $@;
1055             $conf->{$key} = $value;
1056         } else {
1057             $handle_error->("vm $vmid - unable to parse config: $line\n");
1058         }
1059     }
1060
1061     $conf->{description} = $descr if $descr;
1062
1063     delete $res->{snapstate}; # just to be sure
1064
1065     return $res;
1066 }
1067
1068 sub write_pct_config {
1069     my ($filename, $conf) = @_;
1070
1071     delete $conf->{snapstate}; # just to be sure
1072
1073     my $volidlist = PVE::LXC::Config->get_vm_volumes($conf);
1074     my $used_volids = {};
1075     foreach my $vid (@$volidlist) {
1076         $used_volids->{$vid} = 1;
1077     }
1078
1079     # remove 'unusedX' settings if the volume is still used
1080     foreach my $key (keys %$conf) {
1081         my $value = $conf->{$key};
1082         if ($key =~ m/^unused/ && $used_volids->{$value}) {
1083             delete $conf->{$key};
1084         }
1085     }
1086
1087     my $generate_raw_config = sub {
1088         my ($conf) = @_;
1089
1090         my $raw = '';
1091
1092         # add description as comment to top of file
1093         my $descr = $conf->{description} || '';
1094         foreach my $cl (split(/\n/, $descr)) {
1095             $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
1096         }
1097
1098         foreach my $key (sort keys %$conf) {
1099             next if $key eq 'digest' || $key eq 'description' ||
1100                     $key eq 'pending' || $key eq 'snapshots' ||
1101                     $key eq 'snapname' || $key eq 'lxc';
1102             my $value = $conf->{$key};
1103             die "detected invalid newline inside property '$key'\n"
1104                 if $value =~ m/\n/;
1105             $raw .= "$key: $value\n";
1106         }
1107
1108         if (my $lxcconf = $conf->{lxc}) {
1109             foreach my $entry (@$lxcconf) {
1110                 my ($k, $v) = @$entry;
1111                 $raw .= "$k: $v\n";
1112             }
1113         }
1114
1115         return $raw;
1116     };
1117
1118     my $raw = &$generate_raw_config($conf);
1119
1120     if (scalar(keys %{$conf->{pending}})){
1121         $raw .= "\n[pve:pending]\n";
1122         $raw .= &$generate_raw_config($conf->{pending});
1123     }
1124
1125     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
1126         $raw .= "\n[$snapname]\n";
1127         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1128     }
1129
1130     return $raw;
1131 }
1132
1133 sub update_pct_config {
1134     my ($class, $vmid, $conf, $running, $param, $delete, $revert) = @_;
1135
1136     my $storage_cfg = PVE::Storage::config();
1137
1138     foreach my $opt (@$revert) {
1139         delete $conf->{pending}->{$opt};
1140         $class->remove_from_pending_delete($conf, $opt); # also remove from deletion queue
1141     }
1142
1143     # write updates to pending section
1144     my $modified = {}; # record modified options
1145
1146     foreach my $opt (@$delete) {
1147         if (!defined($conf->{$opt}) && !defined($conf->{pending}->{$opt})) {
1148             warn "cannot delete '$opt' - not set in current configuration!\n";
1149             next;
1150         }
1151         $modified->{$opt} = 1;
1152         if ($opt eq 'memory' || $opt eq 'rootfs' || $opt eq 'ostype') {
1153             die "unable to delete required option '$opt'\n";
1154         } elsif ($opt =~ m/^unused(\d+)$/) {
1155             $class->check_protection($conf, "can't remove CT $vmid drive '$opt'");
1156         } elsif ($opt =~ m/^mp(\d+)$/) {
1157             $class->check_protection($conf, "can't remove CT $vmid drive '$opt'");
1158         } elsif ($opt eq 'unprivileged') {
1159             die "unable to delete read-only option: '$opt'\n";
1160         }
1161         $class->add_to_pending_delete($conf, $opt);
1162     }
1163
1164     my $check_content_type = sub {
1165         my ($mp) = @_;
1166         my $sid = PVE::Storage::parse_volume_id($mp->{volume});
1167         my $storage_config = PVE::Storage::storage_config($storage_cfg, $sid);
1168         die "storage '$sid' does not allow content type 'rootdir' (Container)\n"
1169             if !$storage_config->{content}->{rootdir};
1170     };
1171
1172     foreach my $opt (sort keys %$param) { # add/change
1173         $modified->{$opt} = 1;
1174         my $value = $param->{$opt};
1175         if ($opt =~ m/^mp(\d+)$/ || $opt eq 'rootfs') {
1176             $class->check_protection($conf, "can't update CT $vmid drive '$opt'");
1177             my $mp = $class->parse_volume($opt, $value);
1178             $check_content_type->($mp) if ($mp->{type} eq 'volume');
1179         } elsif ($opt eq 'hookscript') {
1180             PVE::GuestHelpers::check_hookscript($value);
1181         } elsif ($opt eq 'nameserver') {
1182             $value = PVE::LXC::verify_nameserver_list($value);
1183         } elsif ($opt eq 'searchdomain') {
1184             $value = PVE::LXC::verify_searchdomain_list($value);
1185         } elsif ($opt eq 'unprivileged') {
1186             die "unable to modify read-only option: '$opt'\n";
1187         } elsif ($opt eq 'tags') {
1188             $value = PVE::GuestHelpers::get_unique_tags($value);
1189         } elsif ($opt =~ m/^net(\d+)$/) {
1190             my $res = PVE::JSONSchema::parse_property_string($netconf_desc, $value);
1191
1192             if (my $mtu = $res->{mtu}) {
1193                 my $bridge_mtu = PVE::Network::read_bridge_mtu($res->{bridge});
1194                 die "$opt: MTU size '$mtu' is bigger than bridge MTU '$bridge_mtu'\n"
1195                     if ($mtu > $bridge_mtu);
1196             }
1197         } elsif ($opt =~ m/^dev(\d+)$/) {
1198             my $device = $class->parse_device($value);
1199
1200             die "Path is not defined for passthrough device $opt"
1201                 if !defined($device->{path});
1202
1203             # Validate device
1204             PVE::LXC::Tools::get_device_mode_and_rdev($device->{path});
1205         }
1206         $conf->{pending}->{$opt} = $value;
1207         $class->remove_from_pending_delete($conf, $opt);
1208     }
1209
1210     my $changes = $class->cleanup_pending($conf);
1211
1212     my $errors = {};
1213     if ($running) {
1214         $class->vmconfig_hotplug_pending($vmid, $conf, $storage_cfg, $modified, $errors);
1215     } else {
1216         $class->vmconfig_apply_pending($vmid, $conf, $storage_cfg, $modified, $errors);
1217     }
1218
1219     return $errors;
1220 }
1221
1222 sub check_type {
1223     my ($class, $key, $value) = @_;
1224
1225     die "unknown setting '$key'\n" if !$confdesc->{$key};
1226
1227     my $type = $confdesc->{$key}->{type};
1228
1229     if (!defined($value)) {
1230         die "got undefined value\n";
1231     }
1232
1233     if ($value =~ m/[\n\r]/) {
1234         die "property contains a line feed\n";
1235     }
1236
1237     if ($type eq 'boolean') {
1238         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1239         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1240         die "type check ('boolean') failed - got '$value'\n";
1241     } elsif ($type eq 'integer') {
1242         return int($1) if $value =~ m/^(\d+)$/;
1243         die "type check ('integer') failed - got '$value'\n";
1244     } elsif ($type eq 'number') {
1245         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1246         die "type check ('number') failed - got '$value'\n";
1247     } elsif ($type eq 'string') {
1248         if (my $fmt = $confdesc->{$key}->{format}) {
1249             PVE::JSONSchema::check_format($fmt, $value);
1250             return $value;
1251         }
1252         return $value;
1253     } else {
1254         die "internal error"
1255     }
1256 }
1257
1258
1259 # add JSON properties for create and set function
1260 sub json_config_properties {
1261     my ($class, $prop) = @_;
1262
1263     foreach my $opt (keys %$confdesc) {
1264         next if $opt eq 'parent' || $opt eq 'snaptime';
1265         next if $prop->{$opt};
1266         $prop->{$opt} = $confdesc->{$opt};
1267     }
1268
1269     return $prop;
1270 }
1271
1272 my $parse_ct_mountpoint_full = sub {
1273     my ($class, $desc, $data, $noerr) = @_;
1274
1275     $data //= '';
1276
1277     my $res;
1278     eval { $res = PVE::JSONSchema::parse_property_string($desc, $data) };
1279     if ($@) {
1280         return undef if $noerr;
1281         die $@;
1282     }
1283
1284     if (defined(my $size = $res->{size})) {
1285         $size = PVE::JSONSchema::parse_size($size);
1286         if (!defined($size)) {
1287             return undef if $noerr;
1288             die "invalid size: $size\n";
1289         }
1290         $res->{size} = $size;
1291     }
1292
1293     $res->{type} = $class->classify_mountpoint($res->{volume});
1294
1295     return $res;
1296 };
1297
1298 sub print_ct_mountpoint {
1299     my ($class, $info, $nomp) = @_;
1300     my $skip = [ 'type' ];
1301     push @$skip, 'mp' if $nomp;
1302     return PVE::JSONSchema::print_property_string($info, $mp_desc, $skip);
1303 }
1304
1305 sub print_ct_unused {
1306     my ($class, $info) = @_;
1307
1308     my $skip = [ 'type' ];
1309     return PVE::JSONSchema::print_property_string($info, $unused_desc, $skip);
1310 }
1311
1312 sub parse_volume {
1313     my ($class, $key, $volume_string, $noerr) = @_;
1314
1315     if ($key eq 'rootfs') {
1316         my $res =  $parse_ct_mountpoint_full->($class, $rootfs_desc, $volume_string, $noerr);
1317         $res->{mp} = '/' if defined($res);
1318         return $res;
1319     } elsif ($key =~ m/^mp\d+$/) {
1320         return $parse_ct_mountpoint_full->($class, $mp_desc, $volume_string, $noerr);
1321     } elsif ($key =~ m/^unused\d+$/) {
1322         return $parse_ct_mountpoint_full->($class, $unused_desc, $volume_string, $noerr);
1323     }
1324
1325     die "parse_volume - unknown type: $key\n" if !$noerr;
1326
1327     return;
1328 }
1329
1330 sub parse_device {
1331     my ($class, $device_string, $noerr) = @_;
1332
1333     my $res = eval { PVE::JSONSchema::parse_property_string($dev_desc, $device_string) };
1334     if ($@) {
1335         return undef if $noerr;
1336         die $@;
1337     }
1338
1339     if (!defined($res->{path})) {
1340         return undef if $noerr;
1341         die "Path has to be defined\n";
1342     }
1343
1344     return $res;
1345 }
1346
1347 sub print_volume {
1348     my ($class, $key, $volume) = @_;
1349
1350     return $class->print_ct_unused($volume) if $key =~ m/^unused(\d+)$/;
1351
1352     return $class->print_ct_mountpoint($volume, $key eq 'rootfs');
1353 }
1354
1355 sub volid_key {
1356     my ($class) = @_;
1357
1358     return 'volume';
1359 }
1360
1361 sub print_lxc_network {
1362     my ($class, $net) = @_;
1363     return PVE::JSONSchema::print_property_string($net, $netconf_desc);
1364 }
1365
1366 sub parse_lxc_network {
1367     my ($class, $data) = @_;
1368
1369     return {} if !$data;
1370
1371     my $res = PVE::JSONSchema::parse_property_string($netconf_desc, $data);
1372
1373     $res->{type} = 'veth';
1374     if (!$res->{hwaddr}) {
1375         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1376         $res->{hwaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1377     }
1378
1379     return $res;
1380 }
1381
1382 sub parse_features {
1383     my ($class, $data) = @_;
1384     return {} if !$data;
1385     return PVE::JSONSchema::parse_property_string($features_desc, $data);
1386 }
1387
1388 sub option_exists {
1389     my ($class, $name) = @_;
1390
1391     return defined($confdesc->{$name});
1392 }
1393 # END JSON config code
1394
1395 # takes a max memory value as KiB and returns an tuple with max and high values
1396 sub calculate_memory_constraints {
1397     my ($memory) = @_;
1398
1399     return if !defined($memory);
1400
1401     # cgroup memory usage is limited by the hard 'max' limit (OOM-killer enforced) and the soft
1402     # 'high' limit (cgroup processes get throttled and put under heavy reclaim pressure).
1403     my $memory_max = int($memory * 1024 * 1024);
1404     # Set the high to 1016/1024 (~99.2%) of the 'max' hard limit clamped to 128 MiB max, to scale
1405     # it for the lower range while having a decent 2^x based rest for 2^y memory configs.
1406     my $memory_high = $memory >= 16 * 1024 ? int(($memory - 128) * 1024 * 1024) : int($memory * 1024 * 1016);
1407
1408     return ($memory_max, $memory_high);
1409 }
1410
1411 my $LXC_FASTPLUG_OPTIONS= {
1412     'description' => 1,
1413     'onboot' => 1,
1414     'startup' => 1,
1415     'protection' => 1,
1416     'hostname' => 1,
1417     'hookscript' => 1,
1418     'cores' => 1,
1419     'tags' => 1,
1420     'lock' => 1,
1421 };
1422
1423 sub vmconfig_hotplug_pending {
1424     my ($class, $vmid, $conf, $storecfg, $selection, $errors) = @_;
1425
1426     my $pid = PVE::LXC::find_lxc_pid($vmid);
1427     my $rootdir = "/proc/$pid/root";
1428
1429     my $add_hotplug_error = sub {
1430         my ($opt, $msg) = @_;
1431         $errors->{$opt} = "unable to hotplug $opt: $msg";
1432     };
1433
1434     foreach my $opt (sort keys %{$conf->{pending}}) { # add/change
1435         next if $selection && !$selection->{$opt};
1436         if ($LXC_FASTPLUG_OPTIONS->{$opt}) {
1437             $conf->{$opt} = delete $conf->{pending}->{$opt};
1438         }
1439     }
1440
1441     my $cgroup = PVE::LXC::CGroup->new($vmid);
1442
1443     # There's no separate swap size to configure, there's memory and "total"
1444     # memory (iow. memory+swap). This means we have to change them together.
1445     my $hotplug_memory_done;
1446     my $hotplug_memory = sub {
1447         my ($new_memory, $new_swap) = @_;
1448
1449         ($new_memory, my $new_memory_high) = calculate_memory_constraints($new_memory);
1450         $new_swap = int($new_swap * 1024 * 1024) if defined($new_swap);
1451         $cgroup->change_memory_limit($new_memory, $new_swap, $new_memory_high);
1452
1453         $hotplug_memory_done = 1;
1454     };
1455
1456     my $pending_delete_hash = $class->parse_pending_delete($conf->{pending}->{delete});
1457     # FIXME: $force deletion is not implemented for CTs
1458     foreach my $opt (sort keys %$pending_delete_hash) {
1459         next if $selection && !$selection->{$opt};
1460         eval {
1461             if ($LXC_FASTPLUG_OPTIONS->{$opt}) {
1462                 # pass
1463             } elsif ($opt =~ m/^unused(\d+)$/) {
1464                 PVE::LXC::delete_mountpoint_volume($storecfg, $vmid, $conf->{$opt})
1465                     if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1466             } elsif ($opt eq 'swap') {
1467                 $hotplug_memory->(undef, 0);
1468             } elsif ($opt eq 'cpulimit') {
1469                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
1470             } elsif ($opt eq 'cpuunits') {
1471                 $cgroup->change_cpu_shares(undef);
1472             } elsif ($opt =~ m/^net(\d)$/) {
1473                 my $netid = $1;
1474                 PVE::Network::veth_delete("veth${vmid}i$netid");
1475                 if ($have_sdn) {
1476                     my $net = PVE::LXC::Config->parse_lxc_network($conf->{$opt});
1477                     print "delete ips from $opt\n";
1478                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{hwaddr}, $conf->{hostname}) };
1479                     warn $@ if $@;
1480                 }
1481             } else {
1482                 die "skip\n"; # skip non-hotpluggable opts
1483             }
1484         };
1485         if (my $err = $@) {
1486             $add_hotplug_error->($opt, $err) if $err ne "skip\n";
1487         } else {
1488             delete $conf->{$opt};
1489             $class->remove_from_pending_delete($conf, $opt);
1490         }
1491     }
1492
1493     foreach my $opt (sort keys %{$conf->{pending}}) {
1494         next if $opt eq 'delete'; # just to be sure
1495         next if $selection && !$selection->{$opt};
1496         my $value = $conf->{pending}->{$opt};
1497         eval {
1498             if ($opt eq 'cpulimit') {
1499                 my $quota = 100000 * $value;
1500                 $cgroup->change_cpu_quota(int(100000 * $value), 100000);
1501             } elsif ($opt eq 'cpuunits') {
1502                 $cgroup->change_cpu_shares($value);
1503             } elsif ($opt =~ m/^net(\d+)$/) {
1504                 my $netid = $1;
1505                 my $net = $class->parse_lxc_network($value);
1506                 $value = $class->print_lxc_network($net);
1507                 PVE::LXC::update_net($vmid, $conf, $opt, $net, $netid, $rootdir);
1508             } elsif ($opt eq 'memory' || $opt eq 'swap') {
1509                 if (!$hotplug_memory_done) { # don't call twice if both opts are passed
1510                     $hotplug_memory->($conf->{pending}->{memory}, $conf->{pending}->{swap});
1511                 }
1512             } elsif ($opt =~ m/^mp(\d+)$/) {
1513                 if (exists($conf->{$opt})) {
1514                     die "skip\n"; # don't try to hotplug over existing mp
1515                 }
1516
1517                 $class->apply_pending_mountpoint($vmid, $conf, $opt, $storecfg, 1);
1518                 # apply_pending_mountpoint modifies the value if it creates a new disk
1519                 $value = $conf->{pending}->{$opt};
1520             } else {
1521                 die "skip\n"; # skip non-hotpluggable
1522             }
1523         };
1524         if (my $err = $@) {
1525             $add_hotplug_error->($opt, $err) if $err ne "skip\n";
1526         } else {
1527             $conf->{$opt} = $value;
1528             delete $conf->{pending}->{$opt};
1529         }
1530     }
1531 }
1532
1533 sub vmconfig_apply_pending {
1534     my ($class, $vmid, $conf, $storecfg, $selection, $errors) = @_;
1535
1536     my $add_apply_error = sub {
1537         my ($opt, $msg) = @_;
1538         my $err_msg = "unable to apply pending change $opt : $msg";
1539         $errors->{$opt} = $err_msg;
1540         warn $err_msg;
1541     };
1542
1543     my $pending_delete_hash = $class->parse_pending_delete($conf->{pending}->{delete});
1544     # FIXME: $force deletion is not implemented for CTs
1545     foreach my $opt (sort keys %$pending_delete_hash) {
1546         next if $selection && !$selection->{$opt};
1547         eval {
1548             if ($opt =~ m/^mp(\d+)$/) {
1549                 my $mp = $class->parse_volume($opt, $conf->{$opt});
1550                 if ($mp->{type} eq 'volume') {
1551                     $class->add_unused_volume($conf, $mp->{volume})
1552                         if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1553                 }
1554             } elsif ($opt =~ m/^unused(\d+)$/) {
1555                 PVE::LXC::delete_mountpoint_volume($storecfg, $vmid, $conf->{$opt})
1556                     if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1557             } elsif ($opt =~ m/^net(\d+)$/) {
1558                 if ($have_sdn) {
1559                     my $net = $class->parse_lxc_network($conf->{$opt});
1560                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{hwaddr}, $conf->{hostname}) };
1561                     warn $@ if $@;
1562                 }
1563             }
1564         };
1565         if (my $err = $@) {
1566             $add_apply_error->($opt, $err);
1567         } else {
1568             delete $conf->{$opt};
1569             $class->remove_from_pending_delete($conf, $opt);
1570         }
1571     }
1572
1573     $class->cleanup_pending($conf);
1574
1575     foreach my $opt (sort keys %{$conf->{pending}}) { # add/change
1576         next if $opt eq 'delete'; # just to be sure
1577         next if $selection && !$selection->{$opt};
1578         eval {
1579             if ($opt =~ m/^mp(\d+)$/) {
1580                 $class->apply_pending_mountpoint($vmid, $conf, $opt, $storecfg, 0);
1581             } elsif ($opt =~ m/^net(\d+)$/) {
1582                 my $netid = $1;
1583                 my $net = $class->parse_lxc_network($conf->{pending}->{$opt});
1584                 $conf->{pending}->{$opt} = $class->print_lxc_network($net);
1585                 if ($have_sdn) {
1586                     if ($conf->{$opt}) {
1587                         my $old_net = $class->parse_lxc_network($conf->{$opt});
1588                         if ($old_net->{bridge} ne $net->{bridge} || $old_net->{hwaddr} ne $net->{hwaddr}) {
1589                             PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{hwaddr}, $conf->{name});
1590                             PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{hostname}, $net->{hwaddr}, $vmid, undef, 1);
1591                         }
1592                     } else {
1593                         PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{hostname}, $net->{hwaddr}, $vmid, undef, 1);
1594                     }
1595                 }
1596             }
1597         };
1598         if (my $err = $@) {
1599             $add_apply_error->($opt, $err);
1600         } else {
1601             $conf->{$opt} = delete $conf->{pending}->{$opt};
1602         }
1603     }
1604 }
1605
1606 my $rescan_volume = sub {
1607     my ($storecfg, $mp) = @_;
1608     eval {
1609         $mp->{size} = PVE::Storage::volume_size_info($storecfg, $mp->{volume}, 5);
1610     };
1611     warn "Could not rescan volume size - $@\n" if $@;
1612 };
1613
1614 sub apply_pending_mountpoint {
1615     my ($class, $vmid, $conf, $opt, $storecfg, $running) = @_;
1616
1617     my $mp = $class->parse_volume($opt, $conf->{pending}->{$opt});
1618     my $old = $conf->{$opt};
1619     if ($mp->{type} eq 'volume' && $mp->{volume} =~ $PVE::LXC::NEW_DISK_RE) {
1620         my $original_value = $conf->{pending}->{$opt};
1621         my $vollist = PVE::LXC::create_disks(
1622             $storecfg,
1623             $vmid,
1624             { $opt => $original_value },
1625             $conf,
1626             1,
1627         );
1628         if ($running) {
1629             # Re-parse mount point:
1630             my $mp = $class->parse_volume($opt, $conf->{pending}->{$opt});
1631             eval {
1632                 PVE::LXC::mountpoint_hotplug($vmid, $conf, $opt, $mp, $storecfg);
1633             };
1634             my $err = $@;
1635             if ($err) {
1636                 PVE::LXC::destroy_disks($storecfg, $vollist);
1637                 # The pending-changes code collects errors but keeps on looping through further
1638                 # pending changes, so unroll the change in $conf as well if destroy_disks()
1639                 # didn't die().
1640                 $conf->{pending}->{$opt} = $original_value;
1641                 die $err;
1642             }
1643         }
1644     } else {
1645         die "skip\n" if $running && defined($old); # TODO: "changing" mount points?
1646         $rescan_volume->($storecfg, $mp) if $mp->{type} eq 'volume';
1647         if ($running) {
1648             PVE::LXC::mountpoint_hotplug($vmid, $conf, $opt, $mp, $storecfg);
1649         }
1650         $conf->{pending}->{$opt} = $class->print_ct_mountpoint($mp);
1651     }
1652
1653     if (defined($old)) {
1654         my $mp = $class->parse_volume($opt, $old);
1655         if ($mp->{type} eq 'volume') {
1656             $class->add_unused_volume($conf, $mp->{volume})
1657                 if !$class->is_volume_in_use($conf, $conf->{$opt}, 1, 1);
1658         }
1659     }
1660 }
1661
1662 sub classify_mountpoint {
1663     my ($class, $vol) = @_;
1664     if ($vol =~ m!^/!) {
1665         return 'device' if $vol =~ m!^/dev/!;
1666         return 'bind';
1667     }
1668     return 'volume';
1669 }
1670
1671 my $__is_volume_in_use = sub {
1672     my ($class, $config, $volid) = @_;
1673     my $used = 0;
1674
1675     $class->foreach_volume($config, sub {
1676         my ($ms, $mountpoint) = @_;
1677         return if $used;
1678         $used = $mountpoint->{type} eq 'volume' && $mountpoint->{volume} eq $volid;
1679     });
1680
1681     return $used;
1682 };
1683
1684 sub is_volume_in_use_by_snapshots {
1685     my ($class, $config, $volid) = @_;
1686
1687     if (my $snapshots = $config->{snapshots}) {
1688         foreach my $snap (keys %$snapshots) {
1689             return 1 if $__is_volume_in_use->($class, $snapshots->{$snap}, $volid);
1690         }
1691     }
1692
1693     return 0;
1694 }
1695
1696 sub is_volume_in_use {
1697     my ($class, $config, $volid, $include_snapshots, $include_pending) = @_;
1698     return 1 if $__is_volume_in_use->($class, $config, $volid);
1699     return 1 if $include_snapshots && $class->is_volume_in_use_by_snapshots($config, $volid);
1700     return 1 if $include_pending && $__is_volume_in_use->($class, $config->{pending}, $volid);
1701     return 0;
1702 }
1703
1704 sub has_dev_console {
1705     my ($class, $conf) = @_;
1706
1707     return !(defined($conf->{console}) && !$conf->{console});
1708 }
1709
1710 sub has_lxc_entry {
1711     my ($class, $conf, $keyname) = @_;
1712
1713     if (my $lxcconf = $conf->{lxc}) {
1714         foreach my $entry (@$lxcconf) {
1715             my ($key, undef) = @$entry;
1716             return 1 if $key eq $keyname;
1717         }
1718     }
1719
1720     return 0;
1721 }
1722
1723 sub get_tty_count {
1724     my ($class, $conf) = @_;
1725
1726     return $conf->{tty} // $confdesc->{tty}->{default};
1727 }
1728
1729 sub get_cmode {
1730     my ($class, $conf) = @_;
1731
1732     return $conf->{cmode} // $confdesc->{cmode}->{default};
1733 }
1734
1735 sub valid_volume_keys {
1736     my ($class, $reverse) = @_;
1737
1738     my @names = ('rootfs');
1739
1740     for (my $i = 0; $i < $MAX_MOUNT_POINTS; $i++) {
1741         push @names, "mp$i";
1742     }
1743
1744     return $reverse ? reverse @names : @names;
1745 }
1746
1747 sub valid_volume_keys_with_unused {
1748     my ($class, $reverse) = @_;
1749     my @names = $class->valid_volume_keys();
1750     for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1751         push @names, "unused$i";
1752     }
1753     return $reverse ? reverse @names : @names;
1754 }
1755
1756 sub get_vm_volumes {
1757     my ($class, $conf, $excludes) = @_;
1758
1759     my $vollist = [];
1760
1761     $class->foreach_volume($conf, sub {
1762         my ($ms, $mountpoint) = @_;
1763
1764         return if $excludes && $ms eq $excludes;
1765
1766         my $volid = $mountpoint->{volume};
1767         return if !$volid || $mountpoint->{type} ne 'volume';
1768
1769         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1770         return if !$sid;
1771
1772         push @$vollist, $volid;
1773     });
1774
1775     return $vollist;
1776 }
1777
1778 sub get_replicatable_volumes {
1779     my ($class, $storecfg, $vmid, $conf, $cleanup, $noerr) = @_;
1780
1781     my $volhash = {};
1782
1783     my $test_volid = sub {
1784         my ($volid, $mountpoint) = @_;
1785
1786         return if !$volid;
1787
1788         my $mptype = $mountpoint->{type};
1789         my $replicate = $mountpoint->{replicate} // 1;
1790
1791         if ($mptype ne 'volume') {
1792             # skip bindmounts if replicate = 0 even for cleanup,
1793             # since bind mounts could not have been replicated ever
1794             return if !$replicate;
1795             die "unable to replicate mountpoint type '$mptype'\n";
1796         }
1797
1798         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, $noerr);
1799         return if !$storeid;
1800
1801         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1802         return if $scfg->{shared};
1803
1804         my ($path, $owner, $vtype) = PVE::Storage::path($storecfg, $volid);
1805         return if !$owner || ($owner != $vmid);
1806
1807         die "unable to replicate volume '$volid', type '$vtype'\n" if $vtype ne 'images';
1808
1809         return if !$cleanup && !$replicate;
1810
1811         if (!PVE::Storage::volume_has_feature($storecfg, 'replicate', $volid)) {
1812             return if $cleanup || $noerr;
1813             die "missing replicate feature on volume '$volid'\n";
1814         }
1815
1816         $volhash->{$volid} = 1;
1817     };
1818
1819     $class->foreach_volume($conf, sub {
1820         my ($ms, $mountpoint) = @_;
1821         $test_volid->($mountpoint->{volume}, $mountpoint);
1822     });
1823
1824     foreach my $snapname (keys %{$conf->{snapshots}}) {
1825         my $snap = $conf->{snapshots}->{$snapname};
1826         $class->foreach_volume($snap, sub {
1827             my ($ms, $mountpoint) = @_;
1828             $test_volid->($mountpoint->{volume}, $mountpoint);
1829         });
1830     }
1831
1832     # add 'unusedX' volumes to volhash
1833     foreach my $key (keys %$conf) {
1834         if ($key =~ m/^unused/) {
1835             $test_volid->($conf->{$key}, { type => 'volume', replicate => 1 });
1836         }
1837     }
1838
1839     return $volhash;
1840 }
1841
1842 sub get_backup_volumes {
1843     my ($class, $conf) = @_;
1844
1845     my $return_volumes = [];
1846
1847     my $test_mountpoint = sub {
1848         my ($key, $volume) = @_;
1849
1850         my ($included, $reason) = $class->mountpoint_backup_enabled($key, $volume);
1851
1852         push @$return_volumes, {
1853             key => $key,
1854             included => $included,
1855             reason => $reason,
1856             volume_config => $volume,
1857         };
1858     };
1859
1860     PVE::LXC::Config->foreach_volume($conf, $test_mountpoint);
1861
1862     return $return_volumes;
1863 }
1864
1865 sub get_derived_property {
1866     my ($class, $conf, $name) = @_;
1867
1868     if ($name eq 'max-cpu') {
1869         return $conf->{cpulimit} || $conf->{cores} || 0;
1870     } elsif ($name eq 'max-memory') {
1871         return ($conf->{memory} || 512) * 1024 * 1024;
1872     } else {
1873         die "unknown derived property - $name\n";
1874     }
1875 }
1876
1877 sub foreach_passthrough_device {
1878     my ($class, $conf, $func, @param) = @_;
1879
1880     for my $key (keys %$conf) {
1881         next if $key !~ m/^dev(\d+)$/;
1882
1883         my $device = $class->parse_device($conf->{$key});
1884
1885         $func->($key, $device, @param);
1886     }
1887 }
1888
1889 1;