PVE/Storage/BTRFSPlugin.pm

   1 package PVE::Storage::BTRFSPlugin;
   2
   3 use strict;
   4 use warnings;
   5
   6 use base qw(PVE::Storage::Plugin);
   7
   8 use Fcntl qw(S_ISDIR O_WRONLY O_CREAT O_EXCL);
   9 use File::Basename qw(dirname);
  10 use File::Path qw(mkpath);
  11 use IO::Dir;
  12 use POSIX qw(EEXIST);
  13
  14 use PVE::Tools qw(run_command dir_glob_foreach);
  15
  16 use PVE::Storage::DirPlugin;
  17
  18 use constant {
  19     BTRFS_FIRST_FREE_OBJECTID => 256,
  20     FS_NOCOW_FL => 0x00800000,
  21     FS_IOC_GETFLAGS => 0x40086602,
  22     FS_IOC_SETFLAGS => 0x80086601,
  23     BTRFS_MAGIC => 0x9123683e,
  24 };
  25
  26 # Configuration (similar to DirPlugin)
  27
  28 sub type {
  29     return 'btrfs';
  30 }
  31
  32 sub plugindata {
  33     return {
  34         content => [
  35             {
  36                 images => 1,
  37                 rootdir => 1,
  38                 vztmpl => 1,
  39                 iso => 1,
  40                 backup => 1,
  41                 snippets => 1,
  42                 none => 1,
  43             },
  44             { images => 1, rootdir => 1 },
  45         ],
  46         format => [ { raw => 1, subvol => 1 }, 'raw', ],
  47     };
  48 }
  49
  50 sub properties {
  51     return {
  52         nocow => {
  53             description => "Set the NOCOW flag on files."
  54                 . " Disables data checksumming and causes data errors to be unrecoverable from"
  55                 . " while allowing direct I/O. Only use this if data does not need to be any more"
  56                 . " safe than on a single ext4 formatted disk with no underlying raid system.",
  57             type => 'boolean',
  58             default => 0,
  59         },
  60     };
  61 }
  62
  63 sub options {
  64     return {
  65         path => { fixed => 1 },
  66         nodes => { optional => 1 },
  67         shared => { optional => 1 },
  68         disable => { optional => 1 },
  69         maxfiles => { optional => 1 },
  70         'prune-backups'=> { optional => 1 },
  71         content => { optional => 1 },
  72         format => { optional => 1 },
  73         is_mountpoint => { optional => 1 },
  74         nocow => { optional => 1 },
  75         mkdir => { optional => 1 },
  76         # TODO: The new variant of mkdir with  `populate` vs `create`...
  77     };
  78 }
  79
  80 # Storage implementation
  81 #
  82 # We use the same volume names are directory plugins, but map *raw* disk image file names into a
  83 # subdirectory.
  84 #
  85 # `vm-VMID-disk-ID.raw`
  86 #   -> `images/VMID/vm-VMID-disk-ID/disk.raw`
  87 #   where the `vm-VMID-disk-ID/` subdirectory is a btrfs subvolume
  88
  89 # Reuse `DirPlugin`'s `check_config`. This simply checks for invalid paths.
  90 sub check_config {
  91     my ($self, $sectionId, $config, $create, $skipSchemaCheck) = @_;
  92     return PVE::Storage::DirPlugin::check_config($self, $sectionId, $config, $create, $skipSchemaCheck);
  93 }
  94
  95 my sub getfsmagic($) {
  96     my ($path) = @_;
  97     # The field type sizes in `struct statfs` are defined in a rather annoying way, and we only
  98     # need the first field, which is a `long` for our supported platforms.
  99     # Should be moved to pve-rs, so this can be the problem of the `libc` crate ;-)
 100     # Just round up and extract what we need:
 101     my $buf = pack('x160');
 102     if (0 != syscall(&PVE::Syscall::SYS_statfs, $path, $buf)) {
 103         die "statfs on '$path' failed - $!\n";
 104     }
 105
 106     return unpack('L!', $buf);
 107 }
 108
 109 my sub assert_btrfs($) {
 110     my ($path) = @_;
 111     die "'$path' is not a btrfs file system\n"
 112         if getfsmagic($path) != BTRFS_MAGIC;
 113 }
 114
 115 sub activate_storage {
 116     my ($class, $storeid, $scfg, $cache) = @_;
 117
 118     my $path = $scfg->{path};
 119     if (!defined($scfg->{mkdir}) || $scfg->{mkdir}) {
 120         mkpath $path;
 121     }
 122
 123     my $mp = PVE::Storage::DirPlugin::parse_is_mountpoint($scfg);
 124     if (defined($mp) && !PVE::Storage::DirPlugin::path_is_mounted($mp, $cache->{mountdata})) {
 125         die "unable to activate storage '$storeid' - directory is expected to be a mount point but"
 126         ." is not mounted: '$mp'\n";
 127     }
 128
 129     assert_btrfs($path); # only assert this stuff now, ensures $path is there and better UX
 130
 131     $class->SUPER::activate_storage($storeid, $scfg, $cache);
 132 }
 133
 134 sub status {
 135     my ($class, $storeid, $scfg, $cache) = @_;
 136     return PVE::Storage::DirPlugin::status($class, $storeid, $scfg, $cache);
 137 }
 138
 139 # TODO: sub get_volume_notes {}
 140
 141 # TODO: sub update_volume_notes {}
 142
 143 # croak would not include the caller from within this module
 144 sub __error {
 145     my ($msg) = @_;
 146     my (undef, $f, $n) = caller(1);
 147     die "$msg at $f: $n\n";
 148 }
 149
 150 # Given a name (eg. `vm-VMID-disk-ID.raw`), take the part up to the format suffix as the name of
 151 # the subdirectory (subvolume).
 152 sub raw_name_to_dir($) {
 153     my ($raw) = @_;
 154
 155     # For the subvolume directory Strip the `.<format>` suffix:
 156     if ($raw =~ /^(.*)\.raw$/) {
 157         return $1;
 158     }
 159
 160     __error "internal error: bad disk name: $raw";
 161 }
 162
 163 sub raw_file_to_subvol($) {
 164     my ($file) = @_;
 165
 166     if ($file =~ m|^(.*)/disk\.raw$|) {
 167         return "$1";
 168     }
 169
 170     __error "internal error: bad raw path: $file";
 171 }
 172
 173 sub filesystem_path {
 174     my ($class, $scfg, $volname, $snapname) = @_;
 175
 176     my ($vtype, $name, $vmid, undef, undef, $isBase, $format) =
 177         $class->parse_volname($volname);
 178
 179     my $path = $class->get_subdir($scfg, $vtype);
 180
 181     $path .= "/$vmid" if $vtype eq 'images';
 182
 183     if (defined($format) && $format eq 'raw') {
 184         my $dir = raw_name_to_dir($name);
 185         if ($snapname) {
 186             $dir .= "\@$snapname";
 187         }
 188         $path .= "/$dir/disk.raw";
 189     } elsif (defined($format) && $format eq 'subvol') {
 190         $path .= "/$name";
 191         if ($snapname) {
 192             $path .= "\@$snapname";
 193         }
 194     } else {
 195         $path .= "/$name";
 196     }
 197
 198     return wantarray ? ($path, $vmid, $vtype) : $path;
 199 }
 200
 201 sub btrfs_cmd {
 202     my ($class, $cmd, $outfunc) = @_;
 203
 204     my $msg = '';
 205     my $func;
 206     if (defined($outfunc)) {
 207         $func = sub {
 208             my $part = &$outfunc(@_);
 209             $msg .= $part if defined($part);
 210         };
 211     } else {
 212         $func = sub { $msg .= "$_[0]\n" };
 213     }
 214     run_command(['btrfs', '-q', @$cmd], errmsg => 'btrfs error', outfunc => $func);
 215
 216     return $msg;
 217 }
 218
 219 sub btrfs_get_subvol_id {
 220     my ($class, $path) = @_;
 221     my $info = $class->btrfs_cmd(['subvolume', 'show', '--', $path]);
 222     if ($info !~ /^\s*(?:Object|Subvolume) ID:\s*(\d+)$/m) {
 223         die "failed to get btrfs subvolume ID from: $info\n";
 224     }
 225     return $1;
 226 }
 227
 228 my sub chattr : prototype($$$) {
 229     my ($fh, $mask, $xor) = @_;
 230
 231     my $flags = pack('L!', 0);
 232     ioctl($fh, FS_IOC_GETFLAGS, $flags) or die "FS_IOC_GETFLAGS failed - $!\n";
 233     $flags = pack('L!', (unpack('L!', $flags) & $mask) ^ $xor);
 234     ioctl($fh, FS_IOC_SETFLAGS, $flags) or die "FS_IOC_SETFLAGS failed - $!\n";
 235     return 1;
 236 }
 237
 238 sub create_base {
 239     my ($class, $storeid, $scfg, $volname) = @_;
 240
 241     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
 242         $class->parse_volname($volname);
 243
 244     my $newname = $name;
 245     $newname =~ s/^vm-/base-/;
 246
 247     # If we're not working with a 'raw' file, which is the only thing that's "different" for btrfs,
 248     # or a subvolume, we forward to the DirPlugin
 249     if ($format ne 'raw' && $format ne 'subvol') {
 250         return PVE::Storage::DirPlugin::create_base(@_);
 251     }
 252
 253     my $path = $class->filesystem_path($scfg, $volname);
 254     my $newvolname = $basename ? "$basevmid/$basename/$vmid/$newname" : "$vmid/$newname";
 255     my $newpath = $class->filesystem_path($scfg, $newvolname);
 256
 257     my $subvol = $path;
 258     my $newsubvol = $newpath;
 259     if ($format eq 'raw') {
 260         $subvol = raw_file_to_subvol($subvol);
 261         $newsubvol = raw_file_to_subvol($newsubvol);
 262     }
 263
 264     rename($subvol, $newsubvol)
 265         || die "rename '$subvol' to '$newsubvol' failed - $!\n";
 266     eval { $class->btrfs_cmd(['property', 'set', $newsubvol, 'ro', 'true']) };
 267     warn $@ if $@;
 268
 269     return $newvolname;
 270 }
 271
 272 sub clone_image {
 273     my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
 274
 275     my ($vtype, $basename, $basevmid, undef, undef, $isBase, $format) =
 276         $class->parse_volname($volname);
 277
 278     # If we're not working with a 'raw' file, which is the only thing that's "different" for btrfs,
 279     # or a subvolume, we forward to the DirPlugin
 280     if ($format ne 'raw' && $format ne 'subvol') {
 281         return PVE::Storage::DirPlugin::clone_image(@_);
 282     }
 283
 284     my $imagedir = $class->get_subdir($scfg, 'images');
 285     $imagedir .= "/$vmid";
 286     mkpath $imagedir;
 287
 288     my $path = $class->filesystem_path($scfg, $volname);
 289     my $newname = $class->find_free_diskname($storeid, $scfg, $vmid, $format, 1);
 290
 291     # For btrfs subvolumes we don't actually need the "link":
 292     #my $newvolname = "$basevmid/$basename/$vmid/$newname";
 293     my $newvolname = "$vmid/$newname";
 294     my $newpath = $class->filesystem_path($scfg, $newvolname);
 295
 296     my $subvol = $path;
 297     my $newsubvol = $newpath;
 298     if ($format eq 'raw') {
 299         $subvol = raw_file_to_subvol($subvol);
 300         $newsubvol = raw_file_to_subvol($newsubvol);
 301     }
 302
 303     $class->btrfs_cmd(['subvolume', 'snapshot', '--', $subvol, $newsubvol]);
 304
 305     return $newvolname;
 306 }
 307
 308 sub alloc_image {
 309     my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
 310
 311     if ($fmt ne 'raw' && $fmt ne 'subvol') {
 312         return $class->SUPER::alloc_image($storeid, $scfg, $vmid, $fmt, $name, $size);
 313     }
 314
 315     # From Plugin.pm:
 316
 317     my $imagedir = $class->get_subdir($scfg, 'images') . "/$vmid";
 318
 319     mkpath $imagedir;
 320
 321     $name = $class->find_free_diskname($storeid, $scfg, $vmid, $fmt, 1) if !$name;
 322
 323     my (undef, $tmpfmt) = PVE::Storage::Plugin::parse_name_dir($name);
 324
 325     die "illegal name '$name' - wrong extension for format ('$tmpfmt != '$fmt')\n"
 326         if $tmpfmt ne $fmt;
 327
 328     # End copy from Plugin.pm
 329
 330     my $subvol = "$imagedir/$name";
 331     # .raw is not part of the directory name
 332     $subvol =~ s/\.raw$//;
 333
 334     die "disk image '$subvol' already exists\n" if -e $subvol;
 335
 336     my $path;
 337     if ($fmt eq 'raw') {
 338         $path = "$subvol/disk.raw";
 339     }
 340
 341     if ($fmt eq 'subvol' && !!$size) {
 342         # NOTE: `btrfs send/recv` actually drops quota information so supporting subvolumes with
 343         # quotas doesn't play nice with send/recv.
 344         die "btrfs quotas are currently not supported, use an unsized subvolume or a raw file\n";
 345     }
 346
 347     $class->btrfs_cmd(['subvolume', 'create', '--', $subvol]);
 348
 349     eval {
 350         if ($fmt eq 'subvol') {
 351             # Nothing to do for now...
 352
 353             # This is how we *would* do it:
 354             # # Use the subvol's default 0/$id qgroup
 355             # eval {
 356             #     # This call should happen at storage creation instead and therefore governed by a
 357             #     # configuration option!
 358             #     # $class->btrfs_cmd(['quota', 'enable', $subvol]);
 359             #     my $id = $class->btrfs_get_subvol_id($subvol);
 360             #     $class->btrfs_cmd(['qgroup', 'limit', "${size}k", "0/$id", $subvol]);
 361             # };
 362         } elsif ($fmt eq 'raw') {
 363             sysopen my $fh, $path, O_WRONLY | O_CREAT | O_EXCL
 364                 or die "failed to create raw file '$path' - $!\n";
 365             chattr($fh, ~FS_NOCOW_FL, FS_NOCOW_FL) if $scfg->{nocow};
 366             truncate($fh, $size * 1024)
 367                 or die "failed to set file size for '$path' - $!\n";
 368             close($fh);
 369         } else {
 370             die "internal format error (format = $fmt)\n";
 371         }
 372     };
 373
 374     if (my $err = $@) {
 375         eval { $class->btrfs_cmd(['subvolume', 'delete', '--', $subvol]); };
 376         warn $@ if $@;
 377         die $err;
 378     }
 379
 380     return "$vmid/$name";
 381 }
 382
 383 # Same as btrfsprogs does:
 384 my sub path_is_subvolume : prototype($) {
 385     my ($path) = @_;
 386     my @stat = stat($path)
 387         or die "stat failed on '$path' - $!\n";
 388     my ($ino, $mode) = @stat[1, 2];
 389     return S_ISDIR($mode) && $ino == BTRFS_FIRST_FREE_OBJECTID;
 390 }
 391
 392 my $BTRFS_VOL_REGEX = qr/((?:vm|base|subvol)-\d+-disk-\d+(?:\.subvol)?)(?:\@(\S+))$/;
 393
 394 # Calls `$code->($volume, $name, $snapshot)` for each subvol in a directory matching our volume
 395 # regex.
 396 my sub foreach_subvol : prototype($$) {
 397     my ($dir, $code) = @_;
 398
 399     dir_glob_foreach($dir, $BTRFS_VOL_REGEX, sub {
 400         my ($volume, $name, $snapshot) = ($1, $2, $3);
 401         return if !path_is_subvolume("$dir/$volume");
 402         $code->($volume, $name, $snapshot);
 403     })
 404 }
 405
 406 sub free_image {
 407     my ($class, $storeid, $scfg, $volname, $isBase, $_format) = @_;
 408
 409     my (undef, undef, $vmid, undef, undef, undef, $format) =
 410         $class->parse_volname($volname);
 411
 412     if (!defined($format) || ($format ne 'subvol' && $format ne 'raw')) {
 413         return $class->SUPER::free_image($storeid, $scfg, $volname, $isBase, $_format);
 414     }
 415
 416     my $path = $class->filesystem_path($scfg, $volname);
 417
 418     my $subvol = $path;
 419     if ($format eq 'raw') {
 420         $subvol = raw_file_to_subvol($path);
 421     }
 422
 423     my $dir = dirname($subvol);
 424     my @snapshot_vols;
 425     foreach_subvol($dir, sub {
 426         my ($volume, $name, $snapshot) = @_;
 427         return if !defined $snapshot;
 428         push @snapshot_vols, "$dir/$volume";
 429     });
 430
 431     $class->btrfs_cmd(['subvolume', 'delete', '--', @snapshot_vols, $subvol]);
 432     # try to cleanup directory to not clutter storage with empty $vmid dirs if
 433     # all images from a guest got deleted
 434     rmdir($dir);
 435
 436     return undef;
 437 }
 438
 439 # Currently not used because quotas clash with send/recv.
 440 # my sub btrfs_subvol_quota {
 441 #     my ($class, $path) = @_;
 442 #     my $id = '0/' . $class->btrfs_get_subvol_id($path);
 443 #     my $search = qr/^\Q$id\E\s+(\d)+\s+\d+\s+(\d+)\s*$/;
 444 #     my ($used, $size);
 445 #     $class->btrfs_cmd(['qgroup', 'show', '--raw', '-rf', '--', $path], sub {
 446 #       return if defined($size);
 447 #       if ($_[0] =~ $search) {
 448 #           ($used, $size) = ($1, $2);
 449 #       }
 450 #     });
 451 #     if (!defined($size)) {
 452 #       # syslog should include more information:
 453 #       syslog('err', "failed to get subvolume size for: $path (id $id)");
 454 #       # UI should only see the last path component:
 455 #       $path =~ s|^.*/||;
 456 #       die "failed to get subvolume size for $path\n";
 457 #     }
 458 #     return wantarray ? ($used, $size) : $size;
 459 # }
 460
 461 sub volume_size_info {
 462     my ($class, $scfg, $storeid, $volname, $timeout) = @_;
 463
 464     my $path = $class->filesystem_path($scfg, $volname);
 465
 466     my $format = ($class->parse_volname($volname))[6];
 467
 468     if (defined($format) && $format eq 'subvol') {
 469         my $ctime = (stat($path))[10];
 470         my ($used, $size) = (0, 0);
 471         #my ($used, $size) = btrfs_subvol_quota($class, $path); # uses wantarray
 472         return wantarray ? ($size, 'subvol', $used, undef, $ctime) : 1;
 473     }
 474
 475     return PVE::Storage::Plugin::file_size_info($path, $timeout);
 476 }
 477
 478 sub volume_resize {
 479     my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
 480
 481     my $format = ($class->parse_volname($volname))[6];
 482     if ($format eq 'subvol') {
 483         my $path = $class->filesystem_path($scfg, $volname);
 484         my $id = '0/' . $class->btrfs_get_subvol_id($path);
 485         $class->btrfs_cmd(['qgroup', 'limit', '--', "${size}k", "0/$id", $path]);
 486         return undef;
 487     }
 488
 489     return PVE::Storage::Plugin::volume_resize(@_);
 490 }
 491
 492 sub volume_snapshot {
 493     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 494
 495     my ($name, $vmid, $format) = ($class->parse_volname($volname))[1,2,6];
 496     if ($format ne 'subvol' && $format ne 'raw') {
 497         return PVE::Storage::Plugin::volume_snapshot(@_);
 498     }
 499
 500     my $path = $class->filesystem_path($scfg, $volname);
 501     my $snap_path = $class->filesystem_path($scfg, $volname, $snap);
 502
 503     if ($format eq 'raw') {
 504         $path = raw_file_to_subvol($path);
 505         $snap_path = raw_file_to_subvol($snap_path);
 506     }
 507
 508     my $snapshot_dir = $class->get_subdir($scfg, 'images') . "/$vmid";
 509     mkpath $snapshot_dir;
 510
 511     $class->btrfs_cmd(['subvolume', 'snapshot', '-r', '--', $path, $snap_path]);
 512     return undef;
 513 }
 514
 515 sub volume_rollback_is_possible {
 516     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 517
 518     return 1;
 519 }
 520
 521 sub volume_snapshot_rollback {
 522     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 523
 524     my ($name, $format) = ($class->parse_volname($volname))[1,6];
 525
 526     if ($format ne 'subvol' && $format ne 'raw') {
 527         return PVE::Storage::Plugin::volume_snapshot_rollback(@_);
 528     }
 529
 530     my $path = $class->filesystem_path($scfg, $volname);
 531     my $snap_path = $class->filesystem_path($scfg, $volname, $snap);
 532
 533     if ($format eq 'raw') {
 534         $path = raw_file_to_subvol($path);
 535         $snap_path = raw_file_to_subvol($snap_path);
 536     }
 537
 538     # Simple version would be:
 539     #   rename old to temp
 540     #   create new
 541     #   on error rename temp back
 542     # But for atomicity in case the rename after create-failure *also* fails, we create the new
 543     # subvol first, then use RENAME_EXCHANGE,
 544     my $tmp_path = "$path.tmp.$$";
 545     $class->btrfs_cmd(['subvolume', 'snapshot', '--', $snap_path, $tmp_path]);
 546     # The paths are absolute, so pass -1 as file descriptors.
 547     my $ok = PVE::Tools::renameat2(-1, $tmp_path, -1, $path, &PVE::Tools::RENAME_EXCHANGE);
 548
 549     eval { $class->btrfs_cmd(['subvolume', 'delete', '--', $tmp_path]) };
 550     warn "failed to remove '$tmp_path' subvolume: $@" if $@;
 551
 552     if (!$ok) {
 553         die "failed to rotate '$tmp_path' into place at '$path' - $!\n";
 554     }
 555
 556     return undef;
 557 }
 558
 559 sub volume_snapshot_delete {
 560     my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 561
 562     my ($name, $vmid, $format) = ($class->parse_volname($volname))[1,2,6];
 563
 564     if ($format ne 'subvol' && $format ne 'raw') {
 565         return PVE::Storage::Plugin::volume_snapshot_delete(@_);
 566     }
 567
 568     my $path = $class->filesystem_path($scfg, $volname, $snap);
 569
 570     if ($format eq 'raw') {
 571         $path = raw_file_to_subvol($path);
 572     }
 573
 574     $class->btrfs_cmd(['subvolume', 'delete', '--', $path]);
 575
 576     return undef;
 577 }
 578
 579 sub volume_has_feature {
 580     my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
 581
 582     my $features = {
 583         snapshot => {
 584             current => { qcow2 => 1, raw => 1, subvol => 1 },
 585             snap => { qcow2 => 1, raw => 1, subvol => 1 }
 586         },
 587         clone => {
 588             base => { qcow2 => 1, raw => 1, subvol => 1, vmdk => 1 },
 589             current => { raw => 1 },
 590             snap => { raw => 1 },
 591         },
 592         template => { current => { qcow2 => 1, raw => 1, vmdk => 1, subvol => 1 } },
 593         copy => {
 594             base => { qcow2 => 1, raw => 1, subvol => 1, vmdk => 1 },
 595             current => { qcow2 => 1, raw => 1, subvol => 1, vmdk => 1 },
 596             snap => { qcow2 => 1, raw => 1, subvol => 1 },
 597         },
 598         sparseinit => { base => {qcow2 => 1, raw => 1, vmdk => 1 },
 599                         current => {qcow2 => 1, raw => 1, vmdk => 1 } },
 600     };
 601
 602     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
 603         $class->parse_volname($volname);
 604
 605     my $key = undef;
 606     if ($snapname) {
 607         $key = 'snap';
 608     } else {
 609         $key =  $isBase ? 'base' : 'current';
 610     }
 611
 612     return 1 if defined($features->{$feature}->{$key}->{$format});
 613
 614     return undef;
 615 }
 616
 617 sub list_images {
 618     my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
 619     my $imagedir = $class->get_subdir($scfg, 'images');
 620
 621     my $res = [];
 622
 623     # Copied from Plugin.pm, with file_size_info calls adapted:
 624     foreach my $fn (<$imagedir/[0-9][0-9]*/*>) {
 625         # different to in Plugin.pm the regex below also excludes '@' as valid file name
 626         next if $fn !~ m@^(/.+/(\d+)/([^/\@.]+(?:\.(qcow2|vmdk|subvol))?))$@;
 627         $fn = $1; # untaint
 628
 629         my $owner = $2;
 630         my $name = $3;
 631         my $ext = $4;
 632
 633         next if !$vollist && defined($vmid) && ($owner ne $vmid);
 634
 635         my $volid = "$storeid:$owner/$name";
 636         my ($size, $format, $used, $parent, $ctime);
 637
 638         if (!$ext) { # raw
 639             $volid .= '.raw';
 640             ($size, $format, $used, $parent, $ctime) = PVE::Storage::Plugin::file_size_info("$fn/disk.raw");
 641         } elsif ($ext eq 'subvol') {
 642             ($used, $size) = (0, 0);
 643             #($used, $size) = btrfs_subvol_quota($class, $fn);
 644             $format = 'subvol';
 645         } else {
 646             ($size, $format, $used, $parent, $ctime) = PVE::Storage::Plugin::file_size_info($fn);
 647         }
 648         next if !($format && defined($size));
 649
 650         if ($vollist) {
 651             next if ! grep { $_ eq $volid } @$vollist;
 652         }
 653
 654         my $info = {
 655             volid => $volid, format => $format,
 656             size => $size, vmid => $owner, used => $used, parent => $parent,
 657         };
 658
 659         $info->{ctime} = $ctime if $ctime;
 660
 661         push @$res, $info;
 662     }
 663
 664     return $res;
 665 }
 666
 667 sub volume_export_formats {
 668     my ($class, $scfg, $storeid, $volname, $snapshot, $base_snapshot, $with_snapshots) = @_;
 669
 670     # We can do whatever `DirPlugin` can do.
 671     my @result = PVE::Storage::Plugin::volume_export_formats(@_);
 672
 673     # `btrfs send` only works on snapshots:
 674     return @result if !defined $snapshot;
 675
 676     # Incremental stream with snapshots is only supported if the snapshots are listed (new api):
 677     return @result if defined($base_snapshot) && $with_snapshots && ref($with_snapshots) ne 'ARRAY';
 678
 679     # Otherwise we do also support `with_snapshots`.
 680
 681     # Finally, `btrfs send` only works on formats where we actually use btrfs subvolumes:
 682     my $format = ($class->parse_volname($volname))[6];
 683     return @result if $format ne 'raw' && $format ne 'subvol';
 684
 685     return ('btrfs', @result);
 686 }
 687
 688 sub volume_import_formats {
 689     my ($class, $scfg, $storeid, $volname, $snapshot, $base_snapshot, $with_snapshots) = @_;
 690
 691     # Same as export-formats, beware the parameter order:
 692     return volume_export_formats(
 693         $class,
 694         $scfg,
 695         $storeid,
 696         $volname,
 697         $snapshot,
 698         $base_snapshot,
 699         $with_snapshots,
 700     );
 701 }
 702
 703 sub volume_export {
 704     my (
 705         $class,
 706         $scfg,
 707         $storeid,
 708         $fh,
 709         $volname,
 710         $format,
 711         $snapshot,
 712         $base_snapshot,
 713         $with_snapshots,
 714     ) = @_;
 715
 716     if ($format ne 'btrfs') {
 717         return PVE::Storage::Plugin::volume_export(@_);
 718     }
 719
 720     die "format 'btrfs' only works on snapshots\n"
 721         if !defined $snapshot;
 722
 723     die "'btrfs' format in incremental mode requires snapshots to be listed explicitly\n"
 724         if defined($base_snapshot) && $with_snapshots && ref($with_snapshots) ne 'ARRAY';
 725
 726     my $volume_format = ($class->parse_volname($volname))[6];
 727
 728     die "btrfs-sending volumes of type $volume_format ('$volname') is not supported\n"
 729         if $volume_format ne 'raw' && $volume_format ne 'subvol';
 730
 731     my $path = $class->path($scfg, $volname, $storeid);
 732
 733     if ($volume_format eq 'raw') {
 734         $path = raw_file_to_subvol($path);
 735     }
 736
 737     my $cmd = ['btrfs', '-q', 'send', '-e'];
 738     if ($base_snapshot) {
 739         my $base = $class->path($scfg, $volname, $storeid, $base_snapshot);
 740         if ($volume_format eq 'raw') {
 741             $base = raw_file_to_subvol($base);
 742         }
 743         push @$cmd, '-p', $base;
 744     }
 745     push @$cmd, '--';
 746     if (ref($with_snapshots) eq 'ARRAY') {
 747         push @$cmd, (map { "$path\@$_" } ($with_snapshots // [])->@*), $path;
 748     } else {
 749         dir_glob_foreach(dirname($path), $BTRFS_VOL_REGEX, sub {
 750             push @$cmd, "$path\@$_[2]" if !(defined($snapshot) && $_[2] eq $snapshot);
 751         });
 752     }
 753     $path .= "\@$snapshot" if defined($snapshot);
 754     push @$cmd, $path;
 755
 756     run_command($cmd, output => '>&'.fileno($fh));
 757     return;
 758 }
 759
 760 sub volume_import {
 761     my (
 762         $class,
 763         $scfg,
 764         $storeid,
 765         $fh,
 766         $volname,
 767         $format,
 768         $snapshot,
 769         $base_snapshot,
 770         $with_snapshots,
 771         $allow_rename,
 772     ) = @_;
 773
 774     if ($format ne 'btrfs') {
 775         return PVE::Storage::Plugin::volume_import(@_);
 776     }
 777
 778     die "format 'btrfs' only works on snapshots\n"
 779         if !defined $snapshot;
 780
 781     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $volume_format) =
 782         $class->parse_volname($volname);
 783
 784     die "btrfs-receiving volumes of type $volume_format ('$volname') is not supported\n"
 785         if $volume_format ne 'raw' && $volume_format ne 'subvol';
 786
 787     if (defined($base_snapshot)) {
 788         my $path = $class->path($scfg, $volname, $storeid, $base_snapshot);
 789         die "base snapshot '$base_snapshot' not found - no such directory '$path'\n"
 790             if !path_is_subvolume($path);
 791     }
 792
 793     my $destination = $class->filesystem_path($scfg, $volname);
 794     if ($volume_format eq 'raw') {
 795         $destination = raw_file_to_subvol($destination);
 796     }
 797
 798     if (!defined($base_snapshot) && -e $destination) {
 799         die "volume $volname already exists\n" if !$allow_rename;
 800         $volname = $class->find_free_diskname($storeid, $scfg, $vmid, $volume_format, 1);
 801     }
 802
 803     my $imagedir = $class->get_subdir($scfg, $vtype);
 804     $imagedir .= "/$vmid" if $vtype eq 'images';
 805
 806     my $tmppath = "$imagedir/recv.$vmid.tmp";
 807     mkdir($imagedir); # FIXME: if $scfg->{mkdir};
 808     if (!mkdir($tmppath)) {
 809         die "temp receive directory already exists at '$tmppath', incomplete concurrent import?\n"
 810             if $! == EEXIST;
 811         die "failed to create temporary receive directory at '$tmppath' - $!\n";
 812     }
 813
 814     my $dh = IO::Dir->new($tmppath)
 815         or die "failed to open temporary receive directory '$tmppath' - $!\n";
 816     eval {
 817         run_command(['btrfs', '-q', 'receive', '-e', '--', $tmppath], input => '<&'.fileno($fh));
 818
 819         # Analyze the received subvolumes;
 820         my ($diskname, $found_snapshot, @snapshots);
 821         $dh->rewind;
 822         while (defined(my $entry = $dh->read)) {
 823             next if $entry eq '.' || $entry eq '..';
 824             next if $entry !~ /^$BTRFS_VOL_REGEX$/;
 825             my ($cur_diskname, $cur_snapshot) = ($1, $2);
 826
 827             die "send stream included a non-snapshot subvolume\n"
 828                 if !defined($cur_snapshot);
 829
 830             if (!defined($diskname)) {
 831                 $diskname = $cur_diskname;
 832             } else {
 833                 die "multiple disks contained in stream ('$diskname' vs '$cur_diskname')\n"
 834                     if $diskname ne $cur_diskname;
 835             }
 836
 837             if ($cur_snapshot eq $snapshot) {
 838                 $found_snapshot = 1;
 839             } else {
 840                 push @snapshots, $cur_snapshot;
 841             }
 842         }
 843
 844         die "send stream did not contain the expected current snapshot '$snapshot'\n"
 845             if !$found_snapshot;
 846
 847         # Rotate the disk into place, first the current state:
 848         # Note that read-only subvolumes cannot be moved into different directories, but for the
 849         # "current" state we also want a writable copy, so start with that:
 850         $class->btrfs_cmd(['property', 'set', "$tmppath/$diskname\@$snapshot", 'ro', 'false']);
 851         PVE::Tools::renameat2(
 852             -1,
 853             "$tmppath/$diskname\@$snapshot",
 854             -1,
 855             $destination,
 856             &PVE::Tools::RENAME_NOREPLACE,
 857         ) or die "failed to move received snapshot '$tmppath/$diskname\@$snapshot'"
 858             . " into place at '$destination' - $!\n";
 859
 860         # Now recreate the actual snapshot:
 861         $class->btrfs_cmd([
 862             'subvolume',
 863             'snapshot',
 864             '-r',
 865             '--',
 866             $destination,
 867             "$destination\@$snapshot",
 868         ]);
 869
 870         # Now go through the remaining snapshots (if any)
 871         foreach my $snap (@snapshots) {
 872             $class->btrfs_cmd(['property', 'set', "$tmppath/$diskname\@$snap", 'ro', 'false']);
 873             PVE::Tools::renameat2(
 874                 -1,
 875                 "$tmppath/$diskname\@$snap",
 876                 -1,
 877                 "$destination\@$snap",
 878                 &PVE::Tools::RENAME_NOREPLACE,
 879             ) or die "failed to move received snapshot '$tmppath/$diskname\@$snap'"
 880                 . " into place at '$destination\@$snap' - $!\n";
 881             eval { $class->btrfs_cmd(['property', 'set', "$destination\@$snap", 'ro', 'true']) };
 882             warn "failed to make $destination\@$snap read-only - $!\n" if $@;
 883         }
 884     };
 885     my $err = $@;
 886
 887     eval {
 888         # Cleanup all the received snapshots we did not move into place, so we can remove the temp
 889         # directory.
 890         if ($dh) {
 891             $dh->rewind;
 892             while (defined(my $entry = $dh->read)) {
 893                 next if $entry eq '.' || $entry eq '..';
 894                 eval { $class->btrfs_cmd(['subvolume', 'delete', '--', "$tmppath/$entry"]) };
 895                 warn $@ if $@;
 896             }
 897             $dh->close; undef $dh;
 898         }
 899         if (!rmdir($tmppath)) {
 900             warn "failed to remove temporary directory '$tmppath' - $!\n"
 901         }
 902     };
 903     warn $@ if $@;
 904     if ($err) {
 905         # clean up if the directory ended up being empty after an error
 906         rmdir($tmppath);
 907         die $err;
 908     }
 909
 910     return "$storeid:$volname";
 911 }
 912
 913 1