PVE/Storage/BTRFSPlugin.pm

   1 package PVE::Storage::BTRFSPlugin;
   2
   3 use strict;
   4 use warnings;
   5
   6 use base qw(PVE::Storage::Plugin);
   7
   8 use Fcntl qw(S_ISDIR O_WRONLY O_CREAT O_EXCL);
   9 use File::Basename qw(dirname);
  10 use File::Path qw(mkpath);
  11 use IO::Dir;
  12 use POSIX qw(EEXIST);
  13
  14 use PVE::Tools qw(run_command dir_glob_foreach);
  15
  16 use PVE::Storage::DirPlugin;
  17
  18 use constant {
  19     BTRFS_FIRST_FREE_OBJECTID => 256,
  20     FS_NOCOW_FL => 0x00800000,
  21     FS_IOC_GETFLAGS => 0x40086602,
  22     FS_IOC_SETFLAGS => 0x80086601,
  23 };
  24
  25 # Configuration (similar to DirPlugin)
  26
  27 sub type {
  28     return 'btrfs';
  29 }
  30
  31 sub plugindata {
  32     return {
  33         content => [
  34             {
  35                 images => 1,
  36                 rootdir => 1,
  37                 vztmpl => 1,
  38                 iso => 1,
  39                 backup => 1,
  40                 snippets => 1,
  41                 none => 1,
  42             },
  43             { images => 1, rootdir => 1 },
  44         ],
  45         format => [ { raw => 1, subvol => 1 }, 'raw', ],
  46     };
  47 }
  48
  49 sub properties {
  50     return {
  51         nocow => {
  52             description => "Set the NOCOW flag on files."
  53                 . " Disables data checksumming and causes data errors to be unrecoverable from"
  54                 . " while allowing direct I/O. Only use this if data does not need to be any more"
  55                 . " safe than on a single ext4 formatted disk with no underlying raid system.",
  56             type => 'boolean',
  57             default => 0,
  58         },
  59     };
  60 }
  61
  62 sub options {
  63     return {
  64         path => { fixed => 1 },
  65         nodes => { optional => 1 },
  66         shared => { optional => 1 },
  67         disable => { optional => 1 },
  68         maxfiles => { optional => 1 },
  69         content => { optional => 1 },
  70         format => { optional => 1 },
  71         is_mountpoint => { optional => 1 },
  72         nocow => { optional => 1 },
  73         # TODO: The new variant of mkdir with  `populate` vs `create`...
  74     };
  75 }
  76
  77 # Storage implementation
  78 #
  79 # We use the same volume names are directory plugins, but map *raw* disk image file names into a
  80 # subdirectory.
  81 #
  82 # `vm-VMID-disk-ID.raw`
  83 #   -> `images/VMID/vm-VMID-disk-ID/disk.raw`
  84 #   where the `vm-VMID-disk-ID/` subdirectory is a btrfs subvolume
  85
  86 # Reuse `DirPlugin`'s `check_config`. This simply checks for invalid paths.
  87 sub check_config {
  88     my ($self, $sectionId, $config, $create, $skipSchemaCheck) = @_;
  89     return PVE::Storage::DirPlugin::check_config($self, $sectionId, $config, $create, $skipSchemaCheck);
  90 }
  91
  92 sub activate_storage {
  93     my ($class, $storeid, $scfg, $cache) = @_;
  94     return PVE::Storage::DirPlugin::activate_storage($class, $storeid, $scfg, $cache);
  95 }
  96
  97 sub status {
  98     my ($class, $storeid, $scfg, $cache) = @_;
  99     return PVE::Storage::DirPlugin::status($class, $storeid, $scfg, $cache);
 100 }
 101
 102 # TODO: sub get_volume_notes {}
 103
 104 # TODO: sub update_volume_notes {}
 105
 106 # croak would not include the caller from within this module
 107 sub __error {
 108     my ($msg) = @_;
 109     my (undef, $f, $n) = caller(1);
 110     die "$msg at $f: $n\n";
 111 }
 112
 113 # Given a name (eg. `vm-VMID-disk-ID.raw`), take the part up to the format suffix as the name of
 114 # the subdirectory (subvolume).
 115 sub raw_name_to_dir($) {
 116     my ($raw) = @_;
 117
 118     # For the subvolume directory Strip the `.<format>` suffix:
 119     if ($raw =~ /^(.*)\.raw$/) {
 120         return $1;
 121     }
 122
 123     __error "internal error: bad disk name: $raw";
 124 }
 125
 126 sub raw_file_to_subvol($) {
 127     my ($file) = @_;
 128
 129     if ($file =~ m|^(.*)/disk\.raw$|) {
 130         return "$1";
 131     }
 132
 133     __error "internal error: bad raw path: $file";
 134 }
 135
 136 sub filesystem_path {
 137     my ($class, $scfg, $volname, $snapname) = @_;
 138
 139     my ($vtype, $name, $vmid, undef, undef, $isBase, $format) =
 140         $class->parse_volname($volname);
 141
 142     my $path = $class->get_subdir($scfg, $vtype);
 143
 144     $path .= "/$vmid" if $vtype eq 'images';
 145
 146     if ($format eq 'raw') {
 147         my $dir = raw_name_to_dir($name);
 148         if ($snapname) {
 149             $dir .= "\@$snapname";
 150         }
 151         $path .= "/$dir/disk.raw";
 152     } elsif ($format eq 'subvol') {
 153         $path .= "/$name";
 154         if ($snapname) {
 155             $path .= "\@$snapname";
 156         }
 157     } else {
 158         $path .= "/$name";
 159     }
 160
 161     return wantarray ? ($path, $vmid, $vtype) : $path;
 162 }
 163
 164 sub btrfs_cmd {
 165     my ($class, $cmd, $outfunc) = @_;
 166
 167     my $msg = '';
 168     my $func;
 169     if (defined($outfunc)) {
 170         $func = sub {
 171             my $part = &$outfunc(@_);
 172             $msg .= $part if defined($part);
 173         };
 174     } else {
 175         $func = sub { $msg .= "$_[0]\n" };
 176     }
 177     run_command(['btrfs', '-q', @$cmd], errmsg => 'btrfs error', outfunc => $func);
 178
 179     return $msg;
 180 }
 181
 182 sub btrfs_get_subvol_id {
 183     my ($class, $path) = @_;
 184     my $info = $class->btrfs_cmd(['subvolume', 'show', '--', $path]);
 185     if ($info !~ /^\s*(?:Object|Subvolume) ID:\s*(\d+)$/m) {
 186         die "failed to get btrfs subvolume ID from: $info\n";
 187     }
 188     return $1;
 189 }
 190
 191 my sub chattr : prototype($$$) {
 192     my ($fh, $mask, $xor) = @_;
 193
 194     my $flags = pack('L!', 0);
 195     ioctl($fh, FS_IOC_GETFLAGS, $flags) or die "FS_IOC_GETFLAGS failed - $!\n";
 196     $flags = pack('L!', (unpack('L!', $flags) & $mask) ^ $xor);
 197     ioctl($fh, FS_IOC_SETFLAGS, $flags) or die "FS_IOC_SETFLAGS failed - $!\n";
 198     return 1;
 199 }
 200
 201 sub create_base {
 202     my ($class, $storeid, $scfg, $volname) = @_;
 203
 204     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
 205         $class->parse_volname($volname);
 206
 207     my $newname = $name;
 208     $newname =~ s/^vm-/base-/;
 209
 210     # If we're not working with a 'raw' file, which is the only thing that's "different" for btrfs,
 211     # or a subvolume, we forward to the DirPlugin
 212     if ($format ne 'raw' && $format ne 'subvol') {
 213         return PVE::Storage::DirPlugin::create_base(@_);
 214     }
 215
 216     my $path = $class->filesystem_path($scfg, $volname);
 217     my $newvolname = $basename ? "$basevmid/$basename/$vmid/$newname" : "$vmid/$newname";
 218     my $newpath = $class->filesystem_path($scfg, $newvolname);
 219
 220     my $subvol = $path;
 221     my $newsubvol = $newpath;
 222     if ($format eq 'raw') {
 223         $subvol = raw_file_to_subvol($subvol);
 224         $newsubvol = raw_file_to_subvol($newsubvol);
 225     }
 226
 227     rename($subvol, $newsubvol)
 228         || die "rename '$subvol' to '$newsubvol' failed - $!\n";
 229     eval { $class->btrfs_cmd(['property', 'set', $newsubvol, 'ro', 'true']) };
 230     warn $@ if $@;
 231
 232     return $newvolname;
 233 }
 234
 235 sub clone_image {
 236     my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
 237
 238     my ($vtype, $basename, $basevmid, undef, undef, $isBase, $format) =
 239         $class->parse_volname($volname);
 240
 241     # If we're not working with a 'raw' file, which is the only thing that's "different" for btrfs,
 242     # or a subvolume, we forward to the DirPlugin
 243     if ($format ne 'raw' && $format ne 'subvol') {
 244         return PVE::Storage::DirPlugin::clone_image(@_);
 245     }
 246
 247     my $imagedir = $class->get_subdir($scfg, 'images');
 248     $imagedir .= "/$vmid";
 249     mkpath $imagedir;
 250
 251     my $path = $class->filesystem_path($scfg, $volname);
 252     my $newname = $class->find_free_diskname($storeid, $scfg, $vmid, $format, 1);
 253
 254     # For btrfs subvolumes we don't actually need the "link":
 255     #my $newvolname = "$basevmid/$basename/$vmid/$newname";
 256     my $newvolname = "$vmid/$newname";
 257     my $newpath = $class->filesystem_path($scfg, $newvolname);
 258
 259     my $subvol = $path;
 260     my $newsubvol = $newpath;
 261     if ($format eq 'raw') {
 262         $subvol = raw_file_to_subvol($subvol);
 263         $newsubvol = raw_file_to_subvol($newsubvol);
 264     }
 265
 266     $class->btrfs_cmd(['subvolume', 'snapshot', '--', $subvol, $newsubvol]);
 267
 268     return $newvolname;
 269 }
 270
 271 sub alloc_image {
 272     my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
 273
 274     if ($fmt ne 'raw' && $fmt ne 'subvol') {
 275         return PVE::Storage::DirPlugin::alloc_image(@_);
 276     }
 277
 278     # From Plugin.pm:
 279
 280     my $imagedir = $class->get_subdir($scfg, 'images') . "/$vmid";
 281
 282     mkpath $imagedir;
 283
 284     $name = $class->find_free_diskname($storeid, $scfg, $vmid, $fmt, 1) if !$name;
 285
 286     my (undef, $tmpfmt) = PVE::Storage::Plugin::parse_name_dir($name);
 287
 288     die "illegal name '$name' - wrong extension for format ('$tmpfmt != '$fmt')\n"
 289         if $tmpfmt ne $fmt;
 290
 291     # End copy from Plugin.pm
 292
 293     my $subvol = "$imagedir/$name";
 294     # .raw is not part of the directory name
 295     $subvol =~ s/\.raw$//;
 296
 297     die "disk image '$subvol' already exists\n" if -e $subvol;
 298
 299     my $path;
 300     if ($fmt eq 'raw') {
 301         $path = "$subvol/disk.raw";
 302     }
 303
 304     if ($fmt eq 'subvol' && !!$size) {
 305         # NOTE: `btrfs send/recv` actually drops quota information so supporting subvolumes with
 306         # quotas doesn't play nice with send/recv.
 307         die "btrfs quotas are currently not supported, use an unsized subvolume or a raw file\n";
 308     }
 309
 310     $class->btrfs_cmd(['subvolume', 'create', '--', $subvol]);
 311
 312     eval {
 313         if ($fmt eq 'subvol') {
 314             # Nothing to do for now...
 315
 316             # This is how we *would* do it:
 317             # # Use the subvol's default 0/$id qgroup
 318             # eval {
 319             #     # This call should happen at storage creation instead and therefore governed by a
 320             #     # configuration option!
 321             #     # $class->btrfs_cmd(['quota', 'enable', $subvol]);
 322             #     my $id = $class->btrfs_get_subvol_id($subvol);
 323             #     $class->btrfs_cmd(['qgroup', 'limit', "${size}k", "0/$id", $subvol]);
 324             # };
 325         } elsif ($fmt eq 'raw') {
 326             sysopen my $fh, $path, O_WRONLY | O_CREAT | O_EXCL
 327                 or die "failed to create raw file '$path' - $!\n";
 328             chattr($fh, ~FS_NOCOW_FL, FS_NOCOW_FL) if $scfg->{nocow};
 329             truncate($fh, $size * 1024)
 330                 or die "failed to set file size for '$path' - $!\n";
 331             close($fh);
 332         } else {
 333             die "internal format error (format = $fmt)\n";
 334         }
 335     };
 336
 337     if (my $err = $@) {
 338         eval { $class->btrfs_cmd(['subvolume', 'delete', '--', $subvol]); };
 339         warn $@ if $@;
 340         die $err;
 341     }
 342
 343     return "$vmid/$name";
 344 }
 345
 346 # Same as btrfsprogs does:
 347 my sub path_is_subvolume : prototype($) {
 348     my ($path) = @_;
 349     my @stat = stat($path)
 350         or die "stat failed on '$path' - $!\n";
 351     my ($ino, $mode) = @stat[1, 2];
 352     return S_ISDIR($mode) && $ino == BTRFS_FIRST_FREE_OBJECTID;
 353 }
 354
 355 my $BTRFS_VOL_REGEX = qr/((?:vm|base|subvol)-\d+-disk-\d+(?:\.subvol)?)(?:\@(\S+))$/;
 356
 357 # Calls `$code->($volume, $name, $snapshot)` for each subvol in a directory matching our volume
 358 # regex.
 359 my sub foreach_subvol : prototype($$) {
 360     my ($dir, $code) = @_;
 361
 362     dir_glob_foreach($dir, $BTRFS_VOL_REGEX, sub {
 363         my ($volume, $name, $snapshot) = ($1, $2, $3);
 364         return if !path_is_subvolume("$dir/$volume");
 365         $code->($volume, $name, $snapshot);
 366     })
 367 }
 368
 369 sub free_image {
 370     my ($class, $storeid, $scfg, $volname, $isBase, $_format) = @_;
 371
 372     my (undef, undef, $vmid, undef, undef, undef, $format) =
 373         $class->parse_volname($volname);
 374
 375     if ($format ne 'subvol' && $format ne 'raw') {
 376         return PVE::Storage::DirPlugin::free_image(@_);
 377     }
 378
 379     my $path = $class->filesystem_path($scfg, $volname);
 380
 381     my $subvol = $path;
 382     if ($format eq 'raw') {
 383         $subvol = raw_file_to_subvol($path);
 384     }
 385
 386     my $dir = dirname($subvol);
 387     my @snapshot_vols;
 388     foreach_subvol($dir, sub {
 389         my ($volume, $name, $snapshot) = @_;
 390         return if !defined $snapshot;
 391         push @snapshot_vols, "$dir/$volume";
 392     });
 393
 394     $class->btrfs_cmd(['subvolume', 'delete', '--', @snapshot_vols, $subvol]);
 395     # try to cleanup directory to not clutter storage with empty $vmid dirs if
 396     # all images from a guest got deleted
 397     rmdir($dir);
 398
 399     return undef;
 400 }
 401
 402 # Currently not used because quotas clash with send/recv.
 403 # my sub btrfs_subvol_quota {
 404 #     my ($class, $path) = @_;
 405 #     my $id = '0/' . $class->btrfs_get_subvol_id($path);
 406 #     my $search = qr/^\Q$id\E\s+(\d)+\s+\d+\s+(\d+)\s*$/;
 407 #     my ($used, $size);
 408 #     $class->btrfs_cmd(['qgroup', 'show', '--raw', '-rf', '--', $path], sub {
 409 #       return if defined($size);
 410 #       if ($_[0] =~ $search) {
 411 #           ($used, $size) = ($1, $2);
 412 #       }
 413 #     });
 414 #     if (!defined($size)) {
 415 #       # syslog should include more information:
 416 #       syslog('err', "failed to get subvolume size for: $path (id $id)");
 417 #       # UI should only see the last path component:
 418 #       $path =~ s|^.*/||;
 419 #       die "failed to get subvolume size for $path\n";
 420 #     }
 421 #     return wantarray ? ($used, $size) : $size;
 422 # }
 423
 424 sub volume_size_info {
 425     my ($class, $scfg, $storeid, $volname, $timeout) = @_;
 426
 427     my $path = $class->filesystem_path($scfg, $volname);
 428
 429     my $format = ($class->parse_volname($volname))[6];
 430
 431     if ($format eq 'subvol') {
 432         my $ctime = (stat($path))[10];
 433         my ($used, $size) = (0, 0);
 434         #my ($used, $size) = btrfs_subvol_quota($class, $path); # uses wantarray
 435         return wantarray ? ($size, 'subvol', $used, undef, $ctime) : 1;
 436     }
 437
 438     return PVE::Storage::Plugin::file_size_info($path, $timeout);
 439 }
 440
 441 sub volume_resize {
 442     my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
 443
 444     my $format = ($class->parse_volname($volname))[6];
 445     if ($format eq 'subvol') {
 446         my $path = $class->filesystem_path($scfg, $volname);
 447         my $id = '0/' . $class->btrfs_get_subvol_id($path);
 448         $class->btrfs_cmd(['qgroup', 'limit', '--', "${size}k", "0/$id", $path]);
 449         return undef;
 450     }
 451
 452     return PVE::Storage::Plugin::volume_resize(@_);
 453 }
 454
 455 sub volume_snapshot {
 456     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 457
 458     my ($name, $vmid, $format) = ($class->parse_volname($volname))[1,2,6];
 459     if ($format ne 'subvol' && $format ne 'raw') {
 460         return PVE::Storage::Plugin::volume_snapshot(@_);
 461     }
 462
 463     my $path = $class->filesystem_path($scfg, $volname);
 464     my $snap_path = $class->filesystem_path($scfg, $volname, $snap);
 465
 466     if ($format eq 'raw') {
 467         $path = raw_file_to_subvol($path);
 468         $snap_path = raw_file_to_subvol($snap_path);
 469     }
 470
 471     my $snapshot_dir = $class->get_subdir($scfg, 'images') . "/$vmid";
 472     mkpath $snapshot_dir;
 473
 474     $class->btrfs_cmd(['subvolume', 'snapshot', '-r', '--', $path, $snap_path]);
 475     return undef;
 476 }
 477
 478 sub volume_rollback_is_possible {
 479     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 480
 481     return 1;
 482 }
 483
 484 sub volume_snapshot_rollback {
 485     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 486
 487     my ($name, $format) = ($class->parse_volname($volname))[1,6];
 488
 489     if ($format ne 'subvol' && $format ne 'raw') {
 490         return PVE::Storage::Plugin::volume_snapshot_rollback(@_);
 491     }
 492
 493     my $path = $class->filesystem_path($scfg, $volname);
 494     my $snap_path = $class->filesystem_path($scfg, $volname, $snap);
 495
 496     if ($format eq 'raw') {
 497         $path = raw_file_to_subvol($path);
 498         $snap_path = raw_file_to_subvol($snap_path);
 499     }
 500
 501     # Simple version would be:
 502     #   rename old to temp
 503     #   create new
 504     #   on error rename temp back
 505     # But for atomicity in case the rename after create-failure *also* fails, we create the new
 506     # subvol first, then use RENAME_EXCHANGE,
 507     my $tmp_path = "$path.tmp.$$";
 508     $class->btrfs_cmd(['subvolume', 'snapshot', '--', $snap_path, $tmp_path]);
 509     # The paths are absolute, so pass -1 as file descriptors.
 510     my $ok = PVE::Tools::renameat2(-1, $tmp_path, -1, $path, &PVE::Tools::RENAME_EXCHANGE);
 511
 512     eval { $class->btrfs_cmd(['subvolume', 'delete', '--', $tmp_path]) };
 513     warn "failed to remove '$tmp_path' subvolume: $@" if $@;
 514
 515     if (!$ok) {
 516         die "failed to rotate '$tmp_path' into place at '$path' - $!\n";
 517     }
 518
 519     return undef;
 520 }
 521
 522 sub volume_snapshot_delete {
 523     my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 524
 525     my ($name, $vmid, $format) = ($class->parse_volname($volname))[1,2,6];
 526
 527     if ($format ne 'subvol' && $format ne 'raw') {
 528         return PVE::Storage::Plugin::volume_snapshot_delete(@_);
 529     }
 530
 531     my $path = $class->filesystem_path($scfg, $volname, $snap);
 532
 533     if ($format eq 'raw') {
 534         $path = raw_file_to_subvol($path);
 535     }
 536
 537     $class->btrfs_cmd(['subvolume', 'delete', '--', $path]);
 538
 539     return undef;
 540 }
 541
 542 sub volume_has_feature {
 543     my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
 544
 545     my $features = {
 546         snapshot => {
 547             current => { qcow2 => 1, raw => 1, subvol => 1 },
 548             snap => { qcow2 => 1, raw => 1, subvol => 1 }
 549         },
 550         clone => {
 551             base => { qcow2 => 1, raw => 1, subvol => 1, vmdk => 1 },
 552             current => { raw => 1 },
 553             snap => { raw => 1 },
 554         },
 555         template => { current => { qcow2 => 1, raw => 1, vmdk => 1, subvol => 1 } },
 556         copy => {
 557             base => { qcow2 => 1, raw => 1, subvol => 1, vmdk => 1 },
 558             current => { qcow2 => 1, raw => 1, subvol => 1, vmdk => 1 },
 559             snap => { qcow2 => 1, raw => 1, subvol => 1 },
 560         },
 561         sparseinit => { base => {qcow2 => 1, raw => 1, vmdk => 1 },
 562                         current => {qcow2 => 1, raw => 1, vmdk => 1 } },
 563     };
 564
 565     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
 566         $class->parse_volname($volname);
 567
 568     my $key = undef;
 569     if ($snapname) {
 570         $key = 'snap';
 571     } else {
 572         $key =  $isBase ? 'base' : 'current';
 573     }
 574
 575     return 1 if defined($features->{$feature}->{$key}->{$format});
 576
 577     return undef;
 578 }
 579
 580 sub list_images {
 581     my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
 582     my $imagedir = $class->get_subdir($scfg, 'images');
 583
 584     my $res = [];
 585
 586     # Copied from Plugin.pm, with file_size_info calls adapted:
 587     foreach my $fn (<$imagedir/[0-9][0-9]*/*>) {
 588         # different to in Plugin.pm the regex below also excludes '@' as valid file name
 589         next if $fn !~ m@^(/.+/(\d+)/([^/\@.]+(?:\.(qcow2|vmdk|subvol))?))$@;
 590         $fn = $1; # untaint
 591
 592         my $owner = $2;
 593         my $name = $3;
 594         my $ext = $4;
 595
 596         next if !$vollist && defined($vmid) && ($owner ne $vmid);
 597
 598         my $volid = "$storeid:$owner/$name";
 599         my ($size, $format, $used, $parent, $ctime);
 600
 601         if (!$ext) { # raw
 602             $volid .= '.raw';
 603             ($size, $format, $used, $parent, $ctime) = PVE::Storage::Plugin::file_size_info("$fn/disk.raw");
 604         } elsif ($ext eq 'subvol') {
 605             ($used, $size) = (0, 0);
 606             #($used, $size) = btrfs_subvol_quota($class, $fn);
 607             $format = 'subvol';
 608         } else {
 609             ($size, $format, $used, $parent, $ctime) = PVE::Storage::Plugin::file_size_info($fn);
 610         }
 611         next if !($format && defined($size));
 612
 613         if ($vollist) {
 614             next if ! grep { $_ eq $volid } @$vollist;
 615         }
 616
 617         my $info = {
 618             volid => $volid, format => $format,
 619             size => $size, vmid => $owner, used => $used, parent => $parent,
 620         };
 621
 622         $info->{ctime} = $ctime if $ctime;
 623
 624         push @$res, $info;
 625     }
 626
 627     return $res;
 628 }
 629
 630 sub volume_export_formats {
 631     my ($class, $scfg, $storeid, $volname, $snapshot, $base_snapshot, $with_snapshots) = @_;
 632
 633     # We can do whatever `DirPlugin` can do.
 634     my @result = PVE::Storage::Plugin::volume_export_formats(@_);
 635
 636     # `btrfs send` only works on snapshots:
 637     return @result if !defined $snapshot;
 638
 639     # Incremental stream with snapshots is only supported if the snapshots are listed (new api):
 640     return @result if defined($base_snapshot) && $with_snapshots && ref($with_snapshots) ne 'ARRAY';
 641
 642     # Otherwise we do also support `with_snapshots`.
 643
 644     # Finally, `btrfs send` only works on formats where we actually use btrfs subvolumes:
 645     my $format = ($class->parse_volname($volname))[6];
 646     return @result if $format ne 'raw' && $format ne 'subvol';
 647
 648     return ('btrfs', @result);
 649 }
 650
 651 sub volume_import_formats {
 652     my ($class, $scfg, $storeid, $volname, $snapshot, $base_snapshot, $with_snapshots) = @_;
 653
 654     # Same as export-formats, beware the parameter order:
 655     return volume_export_formats(
 656         $class,
 657         $scfg,
 658         $storeid,
 659         $volname,
 660         $snapshot,
 661         $base_snapshot,
 662         $with_snapshots,
 663     );
 664 }
 665
 666 sub volume_export {
 667     my (
 668         $class,
 669         $scfg,
 670         $storeid,
 671         $fh,
 672         $volname,
 673         $format,
 674         $snapshot,
 675         $base_snapshot,
 676         $with_snapshots,
 677     ) = @_;
 678
 679     if ($format ne 'btrfs') {
 680         return PVE::Storage::Plugin::volume_export(@_);
 681     }
 682
 683     die "format 'btrfs' only works on snapshots\n"
 684         if !defined $snapshot;
 685
 686     die "'btrfs' format in incremental mode requires snapshots to be listed explicitly\n"
 687         if defined($base_snapshot) && $with_snapshots && ref($with_snapshots) ne 'ARRAY';
 688
 689     my $volume_format = ($class->parse_volname($volname))[6];
 690
 691     die "btrfs-sending volumes of type $volume_format ('$volname') is not supported\n"
 692         if $volume_format ne 'raw' && $volume_format ne 'subvol';
 693
 694     my $path = $class->path($scfg, $volname, $storeid);
 695
 696     if ($volume_format eq 'raw') {
 697         $path = raw_file_to_subvol($path);
 698     }
 699
 700     my $cmd = ['btrfs', '-q', 'send', '-e'];
 701     if ($base_snapshot) {
 702         my $base = $class->path($scfg, $volname, $storeid, $base_snapshot);
 703         if ($volume_format eq 'raw') {
 704             $base = raw_file_to_subvol($base);
 705         }
 706         push @$cmd, '-p', $base;
 707     }
 708     push @$cmd, '--';
 709     if (ref($with_snapshots) eq 'ARRAY') {
 710         push @$cmd, (map { "$path\@$_" } ($with_snapshots // [])->@*), $path;
 711     } else {
 712         dir_glob_foreach(dirname($path), $BTRFS_VOL_REGEX, sub {
 713             push @$cmd, "$path\@$_[2]" if !(defined($snapshot) && $_[2] eq $snapshot);
 714         });
 715     }
 716     $path .= "\@$snapshot" if defined($snapshot);
 717     push @$cmd, $path;
 718
 719     run_command($cmd, output => '>&'.fileno($fh));
 720     return;
 721 }
 722
 723 sub volume_import {
 724     my (
 725         $class,
 726         $scfg,
 727         $storeid,
 728         $fh,
 729         $volname,
 730         $format,
 731         $snapshot,
 732         $base_snapshot,
 733         $with_snapshots,
 734         $allow_rename,
 735     ) = @_;
 736
 737     if ($format ne 'btrfs') {
 738         return PVE::Storage::Plugin::volume_import(@_);
 739     }
 740
 741     die "format 'btrfs' only works on snapshots\n"
 742         if !defined $snapshot;
 743
 744     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $volume_format) =
 745         $class->parse_volname($volname);
 746
 747     die "btrfs-receiving volumes of type $volume_format ('$volname') is not supported\n"
 748         if $volume_format ne 'raw' && $volume_format ne 'subvol';
 749
 750     if (defined($base_snapshot)) {
 751         my $path = $class->path($scfg, $volname, $storeid, $base_snapshot);
 752         die "base snapshot '$base_snapshot' not found - no such directory '$path'\n"
 753             if !path_is_subvolume($path);
 754     }
 755
 756     my $destination = $class->filesystem_path($scfg, $volname);
 757     if ($volume_format eq 'raw') {
 758         $destination = raw_file_to_subvol($destination);
 759     }
 760
 761     if (!defined($base_snapshot) && -e $destination) {
 762         die "volume $volname already exists\n" if !$allow_rename;
 763         $volname = $class->find_free_diskname($storeid, $scfg, $vmid, $volume_format, 1);
 764     }
 765
 766     my $imagedir = $class->get_subdir($scfg, $vtype);
 767     $imagedir .= "/$vmid" if $vtype eq 'images';
 768
 769     my $tmppath = "$imagedir/recv.$vmid.tmp";
 770     mkdir($imagedir); # FIXME: if $scfg->{mkdir};
 771     if (!mkdir($tmppath)) {
 772         die "temp receive directory already exists at '$tmppath', incomplete concurrent import?\n"
 773             if $! == EEXIST;
 774         die "failed to create temporary receive directory at '$tmppath' - $!\n";
 775     }
 776
 777     my $dh = IO::Dir->new($tmppath)
 778         or die "failed to open temporary receive directory '$tmppath' - $!\n";
 779     eval {
 780         run_command(['btrfs', '-q', 'receive', '-e', '--', $tmppath], input => '<&'.fileno($fh));
 781
 782         # Analyze the received subvolumes;
 783         my ($diskname, $found_snapshot, @snapshots);
 784         $dh->rewind;
 785         while (defined(my $entry = $dh->read)) {
 786             next if $entry eq '.' || $entry eq '..';
 787             next if $entry !~ /^$BTRFS_VOL_REGEX$/;
 788             my ($cur_diskname, $cur_snapshot) = ($1, $2);
 789
 790             die "send stream included a non-snapshot subvolume\n"
 791                 if !defined($cur_snapshot);
 792
 793             if (!defined($diskname)) {
 794                 $diskname = $cur_diskname;
 795             } else {
 796                 die "multiple disks contained in stream ('$diskname' vs '$cur_diskname')\n"
 797                     if $diskname ne $cur_diskname;
 798             }
 799
 800             if ($cur_snapshot eq $snapshot) {
 801                 $found_snapshot = 1;
 802             } else {
 803                 push @snapshots, $cur_snapshot;
 804             }
 805         }
 806
 807         die "send stream did not contain the expected current snapshot '$snapshot'\n"
 808             if !$found_snapshot;
 809
 810         # Rotate the disk into place, first the current state:
 811         # Note that read-only subvolumes cannot be moved into different directories, but for the
 812         # "current" state we also want a writable copy, so start with that:
 813         $class->btrfs_cmd(['property', 'set', "$tmppath/$diskname\@$snapshot", 'ro', 'false']);
 814         PVE::Tools::renameat2(
 815             -1,
 816             "$tmppath/$diskname\@$snapshot",
 817             -1,
 818             $destination,
 819             &PVE::Tools::RENAME_NOREPLACE,
 820         ) or die "failed to move received snapshot '$tmppath/$diskname\@$snapshot'"
 821             . " into place at '$destination' - $!\n";
 822
 823         # Now recreate the actual snapshot:
 824         $class->btrfs_cmd([
 825             'subvolume',
 826             'snapshot',
 827             '-r',
 828             '--',
 829             $destination,
 830             "$destination\@$snapshot",
 831         ]);
 832
 833         # Now go through the remaining snapshots (if any)
 834         foreach my $snap (@snapshots) {
 835             $class->btrfs_cmd(['property', 'set', "$tmppath/$diskname\@$snap", 'ro', 'false']);
 836             PVE::Tools::renameat2(
 837                 -1,
 838                 "$tmppath/$diskname\@$snap",
 839                 -1,
 840                 "$destination\@$snap",
 841                 &PVE::Tools::RENAME_NOREPLACE,
 842             ) or die "failed to move received snapshot '$tmppath/$diskname\@$snap'"
 843                 . " into place at '$destination\@$snap' - $!\n";
 844             eval { $class->btrfs_cmd(['property', 'set', "$destination\@$snap", 'ro', 'true']) };
 845             warn "failed to make $destination\@$snap read-only - $!\n" if $@;
 846         }
 847     };
 848     my $err = $@;
 849
 850     eval {
 851         # Cleanup all the received snapshots we did not move into place, so we can remove the temp
 852         # directory.
 853         if ($dh) {
 854             $dh->rewind;
 855             while (defined(my $entry = $dh->read)) {
 856                 next if $entry eq '.' || $entry eq '..';
 857                 eval { $class->btrfs_cmd(['subvolume', 'delete', '--', "$tmppath/$entry"]) };
 858                 warn $@ if $@;
 859             }
 860             $dh->close; undef $dh;
 861         }
 862         if (!rmdir($tmppath)) {
 863             warn "failed to remove temporary directory '$tmppath' - $!\n"
 864         }
 865     };
 866     warn $@ if $@;
 867     if ($err) {
 868         # clean up if the directory ended up being empty after an error
 869         rmdir($tmppath);
 870         die $err;
 871     }
 872
 873     return "$storeid:$volname";
 874 }
 875
 876 1