PVE/Storage/BTRFSPlugin.pm

   1 package PVE::Storage::BTRFSPlugin;
   2
   3 use strict;
   4 use warnings;
   5
   6 use base qw(PVE::Storage::Plugin);
   7
   8 use Fcntl qw(S_ISDIR O_WRONLY O_CREAT O_EXCL);
   9 use File::Basename qw(dirname);
  10 use File::Path qw(mkpath);
  11 use IO::Dir;
  12 use POSIX qw(EEXIST);
  13
  14 use PVE::Tools qw(run_command dir_glob_foreach);
  15
  16 use PVE::Storage::DirPlugin;
  17
  18 use constant {
  19     BTRFS_FIRST_FREE_OBJECTID => 256,
  20     FS_NOCOW_FL => 0x00800000,
  21     FS_IOC_GETFLAGS => 0x40086602,
  22     FS_IOC_SETFLAGS => 0x80086601,
  23     BTRFS_MAGIC => 0x9123683e,
  24 };
  25
  26 # Configuration (similar to DirPlugin)
  27
  28 sub type {
  29     return 'btrfs';
  30 }
  31
  32 sub plugindata {
  33     return {
  34         content => [
  35             {
  36                 images => 1,
  37                 rootdir => 1,
  38                 vztmpl => 1,
  39                 iso => 1,
  40                 backup => 1,
  41                 snippets => 1,
  42                 none => 1,
  43             },
  44             { images => 1, rootdir => 1 },
  45         ],
  46         format => [ { raw => 1, subvol => 1 }, 'raw', ],
  47     };
  48 }
  49
  50 sub properties {
  51     return {
  52         nocow => {
  53             description => "Set the NOCOW flag on files."
  54                 . " Disables data checksumming and causes data errors to be unrecoverable from"
  55                 . " while allowing direct I/O. Only use this if data does not need to be any more"
  56                 . " safe than on a single ext4 formatted disk with no underlying raid system.",
  57             type => 'boolean',
  58             default => 0,
  59         },
  60     };
  61 }
  62
  63 sub options {
  64     return {
  65         path => { fixed => 1 },
  66         nodes => { optional => 1 },
  67         shared => { optional => 1 },
  68         disable => { optional => 1 },
  69         maxfiles => { optional => 1 },
  70         content => { optional => 1 },
  71         format => { optional => 1 },
  72         is_mountpoint => { optional => 1 },
  73         nocow => { optional => 1 },
  74         # TODO: The new variant of mkdir with  `populate` vs `create`...
  75     };
  76 }
  77
  78 # Storage implementation
  79 #
  80 # We use the same volume names are directory plugins, but map *raw* disk image file names into a
  81 # subdirectory.
  82 #
  83 # `vm-VMID-disk-ID.raw`
  84 #   -> `images/VMID/vm-VMID-disk-ID/disk.raw`
  85 #   where the `vm-VMID-disk-ID/` subdirectory is a btrfs subvolume
  86
  87 # Reuse `DirPlugin`'s `check_config`. This simply checks for invalid paths.
  88 sub check_config {
  89     my ($self, $sectionId, $config, $create, $skipSchemaCheck) = @_;
  90     return PVE::Storage::DirPlugin::check_config($self, $sectionId, $config, $create, $skipSchemaCheck);
  91 }
  92
  93 my sub getfsmagic($) {
  94     my ($path) = @_;
  95     # The field type sizes in `struct statfs` are defined in a rather annoying way, and we only
  96     # need the first field, which is a `long` for our supported platforms.
  97     # Should be moved to pve-rs, so this can be the problem of the `libc` crate ;-)
  98     # Just round up and extract what we need:
  99     my $buf = pack('x160');
 100     if (0 != syscall(&PVE::Syscall::SYS_statfs, $path, $buf)) {
 101         die "statfs on '$path' failed - $!\n";
 102     }
 103
 104     return unpack('L!', $buf);
 105 }
 106
 107 my sub assert_btrfs($) {
 108     my ($path) = @_;
 109     die "'$path' is not a btrfs file system\n"
 110         if getfsmagic($path) != BTRFS_MAGIC;
 111 }
 112
 113 sub activate_storage {
 114     my ($class, $storeid, $scfg, $cache) = @_;
 115
 116     my $path = $scfg->{path};
 117     if (!defined($scfg->{mkdir}) || $scfg->{mkdir}) {
 118         mkpath $path;
 119     }
 120
 121     my $mp = PVE::Storage::DirPlugin::parse_is_mountpoint($scfg);
 122     if (defined($mp) && !path_is_mounted($mp, $cache->{mountdata})) {
 123         die "unable to activate storage '$storeid' - directory is expected to be a mount point but"
 124         ." is not mounted: '$mp'\n";
 125     }
 126
 127     assert_btrfs($path); # only assert this stuff now, ensures $path is there and better UX
 128
 129     $class->SUPER::activate_storage($storeid, $scfg, $cache);
 130 }
 131
 132 sub status {
 133     my ($class, $storeid, $scfg, $cache) = @_;
 134     return PVE::Storage::DirPlugin::status($class, $storeid, $scfg, $cache);
 135 }
 136
 137 # TODO: sub get_volume_notes {}
 138
 139 # TODO: sub update_volume_notes {}
 140
 141 # croak would not include the caller from within this module
 142 sub __error {
 143     my ($msg) = @_;
 144     my (undef, $f, $n) = caller(1);
 145     die "$msg at $f: $n\n";
 146 }
 147
 148 # Given a name (eg. `vm-VMID-disk-ID.raw`), take the part up to the format suffix as the name of
 149 # the subdirectory (subvolume).
 150 sub raw_name_to_dir($) {
 151     my ($raw) = @_;
 152
 153     # For the subvolume directory Strip the `.<format>` suffix:
 154     if ($raw =~ /^(.*)\.raw$/) {
 155         return $1;
 156     }
 157
 158     __error "internal error: bad disk name: $raw";
 159 }
 160
 161 sub raw_file_to_subvol($) {
 162     my ($file) = @_;
 163
 164     if ($file =~ m|^(.*)/disk\.raw$|) {
 165         return "$1";
 166     }
 167
 168     __error "internal error: bad raw path: $file";
 169 }
 170
 171 sub filesystem_path {
 172     my ($class, $scfg, $volname, $snapname) = @_;
 173
 174     my ($vtype, $name, $vmid, undef, undef, $isBase, $format) =
 175         $class->parse_volname($volname);
 176
 177     my $path = $class->get_subdir($scfg, $vtype);
 178
 179     $path .= "/$vmid" if $vtype eq 'images';
 180
 181     if ($format eq 'raw') {
 182         my $dir = raw_name_to_dir($name);
 183         if ($snapname) {
 184             $dir .= "\@$snapname";
 185         }
 186         $path .= "/$dir/disk.raw";
 187     } elsif ($format eq 'subvol') {
 188         $path .= "/$name";
 189         if ($snapname) {
 190             $path .= "\@$snapname";
 191         }
 192     } else {
 193         $path .= "/$name";
 194     }
 195
 196     return wantarray ? ($path, $vmid, $vtype) : $path;
 197 }
 198
 199 sub btrfs_cmd {
 200     my ($class, $cmd, $outfunc) = @_;
 201
 202     my $msg = '';
 203     my $func;
 204     if (defined($outfunc)) {
 205         $func = sub {
 206             my $part = &$outfunc(@_);
 207             $msg .= $part if defined($part);
 208         };
 209     } else {
 210         $func = sub { $msg .= "$_[0]\n" };
 211     }
 212     run_command(['btrfs', '-q', @$cmd], errmsg => 'btrfs error', outfunc => $func);
 213
 214     return $msg;
 215 }
 216
 217 sub btrfs_get_subvol_id {
 218     my ($class, $path) = @_;
 219     my $info = $class->btrfs_cmd(['subvolume', 'show', '--', $path]);
 220     if ($info !~ /^\s*(?:Object|Subvolume) ID:\s*(\d+)$/m) {
 221         die "failed to get btrfs subvolume ID from: $info\n";
 222     }
 223     return $1;
 224 }
 225
 226 my sub chattr : prototype($$$) {
 227     my ($fh, $mask, $xor) = @_;
 228
 229     my $flags = pack('L!', 0);
 230     ioctl($fh, FS_IOC_GETFLAGS, $flags) or die "FS_IOC_GETFLAGS failed - $!\n";
 231     $flags = pack('L!', (unpack('L!', $flags) & $mask) ^ $xor);
 232     ioctl($fh, FS_IOC_SETFLAGS, $flags) or die "FS_IOC_SETFLAGS failed - $!\n";
 233     return 1;
 234 }
 235
 236 sub create_base {
 237     my ($class, $storeid, $scfg, $volname) = @_;
 238
 239     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
 240         $class->parse_volname($volname);
 241
 242     my $newname = $name;
 243     $newname =~ s/^vm-/base-/;
 244
 245     # If we're not working with a 'raw' file, which is the only thing that's "different" for btrfs,
 246     # or a subvolume, we forward to the DirPlugin
 247     if ($format ne 'raw' && $format ne 'subvol') {
 248         return PVE::Storage::DirPlugin::create_base(@_);
 249     }
 250
 251     my $path = $class->filesystem_path($scfg, $volname);
 252     my $newvolname = $basename ? "$basevmid/$basename/$vmid/$newname" : "$vmid/$newname";
 253     my $newpath = $class->filesystem_path($scfg, $newvolname);
 254
 255     my $subvol = $path;
 256     my $newsubvol = $newpath;
 257     if ($format eq 'raw') {
 258         $subvol = raw_file_to_subvol($subvol);
 259         $newsubvol = raw_file_to_subvol($newsubvol);
 260     }
 261
 262     rename($subvol, $newsubvol)
 263         || die "rename '$subvol' to '$newsubvol' failed - $!\n";
 264     eval { $class->btrfs_cmd(['property', 'set', $newsubvol, 'ro', 'true']) };
 265     warn $@ if $@;
 266
 267     return $newvolname;
 268 }
 269
 270 sub clone_image {
 271     my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
 272
 273     my ($vtype, $basename, $basevmid, undef, undef, $isBase, $format) =
 274         $class->parse_volname($volname);
 275
 276     # If we're not working with a 'raw' file, which is the only thing that's "different" for btrfs,
 277     # or a subvolume, we forward to the DirPlugin
 278     if ($format ne 'raw' && $format ne 'subvol') {
 279         return PVE::Storage::DirPlugin::clone_image(@_);
 280     }
 281
 282     my $imagedir = $class->get_subdir($scfg, 'images');
 283     $imagedir .= "/$vmid";
 284     mkpath $imagedir;
 285
 286     my $path = $class->filesystem_path($scfg, $volname);
 287     my $newname = $class->find_free_diskname($storeid, $scfg, $vmid, $format, 1);
 288
 289     # For btrfs subvolumes we don't actually need the "link":
 290     #my $newvolname = "$basevmid/$basename/$vmid/$newname";
 291     my $newvolname = "$vmid/$newname";
 292     my $newpath = $class->filesystem_path($scfg, $newvolname);
 293
 294     my $subvol = $path;
 295     my $newsubvol = $newpath;
 296     if ($format eq 'raw') {
 297         $subvol = raw_file_to_subvol($subvol);
 298         $newsubvol = raw_file_to_subvol($newsubvol);
 299     }
 300
 301     $class->btrfs_cmd(['subvolume', 'snapshot', '--', $subvol, $newsubvol]);
 302
 303     return $newvolname;
 304 }
 305
 306 sub alloc_image {
 307     my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
 308
 309     if ($fmt ne 'raw' && $fmt ne 'subvol') {
 310         return PVE::Storage::DirPlugin::alloc_image(@_);
 311     }
 312
 313     # From Plugin.pm:
 314
 315     my $imagedir = $class->get_subdir($scfg, 'images') . "/$vmid";
 316
 317     mkpath $imagedir;
 318
 319     $name = $class->find_free_diskname($storeid, $scfg, $vmid, $fmt, 1) if !$name;
 320
 321     my (undef, $tmpfmt) = PVE::Storage::Plugin::parse_name_dir($name);
 322
 323     die "illegal name '$name' - wrong extension for format ('$tmpfmt != '$fmt')\n"
 324         if $tmpfmt ne $fmt;
 325
 326     # End copy from Plugin.pm
 327
 328     my $subvol = "$imagedir/$name";
 329     # .raw is not part of the directory name
 330     $subvol =~ s/\.raw$//;
 331
 332     die "disk image '$subvol' already exists\n" if -e $subvol;
 333
 334     my $path;
 335     if ($fmt eq 'raw') {
 336         $path = "$subvol/disk.raw";
 337     }
 338
 339     if ($fmt eq 'subvol' && !!$size) {
 340         # NOTE: `btrfs send/recv` actually drops quota information so supporting subvolumes with
 341         # quotas doesn't play nice with send/recv.
 342         die "btrfs quotas are currently not supported, use an unsized subvolume or a raw file\n";
 343     }
 344
 345     $class->btrfs_cmd(['subvolume', 'create', '--', $subvol]);
 346
 347     eval {
 348         if ($fmt eq 'subvol') {
 349             # Nothing to do for now...
 350
 351             # This is how we *would* do it:
 352             # # Use the subvol's default 0/$id qgroup
 353             # eval {
 354             #     # This call should happen at storage creation instead and therefore governed by a
 355             #     # configuration option!
 356             #     # $class->btrfs_cmd(['quota', 'enable', $subvol]);
 357             #     my $id = $class->btrfs_get_subvol_id($subvol);
 358             #     $class->btrfs_cmd(['qgroup', 'limit', "${size}k", "0/$id", $subvol]);
 359             # };
 360         } elsif ($fmt eq 'raw') {
 361             sysopen my $fh, $path, O_WRONLY | O_CREAT | O_EXCL
 362                 or die "failed to create raw file '$path' - $!\n";
 363             chattr($fh, ~FS_NOCOW_FL, FS_NOCOW_FL) if $scfg->{nocow};
 364             truncate($fh, $size * 1024)
 365                 or die "failed to set file size for '$path' - $!\n";
 366             close($fh);
 367         } else {
 368             die "internal format error (format = $fmt)\n";
 369         }
 370     };
 371
 372     if (my $err = $@) {
 373         eval { $class->btrfs_cmd(['subvolume', 'delete', '--', $subvol]); };
 374         warn $@ if $@;
 375         die $err;
 376     }
 377
 378     return "$vmid/$name";
 379 }
 380
 381 # Same as btrfsprogs does:
 382 my sub path_is_subvolume : prototype($) {
 383     my ($path) = @_;
 384     my @stat = stat($path)
 385         or die "stat failed on '$path' - $!\n";
 386     my ($ino, $mode) = @stat[1, 2];
 387     return S_ISDIR($mode) && $ino == BTRFS_FIRST_FREE_OBJECTID;
 388 }
 389
 390 my $BTRFS_VOL_REGEX = qr/((?:vm|base|subvol)-\d+-disk-\d+(?:\.subvol)?)(?:\@(\S+))$/;
 391
 392 # Calls `$code->($volume, $name, $snapshot)` for each subvol in a directory matching our volume
 393 # regex.
 394 my sub foreach_subvol : prototype($$) {
 395     my ($dir, $code) = @_;
 396
 397     dir_glob_foreach($dir, $BTRFS_VOL_REGEX, sub {
 398         my ($volume, $name, $snapshot) = ($1, $2, $3);
 399         return if !path_is_subvolume("$dir/$volume");
 400         $code->($volume, $name, $snapshot);
 401     })
 402 }
 403
 404 sub free_image {
 405     my ($class, $storeid, $scfg, $volname, $isBase, $_format) = @_;
 406
 407     my (undef, undef, $vmid, undef, undef, undef, $format) =
 408         $class->parse_volname($volname);
 409
 410     if ($format ne 'subvol' && $format ne 'raw') {
 411         return PVE::Storage::DirPlugin::free_image(@_);
 412     }
 413
 414     my $path = $class->filesystem_path($scfg, $volname);
 415
 416     my $subvol = $path;
 417     if ($format eq 'raw') {
 418         $subvol = raw_file_to_subvol($path);
 419     }
 420
 421     my $dir = dirname($subvol);
 422     my @snapshot_vols;
 423     foreach_subvol($dir, sub {
 424         my ($volume, $name, $snapshot) = @_;
 425         return if !defined $snapshot;
 426         push @snapshot_vols, "$dir/$volume";
 427     });
 428
 429     $class->btrfs_cmd(['subvolume', 'delete', '--', @snapshot_vols, $subvol]);
 430     # try to cleanup directory to not clutter storage with empty $vmid dirs if
 431     # all images from a guest got deleted
 432     rmdir($dir);
 433
 434     return undef;
 435 }
 436
 437 # Currently not used because quotas clash with send/recv.
 438 # my sub btrfs_subvol_quota {
 439 #     my ($class, $path) = @_;
 440 #     my $id = '0/' . $class->btrfs_get_subvol_id($path);
 441 #     my $search = qr/^\Q$id\E\s+(\d)+\s+\d+\s+(\d+)\s*$/;
 442 #     my ($used, $size);
 443 #     $class->btrfs_cmd(['qgroup', 'show', '--raw', '-rf', '--', $path], sub {
 444 #       return if defined($size);
 445 #       if ($_[0] =~ $search) {
 446 #           ($used, $size) = ($1, $2);
 447 #       }
 448 #     });
 449 #     if (!defined($size)) {
 450 #       # syslog should include more information:
 451 #       syslog('err', "failed to get subvolume size for: $path (id $id)");
 452 #       # UI should only see the last path component:
 453 #       $path =~ s|^.*/||;
 454 #       die "failed to get subvolume size for $path\n";
 455 #     }
 456 #     return wantarray ? ($used, $size) : $size;
 457 # }
 458
 459 sub volume_size_info {
 460     my ($class, $scfg, $storeid, $volname, $timeout) = @_;
 461
 462     my $path = $class->filesystem_path($scfg, $volname);
 463
 464     my $format = ($class->parse_volname($volname))[6];
 465
 466     if ($format eq 'subvol') {
 467         my $ctime = (stat($path))[10];
 468         my ($used, $size) = (0, 0);
 469         #my ($used, $size) = btrfs_subvol_quota($class, $path); # uses wantarray
 470         return wantarray ? ($size, 'subvol', $used, undef, $ctime) : 1;
 471     }
 472
 473     return PVE::Storage::Plugin::file_size_info($path, $timeout);
 474 }
 475
 476 sub volume_resize {
 477     my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
 478
 479     my $format = ($class->parse_volname($volname))[6];
 480     if ($format eq 'subvol') {
 481         my $path = $class->filesystem_path($scfg, $volname);
 482         my $id = '0/' . $class->btrfs_get_subvol_id($path);
 483         $class->btrfs_cmd(['qgroup', 'limit', '--', "${size}k", "0/$id", $path]);
 484         return undef;
 485     }
 486
 487     return PVE::Storage::Plugin::volume_resize(@_);
 488 }
 489
 490 sub volume_snapshot {
 491     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 492
 493     my ($name, $vmid, $format) = ($class->parse_volname($volname))[1,2,6];
 494     if ($format ne 'subvol' && $format ne 'raw') {
 495         return PVE::Storage::Plugin::volume_snapshot(@_);
 496     }
 497
 498     my $path = $class->filesystem_path($scfg, $volname);
 499     my $snap_path = $class->filesystem_path($scfg, $volname, $snap);
 500
 501     if ($format eq 'raw') {
 502         $path = raw_file_to_subvol($path);
 503         $snap_path = raw_file_to_subvol($snap_path);
 504     }
 505
 506     my $snapshot_dir = $class->get_subdir($scfg, 'images') . "/$vmid";
 507     mkpath $snapshot_dir;
 508
 509     $class->btrfs_cmd(['subvolume', 'snapshot', '-r', '--', $path, $snap_path]);
 510     return undef;
 511 }
 512
 513 sub volume_rollback_is_possible {
 514     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 515
 516     return 1;
 517 }
 518
 519 sub volume_snapshot_rollback {
 520     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 521
 522     my ($name, $format) = ($class->parse_volname($volname))[1,6];
 523
 524     if ($format ne 'subvol' && $format ne 'raw') {
 525         return PVE::Storage::Plugin::volume_snapshot_rollback(@_);
 526     }
 527
 528     my $path = $class->filesystem_path($scfg, $volname);
 529     my $snap_path = $class->filesystem_path($scfg, $volname, $snap);
 530
 531     if ($format eq 'raw') {
 532         $path = raw_file_to_subvol($path);
 533         $snap_path = raw_file_to_subvol($snap_path);
 534     }
 535
 536     # Simple version would be:
 537     #   rename old to temp
 538     #   create new
 539     #   on error rename temp back
 540     # But for atomicity in case the rename after create-failure *also* fails, we create the new
 541     # subvol first, then use RENAME_EXCHANGE,
 542     my $tmp_path = "$path.tmp.$$";
 543     $class->btrfs_cmd(['subvolume', 'snapshot', '--', $snap_path, $tmp_path]);
 544     # The paths are absolute, so pass -1 as file descriptors.
 545     my $ok = PVE::Tools::renameat2(-1, $tmp_path, -1, $path, &PVE::Tools::RENAME_EXCHANGE);
 546
 547     eval { $class->btrfs_cmd(['subvolume', 'delete', '--', $tmp_path]) };
 548     warn "failed to remove '$tmp_path' subvolume: $@" if $@;
 549
 550     if (!$ok) {
 551         die "failed to rotate '$tmp_path' into place at '$path' - $!\n";
 552     }
 553
 554     return undef;
 555 }
 556
 557 sub volume_snapshot_delete {
 558     my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 559
 560     my ($name, $vmid, $format) = ($class->parse_volname($volname))[1,2,6];
 561
 562     if ($format ne 'subvol' && $format ne 'raw') {
 563         return PVE::Storage::Plugin::volume_snapshot_delete(@_);
 564     }
 565
 566     my $path = $class->filesystem_path($scfg, $volname, $snap);
 567
 568     if ($format eq 'raw') {
 569         $path = raw_file_to_subvol($path);
 570     }
 571
 572     $class->btrfs_cmd(['subvolume', 'delete', '--', $path]);
 573
 574     return undef;
 575 }
 576
 577 sub volume_has_feature {
 578     my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
 579
 580     my $features = {
 581         snapshot => {
 582             current => { qcow2 => 1, raw => 1, subvol => 1 },
 583             snap => { qcow2 => 1, raw => 1, subvol => 1 }
 584         },
 585         clone => {
 586             base => { qcow2 => 1, raw => 1, subvol => 1, vmdk => 1 },
 587             current => { raw => 1 },
 588             snap => { raw => 1 },
 589         },
 590         template => { current => { qcow2 => 1, raw => 1, vmdk => 1, subvol => 1 } },
 591         copy => {
 592             base => { qcow2 => 1, raw => 1, subvol => 1, vmdk => 1 },
 593             current => { qcow2 => 1, raw => 1, subvol => 1, vmdk => 1 },
 594             snap => { qcow2 => 1, raw => 1, subvol => 1 },
 595         },
 596         sparseinit => { base => {qcow2 => 1, raw => 1, vmdk => 1 },
 597                         current => {qcow2 => 1, raw => 1, vmdk => 1 } },
 598     };
 599
 600     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
 601         $class->parse_volname($volname);
 602
 603     my $key = undef;
 604     if ($snapname) {
 605         $key = 'snap';
 606     } else {
 607         $key =  $isBase ? 'base' : 'current';
 608     }
 609
 610     return 1 if defined($features->{$feature}->{$key}->{$format});
 611
 612     return undef;
 613 }
 614
 615 sub list_images {
 616     my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
 617     my $imagedir = $class->get_subdir($scfg, 'images');
 618
 619     my $res = [];
 620
 621     # Copied from Plugin.pm, with file_size_info calls adapted:
 622     foreach my $fn (<$imagedir/[0-9][0-9]*/*>) {
 623         # different to in Plugin.pm the regex below also excludes '@' as valid file name
 624         next if $fn !~ m@^(/.+/(\d+)/([^/\@.]+(?:\.(qcow2|vmdk|subvol))?))$@;
 625         $fn = $1; # untaint
 626
 627         my $owner = $2;
 628         my $name = $3;
 629         my $ext = $4;
 630
 631         next if !$vollist && defined($vmid) && ($owner ne $vmid);
 632
 633         my $volid = "$storeid:$owner/$name";
 634         my ($size, $format, $used, $parent, $ctime);
 635
 636         if (!$ext) { # raw
 637             $volid .= '.raw';
 638             ($size, $format, $used, $parent, $ctime) = PVE::Storage::Plugin::file_size_info("$fn/disk.raw");
 639         } elsif ($ext eq 'subvol') {
 640             ($used, $size) = (0, 0);
 641             #($used, $size) = btrfs_subvol_quota($class, $fn);
 642             $format = 'subvol';
 643         } else {
 644             ($size, $format, $used, $parent, $ctime) = PVE::Storage::Plugin::file_size_info($fn);
 645         }
 646         next if !($format && defined($size));
 647
 648         if ($vollist) {
 649             next if ! grep { $_ eq $volid } @$vollist;
 650         }
 651
 652         my $info = {
 653             volid => $volid, format => $format,
 654             size => $size, vmid => $owner, used => $used, parent => $parent,
 655         };
 656
 657         $info->{ctime} = $ctime if $ctime;
 658
 659         push @$res, $info;
 660     }
 661
 662     return $res;
 663 }
 664
 665 sub volume_export_formats {
 666     my ($class, $scfg, $storeid, $volname, $snapshot, $base_snapshot, $with_snapshots) = @_;
 667
 668     # We can do whatever `DirPlugin` can do.
 669     my @result = PVE::Storage::Plugin::volume_export_formats(@_);
 670
 671     # `btrfs send` only works on snapshots:
 672     return @result if !defined $snapshot;
 673
 674     # Incremental stream with snapshots is only supported if the snapshots are listed (new api):
 675     return @result if defined($base_snapshot) && $with_snapshots && ref($with_snapshots) ne 'ARRAY';
 676
 677     # Otherwise we do also support `with_snapshots`.
 678
 679     # Finally, `btrfs send` only works on formats where we actually use btrfs subvolumes:
 680     my $format = ($class->parse_volname($volname))[6];
 681     return @result if $format ne 'raw' && $format ne 'subvol';
 682
 683     return ('btrfs', @result);
 684 }
 685
 686 sub volume_import_formats {
 687     my ($class, $scfg, $storeid, $volname, $snapshot, $base_snapshot, $with_snapshots) = @_;
 688
 689     # Same as export-formats, beware the parameter order:
 690     return volume_export_formats(
 691         $class,
 692         $scfg,
 693         $storeid,
 694         $volname,
 695         $snapshot,
 696         $base_snapshot,
 697         $with_snapshots,
 698     );
 699 }
 700
 701 sub volume_export {
 702     my (
 703         $class,
 704         $scfg,
 705         $storeid,
 706         $fh,
 707         $volname,
 708         $format,
 709         $snapshot,
 710         $base_snapshot,
 711         $with_snapshots,
 712     ) = @_;
 713
 714     if ($format ne 'btrfs') {
 715         return PVE::Storage::Plugin::volume_export(@_);
 716     }
 717
 718     die "format 'btrfs' only works on snapshots\n"
 719         if !defined $snapshot;
 720
 721     die "'btrfs' format in incremental mode requires snapshots to be listed explicitly\n"
 722         if defined($base_snapshot) && $with_snapshots && ref($with_snapshots) ne 'ARRAY';
 723
 724     my $volume_format = ($class->parse_volname($volname))[6];
 725
 726     die "btrfs-sending volumes of type $volume_format ('$volname') is not supported\n"
 727         if $volume_format ne 'raw' && $volume_format ne 'subvol';
 728
 729     my $path = $class->path($scfg, $volname, $storeid);
 730
 731     if ($volume_format eq 'raw') {
 732         $path = raw_file_to_subvol($path);
 733     }
 734
 735     my $cmd = ['btrfs', '-q', 'send', '-e'];
 736     if ($base_snapshot) {
 737         my $base = $class->path($scfg, $volname, $storeid, $base_snapshot);
 738         if ($volume_format eq 'raw') {
 739             $base = raw_file_to_subvol($base);
 740         }
 741         push @$cmd, '-p', $base;
 742     }
 743     push @$cmd, '--';
 744     if (ref($with_snapshots) eq 'ARRAY') {
 745         push @$cmd, (map { "$path\@$_" } ($with_snapshots // [])->@*), $path;
 746     } else {
 747         dir_glob_foreach(dirname($path), $BTRFS_VOL_REGEX, sub {
 748             push @$cmd, "$path\@$_[2]" if !(defined($snapshot) && $_[2] eq $snapshot);
 749         });
 750     }
 751     $path .= "\@$snapshot" if defined($snapshot);
 752     push @$cmd, $path;
 753
 754     run_command($cmd, output => '>&'.fileno($fh));
 755     return;
 756 }
 757
 758 sub volume_import {
 759     my (
 760         $class,
 761         $scfg,
 762         $storeid,
 763         $fh,
 764         $volname,
 765         $format,
 766         $snapshot,
 767         $base_snapshot,
 768         $with_snapshots,
 769         $allow_rename,
 770     ) = @_;
 771
 772     if ($format ne 'btrfs') {
 773         return PVE::Storage::Plugin::volume_import(@_);
 774     }
 775
 776     die "format 'btrfs' only works on snapshots\n"
 777         if !defined $snapshot;
 778
 779     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $volume_format) =
 780         $class->parse_volname($volname);
 781
 782     die "btrfs-receiving volumes of type $volume_format ('$volname') is not supported\n"
 783         if $volume_format ne 'raw' && $volume_format ne 'subvol';
 784
 785     if (defined($base_snapshot)) {
 786         my $path = $class->path($scfg, $volname, $storeid, $base_snapshot);
 787         die "base snapshot '$base_snapshot' not found - no such directory '$path'\n"
 788             if !path_is_subvolume($path);
 789     }
 790
 791     my $destination = $class->filesystem_path($scfg, $volname);
 792     if ($volume_format eq 'raw') {
 793         $destination = raw_file_to_subvol($destination);
 794     }
 795
 796     if (!defined($base_snapshot) && -e $destination) {
 797         die "volume $volname already exists\n" if !$allow_rename;
 798         $volname = $class->find_free_diskname($storeid, $scfg, $vmid, $volume_format, 1);
 799     }
 800
 801     my $imagedir = $class->get_subdir($scfg, $vtype);
 802     $imagedir .= "/$vmid" if $vtype eq 'images';
 803
 804     my $tmppath = "$imagedir/recv.$vmid.tmp";
 805     mkdir($imagedir); # FIXME: if $scfg->{mkdir};
 806     if (!mkdir($tmppath)) {
 807         die "temp receive directory already exists at '$tmppath', incomplete concurrent import?\n"
 808             if $! == EEXIST;
 809         die "failed to create temporary receive directory at '$tmppath' - $!\n";
 810     }
 811
 812     my $dh = IO::Dir->new($tmppath)
 813         or die "failed to open temporary receive directory '$tmppath' - $!\n";
 814     eval {
 815         run_command(['btrfs', '-q', 'receive', '-e', '--', $tmppath], input => '<&'.fileno($fh));
 816
 817         # Analyze the received subvolumes;
 818         my ($diskname, $found_snapshot, @snapshots);
 819         $dh->rewind;
 820         while (defined(my $entry = $dh->read)) {
 821             next if $entry eq '.' || $entry eq '..';
 822             next if $entry !~ /^$BTRFS_VOL_REGEX$/;
 823             my ($cur_diskname, $cur_snapshot) = ($1, $2);
 824
 825             die "send stream included a non-snapshot subvolume\n"
 826                 if !defined($cur_snapshot);
 827
 828             if (!defined($diskname)) {
 829                 $diskname = $cur_diskname;
 830             } else {
 831                 die "multiple disks contained in stream ('$diskname' vs '$cur_diskname')\n"
 832                     if $diskname ne $cur_diskname;
 833             }
 834
 835             if ($cur_snapshot eq $snapshot) {
 836                 $found_snapshot = 1;
 837             } else {
 838                 push @snapshots, $cur_snapshot;
 839             }
 840         }
 841
 842         die "send stream did not contain the expected current snapshot '$snapshot'\n"
 843             if !$found_snapshot;
 844
 845         # Rotate the disk into place, first the current state:
 846         # Note that read-only subvolumes cannot be moved into different directories, but for the
 847         # "current" state we also want a writable copy, so start with that:
 848         $class->btrfs_cmd(['property', 'set', "$tmppath/$diskname\@$snapshot", 'ro', 'false']);
 849         PVE::Tools::renameat2(
 850             -1,
 851             "$tmppath/$diskname\@$snapshot",
 852             -1,
 853             $destination,
 854             &PVE::Tools::RENAME_NOREPLACE,
 855         ) or die "failed to move received snapshot '$tmppath/$diskname\@$snapshot'"
 856             . " into place at '$destination' - $!\n";
 857
 858         # Now recreate the actual snapshot:
 859         $class->btrfs_cmd([
 860             'subvolume',
 861             'snapshot',
 862             '-r',
 863             '--',
 864             $destination,
 865             "$destination\@$snapshot",
 866         ]);
 867
 868         # Now go through the remaining snapshots (if any)
 869         foreach my $snap (@snapshots) {
 870             $class->btrfs_cmd(['property', 'set', "$tmppath/$diskname\@$snap", 'ro', 'false']);
 871             PVE::Tools::renameat2(
 872                 -1,
 873                 "$tmppath/$diskname\@$snap",
 874                 -1,
 875                 "$destination\@$snap",
 876                 &PVE::Tools::RENAME_NOREPLACE,
 877             ) or die "failed to move received snapshot '$tmppath/$diskname\@$snap'"
 878                 . " into place at '$destination\@$snap' - $!\n";
 879             eval { $class->btrfs_cmd(['property', 'set', "$destination\@$snap", 'ro', 'true']) };
 880             warn "failed to make $destination\@$snap read-only - $!\n" if $@;
 881         }
 882     };
 883     my $err = $@;
 884
 885     eval {
 886         # Cleanup all the received snapshots we did not move into place, so we can remove the temp
 887         # directory.
 888         if ($dh) {
 889             $dh->rewind;
 890             while (defined(my $entry = $dh->read)) {
 891                 next if $entry eq '.' || $entry eq '..';
 892                 eval { $class->btrfs_cmd(['subvolume', 'delete', '--', "$tmppath/$entry"]) };
 893                 warn $@ if $@;
 894             }
 895             $dh->close; undef $dh;
 896         }
 897         if (!rmdir($tmppath)) {
 898             warn "failed to remove temporary directory '$tmppath' - $!\n"
 899         }
 900     };
 901     warn $@ if $@;
 902     if ($err) {
 903         # clean up if the directory ended up being empty after an error
 904         rmdir($tmppath);
 905         die $err;
 906     }
 907
 908     return "$storeid:$volname";
 909 }
 910
 911 1