1 package PVE
::Storage
::BTRFSPlugin
;
6 use base
qw(PVE::Storage::Plugin);
8 use Fcntl
qw(S_ISDIR O_WRONLY O_CREAT O_EXCL);
9 use File
::Basename
qw(dirname);
10 use File
::Path
qw(mkpath);
13 use PVE
::Tools
qw(run_command);
15 use PVE
::Storage
::DirPlugin
;
18 BTRFS_FIRST_FREE_OBJECTID
=> 256,
19 FS_NOCOW_FL
=> 0x00800000,
20 FS_IOC_GETFLAGS
=> 0x40086602,
21 FS_IOC_SETFLAGS
=> 0x80086601,
24 # Configuration (similar to DirPlugin)
42 { images
=> 1, rootdir
=> 1 },
44 format
=> [ { raw
=> 1, qcow2
=> 1, vmdk
=> 1, subvol
=> 1 }, 'raw', ],
50 path
=> { fixed
=> 1 },
51 nodes
=> { optional
=> 1 },
52 shared
=> { optional
=> 1 },
53 disable
=> { optional
=> 1 },
54 maxfiles
=> { optional
=> 1 },
55 content
=> { optional
=> 1 },
56 format
=> { optional
=> 1 },
57 is_mountpoint
=> { optional
=> 1 },
58 # TODO: The new variant of mkdir with `populate` vs `create`...
62 # Storage implementation
64 # We use the same volume names are directory plugins, but map *raw* disk image file names into a
67 # `vm-VMID-disk-ID.raw`
68 # -> `images/VMID/vm-VMID-disk-ID/disk.raw`
69 # where the `vm-VMID-disk-ID/` subdirectory is a btrfs subvolume
71 # Reuse `DirPlugin`'s `check_config`. This simply checks for invalid paths.
73 my ($self, $sectionId, $config, $create, $skipSchemaCheck) = @_;
74 return PVE
::Storage
::DirPlugin
::check_config
($self, $sectionId, $config, $create, $skipSchemaCheck);
77 sub activate_storage
{
78 my ($class, $storeid, $scfg, $cache) = @_;
79 return PVE
::Storage
::DirPlugin
::activate_storage
($class, $storeid, $scfg, $cache);
83 my ($class, $storeid, $scfg, $cache) = @_;
84 return PVE
::Storage
::DirPlugin
::status
($class, $storeid, $scfg, $cache);
87 # TODO: sub get_volume_notes {}
89 # TODO: sub update_volume_notes {}
91 # croak would not include the caller from within this module
94 my (undef, $f, $n) = caller(1);
95 die "$msg at $f: $n\n";
98 # Given a name (eg. `vm-VMID-disk-ID.raw`), take the part up to the format suffix as the name of
99 # the subdirectory (subvolume).
100 sub raw_name_to_dir
($) {
103 # For the subvolume directory Strip the `.<format>` suffix:
104 if ($raw =~ /^(.*)\.raw$/) {
108 __error
"internal error: bad disk name: $raw";
111 sub raw_file_to_subvol
($) {
114 if ($file =~ m
|^(.*)/disk\
.raw
$|) {
118 __error
"internal error: bad raw path: $file";
121 sub filesystem_path
{
122 my ($class, $scfg, $volname, $snapname) = @_;
124 my ($vtype, $name, $vmid, undef, undef, $isBase, $format) =
125 $class->parse_volname($volname);
127 my $path = $class->get_subdir($scfg, $vtype);
129 $path .= "/$vmid" if $vtype eq 'images';
131 if ($format eq 'raw') {
132 my $dir = raw_name_to_dir
($name);
134 $dir .= "\@$snapname";
136 $path .= "/$dir/disk.raw";
137 } elsif ($format eq 'subvol') {
140 $path .= "\@$snapname";
146 return wantarray ?
($path, $vmid, $vtype) : $path;
150 my ($class, $cmd, $outfunc) = @_;
154 if (defined($outfunc)) {
156 my $part = &$outfunc(@_);
157 $msg .= $part if defined($part);
160 $func = sub { $msg .= "$_[0]\n" };
162 run_command
(['btrfs', '-q', @$cmd], errmsg
=> 'btrfs error', outfunc
=> $func);
167 sub btrfs_get_subvol_id
{
168 my ($class, $path) = @_;
169 my $info = $class->btrfs_cmd(['subvolume', 'show', '--', $path]);
170 if ($info !~ /^\s*(?:Object|Subvolume) ID:\s*(\d+)$/m) {
171 die "failed to get btrfs subvolume ID from: $info\n";
176 my sub chattr
: prototype($$$) {
177 my ($fh, $mask, $xor) = @_;
179 my $flags = pack('L!', 0);
180 ioctl($fh, FS_IOC_GETFLAGS
, $flags) or die "FS_IOC_GETFLAGS failed - $!\n";
181 $flags = pack('L!', (unpack('L!', $flags) & $mask) ^ $xor);
182 ioctl($fh, FS_IOC_SETFLAGS
, $flags) or die "FS_IOC_SETFLAGS failed - $!\n";
187 my ($class, $storeid, $scfg, $volname) = @_;
189 my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
190 $class->parse_volname($volname);
193 $newname =~ s/^vm-/base-/;
195 # If we're not working with a 'raw' file, which is the only thing that's "different" for btrfs,
196 # or a subvolume, we forward to the DirPlugin
197 if ($format ne 'raw' && $format ne 'subvol') {
198 return PVE
::Storage
::DirPlugin
::create_base
(@_);
201 my $path = $class->filesystem_path($scfg, $volname);
202 my $newvolname = $basename ?
"$basevmid/$basename/$vmid/$newname" : "$vmid/$newname";
203 my $newpath = $class->filesystem_path($scfg, $newvolname);
206 my $newsubvol = $newpath;
207 if ($format eq 'raw') {
208 $subvol = raw_file_to_subvol
($subvol);
209 $newsubvol = raw_file_to_subvol
($newsubvol);
212 rename($subvol, $newsubvol)
213 || die "rename '$subvol' to '$newsubvol' failed - $!\n";
214 eval { $class->btrfs_cmd(['property', 'set', $newsubvol, 'ro', 'true']) };
221 my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
223 my ($vtype, $basename, $basevmid, undef, undef, $isBase, $format) =
224 $class->parse_volname($volname);
226 # If we're not working with a 'raw' file, which is the only thing that's "different" for btrfs,
227 # or a subvolume, we forward to the DirPlugin
228 if ($format ne 'raw' && $format ne 'subvol') {
229 return PVE
::Storage
::DirPlugin
::clone_image
(@_);
232 my $imagedir = $class->get_subdir($scfg, 'images');
233 $imagedir .= "/$vmid";
236 my $path = $class->filesystem_path($scfg, $volname);
237 my $newname = $class->find_free_diskname($storeid, $scfg, $vmid, $format, 1);
239 # For btrfs subvolumes we don't actually need the "link":
240 #my $newvolname = "$basevmid/$basename/$vmid/$newname";
241 my $newvolname = "$vmid/$newname";
242 my $newpath = $class->filesystem_path($scfg, $newvolname);
245 my $newsubvol = $newpath;
246 if ($format eq 'raw') {
247 $subvol = raw_file_to_subvol
($subvol);
248 $newsubvol = raw_file_to_subvol
($newsubvol);
251 $class->btrfs_cmd(['subvolume', 'snapshot', '--', $subvol, $newsubvol]);
257 my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
259 if ($fmt ne 'raw' && $fmt ne 'subvol') {
260 return PVE
::Storage
::DirPlugin
::alloc_image
(@_);
265 my $imagedir = $class->get_subdir($scfg, 'images') . "/$vmid";
269 $name = $class->find_free_diskname($storeid, $scfg, $vmid, $fmt, 1) if !$name;
271 my (undef, $tmpfmt) = PVE
::Storage
::Plugin
::parse_name_dir
($name);
273 die "illegal name '$name' - wrong extension for format ('$tmpfmt != '$fmt')\n"
276 # End copy from Plugin.pm
278 my $subvol = "$imagedir/$name";
279 # .raw is not part of the directory name
280 $subvol =~ s/\.raw$//;
282 die "disk image '$subvol' already exists\n" if -e
$subvol;
286 $path = "$subvol/disk.raw";
289 if ($fmt eq 'subvol' && !!$size) {
290 # NOTE: `btrfs send/recv` actually drops quota information so supporting subvolumes with
291 # quotas doesn't play nice with send/recv.
292 die "btrfs quotas are currently not supported, use an unsized subvolume or a raw file\n";
295 $class->btrfs_cmd(['subvolume', 'create', '--', $subvol]);
298 if ($fmt eq 'subvol') {
299 # Nothing to do for now...
301 # This is how we *would* do it:
302 # # Use the subvol's default 0/$id qgroup
304 # # This call should happen at storage creation instead and therefore governed by a
305 # # configuration option!
306 # # $class->btrfs_cmd(['quota', 'enable', $subvol]);
307 # my $id = $class->btrfs_get_subvol_id($subvol);
308 # $class->btrfs_cmd(['qgroup', 'limit', "${size}k", "0/$id", $subvol]);
310 } elsif ($fmt eq 'raw') {
311 sysopen my $fh, $path, O_WRONLY
| O_CREAT
| O_EXCL
312 or die "failed to create raw file '$path' - $!\n";
313 chattr
($fh, ~FS_NOCOW_FL
, FS_NOCOW_FL
);
314 truncate($fh, $size * 1024)
315 or die "failed to set file size for '$path' - $!\n";
318 die "internal format error (format = $fmt)\n";
323 eval { $class->btrfs_cmd(['subvolume', 'delete', '--', $subvol]); };
328 return "$vmid/$name";
331 # Same as btrfsprogs does:
332 my sub path_is_subvolume
: prototype($) {
334 my @stat = stat($path)
335 or die "stat failed on '$path' - $!\n";
336 my ($ino, $mode) = @stat[1, 2];
337 return S_ISDIR
($mode) && $ino == BTRFS_FIRST_FREE_OBJECTID
;
340 my $BTRFS_VOL_REGEX = qr/((?:vm|base|subvol)-\d+-disk-\d+(?:\.subvol)?)(?:\@(\S+))$/;
342 # Calls `$code->($volume, $name, $snapshot)` for each subvol in a directory matching our volume
344 my sub foreach_subvol
: prototype($$) {
345 my ($dir, $code) = @_;
347 dir_glob_foreach
($dir, $BTRFS_VOL_REGEX, sub {
348 my ($volume, $name, $snapshot) = ($1, $2, $3);
349 return if !path_is_subvolume
("$dir/$volume");
350 $code->($volume, $name, $snapshot);
355 my ($class, $storeid, $scfg, $volname, $isBase, $_format) = @_;
357 my (undef, undef, $vmid, undef, undef, undef, $format) =
358 $class->parse_volname($volname);
360 if ($format ne 'subvol' && $format ne 'raw') {
361 return PVE
::Storage
::DirPlugin
::free_image
(@_);
364 my $path = $class->filesystem_path($scfg, $volname);
367 if ($format eq 'raw') {
368 $subvol = raw_file_to_subvol
($path);
371 my $dir = dirname
($subvol);
373 foreach_subvol
($dir, sub {
374 my ($volume, $name, $snapshot) = @_;
375 return if !defined $snapshot;
376 push @snapshot_vols, "$dir/$volume";
379 $class->btrfs_cmd(['subvolume', 'delete', '--', @snapshot_vols, $subvol]);
380 # try to cleanup directory to not clutter storage with empty $vmid dirs if
381 # all images from a guest got deleted
387 # Currently not used because quotas clash with send/recv.
388 # my sub btrfs_subvol_quota {
389 # my ($class, $path) = @_;
390 # my $id = '0/' . $class->btrfs_get_subvol_id($path);
391 # my $search = qr/^\Q$id\E\s+(\d)+\s+\d+\s+(\d+)\s*$/;
393 # $class->btrfs_cmd(['qgroup', 'show', '--raw', '-rf', '--', $path], sub {
394 # return if defined($size);
395 # if ($_[0] =~ $search) {
396 # ($used, $size) = ($1, $2);
399 # if (!defined($size)) {
400 # # syslog should include more information:
401 # syslog('err', "failed to get subvolume size for: $path (id $id)");
402 # # UI should only see the last path component:
404 # die "failed to get subvolume size for $path\n";
406 # return wantarray ? ($used, $size) : $size;
409 sub volume_size_info
{
410 my ($class, $scfg, $storeid, $volname, $timeout) = @_;
412 my $path = $class->filesystem_path($scfg, $volname);
414 my $format = ($class->parse_volname($volname))[6];
416 if ($format eq 'subvol') {
417 my $ctime = (stat($path))[10];
418 my ($used, $size) = (0, 0);
419 #my ($used, $size) = btrfs_subvol_quota($class, $path); # uses wantarray
420 return wantarray ?
($size, 'subvol', $used, undef, $ctime) : 1;
423 return PVE
::Storage
::Plugin
::file_size_info
($path, $timeout);
427 my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
429 my $format = ($class->parse_volname($volname))[6];
430 if ($format eq 'subvol') {
431 my $path = $class->filesystem_path($scfg, $volname);
432 my $id = '0/' . $class->btrfs_get_subvol_id($path);
433 $class->btrfs_cmd(['qgroup', 'limit', '--', "${size}k", "0/$id", $path]);
437 return PVE
::Storage
::Plugin
::volume_resize
(@_);
440 sub volume_snapshot
{
441 my ($class, $scfg, $storeid, $volname, $snap) = @_;
443 my ($name, $vmid, $format) = ($class->parse_volname($volname))[1,2,6];
444 if ($format ne 'subvol' && $format ne 'raw') {
445 return PVE
::Storage
::Plugin
::volume_snapshot
(@_);
448 my $path = $class->filesystem_path($scfg, $volname);
449 my $snap_path = $class->filesystem_path($scfg, $volname, $snap);
451 if ($format eq 'raw') {
452 $path = raw_file_to_subvol
($path);
453 $snap_path = raw_file_to_subvol
($snap_path);
456 my $snapshot_dir = $class->get_subdir($scfg, 'images') . "/$vmid";
457 mkpath
$snapshot_dir;
459 $class->btrfs_cmd(['subvolume', 'snapshot', '-r', '--', $path, $snap_path]);
463 sub volume_rollback_is_possible
{
464 my ($class, $scfg, $storeid, $volname, $snap) = @_;
469 sub volume_snapshot_rollback
{
470 my ($class, $scfg, $storeid, $volname, $snap) = @_;
472 my ($name, $format) = ($class->parse_volname($volname))[1,6];
474 if ($format ne 'subvol' && $format ne 'raw') {
475 return PVE
::Storage
::Plugin
::volume_snapshot_rollback
(@_);
478 my $path = $class->filesystem_path($scfg, $volname);
479 my $snap_path = $class->filesystem_path($scfg, $volname, $snap);
481 if ($format eq 'raw') {
482 $path = raw_file_to_subvol
($path);
483 $snap_path = raw_file_to_subvol
($snap_path);
486 # Simple version would be:
489 # on error rename temp back
490 # But for atomicity in case the rename after create-failure *also* fails, we create the new
491 # subvol first, then use RENAME_EXCHANGE,
492 my $tmp_path = "$path.tmp.$$";
493 $class->btrfs_cmd(['subvolume', 'snapshot', '--', $snap_path, $tmp_path]);
494 # The paths are absolute, so pass -1 as file descriptors.
495 my $ok = PVE
::Tools
::renameat2
(-1, $tmp_path, -1, $path, &PVE
::Tools
::RENAME_EXCHANGE
);
497 eval { $class->btrfs_cmd(['subvolume', 'delete', '--', $tmp_path]) };
498 warn "failed to remove '$tmp_path' subvolume: $@" if $@;
501 die "failed to rotate '$tmp_path' into place at '$path' - $!\n";
507 sub volume_snapshot_delete
{
508 my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
510 my ($name, $vmid, $format) = ($class->parse_volname($volname))[1,2,6];
512 if ($format ne 'subvol' && $format ne 'raw') {
513 return PVE
::Storage
::Plugin
::volume_snapshot_delete
(@_);
516 my $path = $class->filesystem_path($scfg, $volname, $snap);
518 if ($format eq 'raw') {
519 $path = raw_file_to_subvol
($path);
522 $class->btrfs_cmd(['subvolume', 'delete', '--', $path]);
527 sub volume_has_feature
{
528 my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
532 current
=> { qcow2
=> 1, raw
=> 1, subvol
=> 1 },
533 snap
=> { qcow2
=> 1, raw
=> 1, subvol
=> 1 }
536 base
=> { qcow2
=> 1, raw
=> 1, subvol
=> 1, vmdk
=> 1 },
537 current
=> { raw
=> 1 },
538 snap
=> { raw
=> 1 },
540 template
=> { current
=> { qcow2
=> 1, raw
=> 1, vmdk
=> 1, subvol
=> 1 } },
542 base
=> { qcow2
=> 1, raw
=> 1, subvol
=> 1, vmdk
=> 1 },
543 current
=> { qcow2
=> 1, raw
=> 1, subvol
=> 1, vmdk
=> 1 },
544 snap
=> { qcow2
=> 1, raw
=> 1, subvol
=> 1 },
546 sparseinit
=> { base
=> {qcow2
=> 1, raw
=> 1, vmdk
=> 1 },
547 current
=> {qcow2
=> 1, raw
=> 1, vmdk
=> 1 } },
550 my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
551 $class->parse_volname($volname);
557 $key = $isBase ?
'base' : 'current';
560 return 1 if defined($features->{$feature}->{$key}->{$format});
566 my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
567 my $imagedir = $class->get_subdir($scfg, 'images');
571 # Copied from Plugin.pm, with file_size_info calls adapted:
572 foreach my $fn (<$imagedir/[0-9][0-9]*/*>) {
573 # different to in Plugin.pm the regex below also excludes '@' as valid file name
574 next if $fn !~ m
@^(/.+/(\d
+)/([^/\
@.]+(?
:\
.(qcow2
|vmdk
|subvol
))?
))$@;
581 next if !$vollist && defined($vmid) && ($owner ne $vmid);
583 my $volid = "$storeid:$owner/$name";
584 my ($size, $format, $used, $parent, $ctime);
588 ($size, $format, $used, $parent, $ctime) = PVE
::Storage
::Plugin
::file_size_info
("$fn/disk.raw");
589 } elsif ($ext eq 'subvol') {
590 ($used, $size) = (0, 0);
591 #($used, $size) = btrfs_subvol_quota($class, $fn);
594 ($size, $format, $used, $parent, $ctime) = PVE
::Storage
::Plugin
::file_size_info
($fn);
596 next if !($format && defined($size));
599 next if ! grep { $_ eq $volid } @$vollist;
603 volid
=> $volid, format
=> $format,
604 size
=> $size, vmid
=> $owner, used
=> $used, parent
=> $parent,
607 $info->{ctime
} = $ctime if $ctime;
615 # For now we don't implement `btrfs send/recv` as it needs some updates to our import/export API
618 sub volume_export_formats
{
619 return PVE
::Storage
::DirPlugin
::volume_export_formats
(@_);
623 return PVE
::Storage
::DirPlugin
::volume_export
(@_);
626 sub volume_import_formats
{
627 return PVE
::Storage
::DirPlugin
::volume_import_formats
(@_);
631 return PVE
::Storage
::DirPlugin
::volume_import
(@_);