PVE/Storage/ZFSPoolPlugin.pm

   1 package PVE::Storage::ZFSPoolPlugin;
   2
   3 use strict;
   4 use warnings;
   5 use IO::File;
   6 use POSIX;
   7 use PVE::Tools qw(run_command);
   8 use PVE::Storage::Plugin;
   9 use PVE::RPCEnvironment;
  10
  11 use base qw(PVE::Storage::Plugin);
  12
  13 sub type {
  14     return 'zfspool';
  15 }
  16
  17 sub plugindata {
  18     return {
  19         content => [ {images => 1, rootdir => 1}, {images => 1 , rootdir => 1}],
  20         format => [ { raw => 1, subvol => 1 } , 'raw' ],
  21     };
  22 }
  23
  24 sub properties {
  25     return {
  26         blocksize => {
  27             description => "block size",
  28             type => 'string',
  29         },
  30         sparse => {
  31             description => "use sparse volumes",
  32             type => 'boolean',
  33         },
  34     };
  35 }
  36
  37 sub options {
  38     return {
  39         pool => { fixed => 1 },
  40         blocksize => { optional => 1 },
  41         sparse => { optional => 1 },
  42         nodes => { optional => 1 },
  43         disable => { optional => 1 },
  44         content => { optional => 1 },
  45     };
  46 }
  47
  48 # static zfs helper methods
  49
  50 sub zfs_parse_size {
  51     my ($text) = @_;
  52
  53     return 0 if !$text;
  54
  55     if ($text =~ m/^(\d+(\.\d+)?)([TGMK])?$/) {
  56
  57         my ($size, $reminder, $unit) = ($1, $2, $3);
  58
  59         if ($unit) {
  60             if ($unit eq 'K') {
  61                 $size *= 1024;
  62             } elsif ($unit eq 'M') {
  63                 $size *= 1024*1024;
  64             } elsif ($unit eq 'G') {
  65                 $size *= 1024*1024*1024;
  66             } elsif ($unit eq 'T') {
  67                 $size *= 1024*1024*1024*1024;
  68             } else {
  69                 die "got unknown zfs size unit '$unit'\n";
  70             }
  71         }
  72
  73         if ($reminder) {
  74             $size = ceil($size);
  75         }
  76
  77         return $size;
  78
  79     }
  80
  81     warn "unable to parse zfs size '$text'\n";
  82
  83     return 0;
  84 }
  85
  86 sub zfs_parse_zvol_list {
  87     my ($text) = @_;
  88
  89     my $list = ();
  90
  91     return $list if !$text;
  92
  93     my @lines = split /\n/, $text;
  94     foreach my $line (@lines) {
  95         my ($dataset, $size, $origin, $type, $refquota) = split(/\s+/, $line);
  96         next if !($type eq 'volume' || $type eq 'filesystem');
  97
  98         my $zvol = {};
  99         my @parts = split /\//, $dataset;
 100         next if scalar(@parts) < 2; # we need pool/name
 101         my $name = pop @parts;
 102         my $pool = join('/', @parts);
 103
 104         next unless $name =~ m!^(vm|base|subvol|basevol)-(\d+)-(\S+)$!;
 105         $zvol->{owner} = $2;
 106
 107         $zvol->{pool} = $pool;
 108         $zvol->{name} = $name;
 109         if ($type eq 'filesystem') {
 110             if ($refquota eq 'none') {
 111                 $zvol->{size} = 0;
 112             } else {
 113                 $zvol->{size} = zfs_parse_size($refquota);
 114             }
 115             $zvol->{format} = 'subvol';
 116         } else {
 117             $zvol->{size} = zfs_parse_size($size);
 118             $zvol->{format} = 'raw';
 119         }
 120         if ($origin !~ /^-$/) {
 121             $zvol->{origin} = $origin;
 122         }
 123         push @$list, $zvol;
 124     }
 125
 126     return $list;
 127 }
 128
 129 sub parse_volname {
 130     my ($class, $volname) = @_;
 131
 132     if ($volname =~ m/^(((base|basevol)-(\d+)-\S+)\/)?((base|basevol|vm|subvol)-(\d+)-\S+)$/) {
 133         my $format = ($6 eq 'subvol' || $6 eq 'basevol') ? 'subvol' : 'raw';
 134         my $isBase = ($6 eq 'base' || $6 eq 'basevol');
 135         return ('images', $5, $7, $2, $4, $isBase, $format);
 136     }
 137
 138     die "unable to parse zfs volume name '$volname'\n";
 139 }
 140
 141 # virtual zfs methods (subclass can overwrite them)
 142
 143 sub path {
 144     my ($class, $scfg, $volname, $storeid, $snapname) = @_;
 145
 146     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
 147
 148     my $path = '';
 149
 150     if ($vtype eq "images") {
 151         if ($name =~ m/^subvol-/ || $name =~ m/^basevol-/) {
 152             # fixme: we currently assume standard mount point?!
 153             $path = "/$scfg->{pool}/$name";
 154         } else {
 155             $path = "/dev/zvol/$scfg->{pool}/$name";
 156         }
 157         $path .= "\@$snapname" if defined($snapname);
 158     } else {
 159         die "$vtype is not allowed in ZFSPool!";
 160     }
 161
 162     return ($path, $vmid, $vtype);
 163 }
 164
 165 sub zfs_request {
 166     my ($class, $scfg, $timeout, $method, @params) = @_;
 167
 168     my $default_timeout = PVE::RPCEnvironment->is_worker() ? 60*60 : 5;
 169
 170     my $cmd = [];
 171
 172     if ($method eq 'zpool_list') {
 173         push @$cmd, 'zpool', 'list';
 174     } elsif ($method eq 'zpool_import') {
 175         push @$cmd, 'zpool', 'import';
 176         $default_timeout = 15 if $default_timeout < 15;
 177     } else {
 178         push @$cmd, 'zfs', $method;
 179     }
 180
 181     push @$cmd, @params;
 182
 183     my $msg = '';
 184
 185     my $output = sub {
 186         my $line = shift;
 187         $msg .= "$line\n";
 188     };
 189
 190     $timeout =  $default_timeout if !$timeout;
 191
 192     run_command($cmd, errmsg => "zfs error", outfunc => $output, timeout => $timeout);
 193
 194     return $msg;
 195 }
 196
 197 sub alloc_image {
 198     my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
 199
 200     my $volname = $name;
 201
 202     if ($fmt eq 'raw') {
 203
 204         die "illegal name '$volname' - sould be 'vm-$vmid-*'\n"
 205             if $volname && $volname !~ m/^vm-$vmid-/;
 206         $volname = $class->zfs_find_free_diskname($storeid, $scfg, $vmid, $fmt)
 207             if !$volname;
 208
 209         $class->zfs_create_zvol($scfg, $volname, $size);
 210         my $devname = "/dev/zvol/$scfg->{pool}/$volname";
 211
 212         run_command("udevadm trigger --subsystem-match block");
 213         system('udevadm', 'settle', '--timeout', '10', "--exit-if-exists=${devname}");
 214
 215     } elsif ( $fmt eq 'subvol') {
 216
 217         die "illegal name '$volname' - sould be 'subvol-$vmid-*'\n"
 218             if $volname && $volname !~ m/^subvol-$vmid-/;
 219         $volname = $class->zfs_find_free_diskname($storeid, $scfg, $vmid, $fmt)
 220             if !$volname;
 221
 222         die "illegal name '$volname' - sould be 'subvol-$vmid-*'\n"
 223             if $volname !~ m/^subvol-$vmid-/;
 224
 225         $class->zfs_create_subvol($scfg, $volname, $size);
 226
 227     } else {
 228         die "unsupported format '$fmt'";
 229     }
 230
 231     return $volname;
 232 }
 233
 234 sub free_image {
 235     my ($class, $storeid, $scfg, $volname, $isBase) = @_;
 236
 237     my (undef, $name, undef) = $class->parse_volname($volname);
 238
 239     $class->zfs_delete_zvol($scfg, $name);
 240
 241     return undef;
 242 }
 243
 244 sub list_images {
 245     my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
 246
 247     $cache->{zfs} = $class->zfs_list_zvol($scfg) if !$cache->{zfs};
 248     my $zfspool = $scfg->{pool};
 249     my $res = [];
 250
 251     if (my $dat = $cache->{zfs}->{$zfspool}) {
 252
 253         foreach my $image (keys %$dat) {
 254
 255             my $info = $dat->{$image};
 256
 257             my $volname = $info->{name};
 258             my $parent = $info->{parent};
 259             my $owner = $info->{vmid};
 260
 261             if ($parent && $parent =~ m/^(\S+)\@__base__$/) {
 262                 my ($basename) = ($1);
 263                 $info->{volid} = "$storeid:$basename/$volname";
 264             } else {
 265                 $info->{volid} = "$storeid:$volname";
 266             }
 267
 268             if ($vollist) {
 269                 my $found = grep { $_ eq $info->{volid} } @$vollist;
 270                 next if !$found;
 271             } else {
 272                 next if defined ($vmid) && ($owner ne $vmid);
 273             }
 274
 275             push @$res, $info;
 276         }
 277     }
 278     return $res;
 279 }
 280
 281 sub zfs_get_pool_stats {
 282     my ($class, $scfg) = @_;
 283
 284     my $available = 0;
 285     my $used = 0;
 286
 287     my $text = $class->zfs_request($scfg, undef, 'get', '-o', 'value', '-Hp',
 288                'available,used', $scfg->{pool});
 289
 290     my @lines = split /\n/, $text;
 291
 292     if($lines[0] =~ /^(\d+)$/) {
 293         $available = $1;
 294     }
 295
 296     if($lines[1] =~ /^(\d+)$/) {
 297         $used = $1;
 298     }
 299
 300     return ($available, $used);
 301 }
 302
 303 sub zfs_create_zvol {
 304     my ($class, $scfg, $zvol, $size) = @_;
 305
 306     my $cmd = ['create'];
 307
 308     push @$cmd, '-s' if $scfg->{sparse};
 309
 310     push @$cmd, '-b', $scfg->{blocksize} if $scfg->{blocksize};
 311
 312     push @$cmd, '-V', "${size}k", "$scfg->{pool}/$zvol";
 313
 314     $class->zfs_request($scfg, undef, @$cmd);
 315 }
 316
 317 sub zfs_create_subvol {
 318     my ($class, $scfg, $volname, $size) = @_;
 319
 320     my $dataset = "$scfg->{pool}/$volname";
 321
 322     my $cmd = ['create', '-o', 'acltype=posixacl', '-o', 'xattr=sa',
 323                '-o', "refquota=${size}k", $dataset];
 324
 325     $class->zfs_request($scfg, undef, @$cmd);
 326 }
 327
 328 sub zfs_delete_zvol {
 329     my ($class, $scfg, $zvol) = @_;
 330
 331     my $err;
 332
 333     for (my $i = 0; $i < 6; $i++) {
 334
 335         eval { $class->zfs_request($scfg, undef, 'destroy', '-r', "$scfg->{pool}/$zvol"); };
 336         if ($err = $@) {
 337             if ($err =~ m/^zfs error:(.*): dataset is busy.*/) {
 338                 sleep(1);
 339             } elsif ($err =~ m/^zfs error:.*: dataset does not exist.*$/) {
 340                 $err = undef;
 341                 last;
 342             } else {
 343                 die $err;
 344             }
 345         } else {
 346             last;
 347         }
 348     }
 349
 350     die $err if $err;
 351 }
 352
 353 sub zfs_list_zvol {
 354     my ($class, $scfg) = @_;
 355
 356     my $text = $class->zfs_request($scfg, 10, 'list', '-o', 'name,volsize,origin,type,refquota', '-t', 'volume,filesystem', '-Hr');
 357     my $zvols = zfs_parse_zvol_list($text);
 358     return undef if !$zvols;
 359
 360     my $list = ();
 361     foreach my $zvol (@$zvols) {
 362         my $pool = $zvol->{pool};
 363         my $name = $zvol->{name};
 364         my $parent = $zvol->{origin};
 365         if($zvol->{origin} && $zvol->{origin} =~ m/^$scfg->{pool}\/(\S+)$/){
 366             $parent = $1;
 367         }
 368
 369         $list->{$pool}->{$name} = {
 370             name => $name,
 371             size => $zvol->{size},
 372             parent => $parent,
 373             format => $zvol->{format},
 374             vmid => $zvol->{owner},
 375         };
 376     }
 377
 378     return $list;
 379 }
 380
 381 sub zfs_find_free_diskname {
 382     my ($class, $storeid, $scfg, $vmid, $format) = @_;
 383
 384     my $name = undef;
 385     my $volumes = $class->zfs_list_zvol($scfg);
 386
 387     my $disk_ids = {};
 388     my $dat = $volumes->{$scfg->{pool}};
 389
 390     foreach my $image (keys %$dat) {
 391         my $volname = $dat->{$image}->{name};
 392         if ($volname =~ m/(vm|base|subvol|basevol)-$vmid-disk-(\d+)/){
 393             $disk_ids->{$2} = 1;
 394         }
 395     }
 396
 397     for (my $i = 1; $i < 100; $i++) {
 398         if (!$disk_ids->{$i}) {
 399             return $format eq 'subvol' ? "subvol-$vmid-disk-$i" : "vm-$vmid-disk-$i";
 400         }
 401     }
 402
 403     die "unable to allocate an image name for VM $vmid in storage '$storeid'\n";
 404 }
 405
 406 sub zfs_get_latest_snapshot {
 407     my ($class, $scfg, $volname) = @_;
 408
 409     my $vname = ($class->parse_volname($volname))[1];
 410
 411     # abort rollback if snapshot is not the latest
 412     my @params = ('-t', 'snapshot', '-o', 'name', '-s', 'creation');
 413     my $text = $class->zfs_request($scfg, undef, 'list', @params);
 414     my @snapshots = split(/\n/, $text);
 415
 416     my $recentsnap;
 417     foreach (@snapshots) {
 418         if (/$scfg->{pool}\/$vname/) {
 419             s/^.*@//;
 420             $recentsnap = $_;
 421         }
 422     }
 423
 424     return $recentsnap;
 425 }
 426
 427 sub status {
 428     my ($class, $storeid, $scfg, $cache) = @_;
 429
 430     my $total = 0;
 431     my $free = 0;
 432     my $used = 0;
 433     my $active = 0;
 434
 435     eval {
 436         ($free, $used) = $class->zfs_get_pool_stats($scfg);
 437         $active = 1;
 438         $total = $free + $used;
 439     };
 440     warn $@ if $@;
 441
 442     return ($total, $free, $used, $active);
 443 }
 444
 445 sub volume_size_info {
 446     my ($class, $scfg, $storeid, $volname, $timeout) = @_;
 447
 448     my (undef, $vname, undef, undef, undef, undef, $format) =
 449         $class->parse_volname($volname);
 450
 451     my $attr = $format eq 'subvol' ? 'refquota' : 'volsize';
 452     my $text = $class->zfs_request($scfg, undef, 'get', '-Hp', $attr, "$scfg->{pool}/$vname");
 453     if ($text =~ /\s$attr\s(\d+)\s/) {
 454         return $1;
 455     }
 456
 457     die "Could not get zfs volume size\n";
 458 }
 459
 460 sub volume_snapshot {
 461     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 462
 463     my $vname = ($class->parse_volname($volname))[1];
 464
 465     $class->zfs_request($scfg, undef, 'snapshot', "$scfg->{pool}/$vname\@$snap");
 466 }
 467
 468 sub volume_send {
 469     my ($class, $scfg, $storeid, $volname, $ip, $snap,
 470         $incremental_snap, $verbose, $limit, $target_path) = @_;
 471
 472     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
 473
 474     my $zpath = "$scfg->{pool}/$name";
 475
 476     die "$vtype is not allowed in ZFSPool!" if ($vtype ne "images");
 477
 478     my $cmdsend = [];
 479     my $cmdlimit = [];
 480
 481     push @$cmdsend, 'zfs', 'send', '-R';
 482     push @$cmdsend, '-v' if defined($verbose);
 483
 484     if( defined($incremental_snap)) {
 485         push @$cmdsend, '-I', "$zpath\@${incremental_snap}";
 486     }
 487
 488     push @$cmdsend, '--', "$zpath\@${snap}";
 489
 490     # limit in kByte/s
 491     if ($limit){
 492         my $bwl = $limit * 1024;
 493         push @$cmdlimit, 'cstream', '-t', $bwl;
 494     }
 495
 496     my $cmdrecv = [];
 497
 498     push @$cmdrecv, 'ssh', '-o', 'BatchMode=yes', "root\@${ip}", '--' if $ip;
 499     push @$cmdrecv, 'zfs', 'recv', '-F', '--';
 500
 501     $zpath = $target_path if defined($target_path);
 502     push @$cmdrecv, $zpath;
 503
 504
 505     if ($limit) {
 506         eval { run_command([$cmdsend, $cmdlimit, $cmdrecv]) };
 507     } else {
 508         eval { run_command([$cmdsend, $cmdrecv]) };
 509     }
 510
 511     if (my $err = $@) {
 512         die $err;
 513     }
 514 }
 515
 516 sub volume_snapshot_delete {
 517     my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 518
 519     my $vname = ($class->parse_volname($volname))[1];
 520
 521     $class->deactivate_volume($storeid, $scfg, $vname, $snap, {});
 522     $class->zfs_request($scfg, undef, 'destroy', "$scfg->{pool}/$vname\@$snap");
 523 }
 524
 525 sub volume_snapshot_rollback {
 526     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 527
 528     my $vname = ($class->parse_volname($volname))[1];
 529
 530     $class->zfs_request($scfg, undef, 'rollback', "$scfg->{pool}/$vname\@$snap");
 531 }
 532
 533 sub volume_rollback_is_possible {
 534     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 535
 536     my $recentsnap = $class->zfs_get_latest_snapshot($scfg, $volname);
 537     if ($snap ne $recentsnap) {
 538         die "can't rollback, more recent snapshots exist\n";
 539     }
 540
 541     return 1;
 542 }
 543
 544 sub activate_storage {
 545     my ($class, $storeid, $scfg, $cache) = @_;
 546
 547     # Note: $scfg->{pool} can include dataset <pool>/<dataset>
 548     my $pool = $scfg->{pool};
 549     $pool =~ s!/.*$!!;
 550
 551     my @param = ('-o', 'name', '-H', "$pool");
 552     my $res;
 553     eval {
 554         $res = $class->zfs_request($scfg, undef, 'zpool_list', @param);
 555     };
 556
 557     if ($@ || !defined($res) || $res !~ $pool) {
 558         eval {
 559             @param = ('-d', '/dev/disk/by-id/', "$pool");
 560             $class->zfs_request($scfg, undef, 'zpool_import', @param);
 561         };
 562         die "could not activate storage '$storeid', $@\n" if $@;
 563     }
 564     return 1;
 565 }
 566
 567 sub deactivate_storage {
 568     my ($class, $storeid, $scfg, $cache) = @_;
 569     return 1;
 570 }
 571
 572 sub activate_volume {
 573     my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
 574     return 1;
 575 }
 576
 577 sub deactivate_volume {
 578     my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
 579     return 1;
 580 }
 581
 582 sub clone_image {
 583     my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
 584
 585     $snap ||= '__base__';
 586
 587     my ($vtype, $basename, $basevmid, undef, undef, $isBase, $format) =
 588         $class->parse_volname($volname);
 589
 590     die "clone_image only works on base images\n" if !$isBase;
 591
 592     my $name = $class->zfs_find_free_diskname($storeid, $scfg, $vmid, $format);
 593
 594     if ($format eq 'subvol') {
 595         my $size = $class->zfs_request($scfg, undef, 'list', '-H', '-o', 'refquota', "$scfg->{pool}/$basename");
 596         chomp($size);
 597         $class->zfs_request($scfg, undef, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$name", '-o', "refquota=$size");
 598     } else {
 599         $class->zfs_request($scfg, undef, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$name");
 600     }
 601
 602     return "$basename/$name";
 603 }
 604
 605 sub create_base {
 606     my ($class, $storeid, $scfg, $volname) = @_;
 607
 608     my $snap = '__base__';
 609
 610     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
 611         $class->parse_volname($volname);
 612
 613     die "create_base not possible with base image\n" if $isBase;
 614
 615     my $newname = $name;
 616     if ( $format eq 'subvol' ) {
 617         $newname =~ s/^subvol-/basevol-/;
 618     } else {
 619         $newname =~ s/^vm-/base-/;
 620     }
 621     my $newvolname = $basename ? "$basename/$newname" : "$newname";
 622
 623     $class->zfs_request($scfg, undef, 'rename', "$scfg->{pool}/$name", "$scfg->{pool}/$newname");
 624
 625     my $running  = undef; #fixme : is create_base always offline ?
 626
 627     $class->volume_snapshot($scfg, $storeid, $newname, $snap, $running);
 628
 629     return $newvolname;
 630 }
 631
 632 sub volume_resize {
 633     my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
 634
 635     my $new_size = int($size/1024);
 636
 637     my (undef, $vname, undef, undef, undef, undef, $format) =
 638         $class->parse_volname($volname);
 639
 640     my $attr = $format eq 'subvol' ? 'refquota' : 'volsize';
 641
 642     $class->zfs_request($scfg, undef, 'set', "$attr=${new_size}k", "$scfg->{pool}/$vname");
 643
 644     return $new_size;
 645 }
 646
 647 sub volume_has_feature {
 648     my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
 649
 650     my $features = {
 651         snapshot => { current => 1, snap => 1},
 652         clone => { base => 1},
 653         template => { current => 1},
 654         copy => { base => 1, current => 1},
 655         sparseinit => { base => 1, current => 1},
 656     };
 657
 658     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
 659         $class->parse_volname($volname);
 660
 661     my $key = undef;
 662
 663     if ($snapname) {
 664         $key = 'snap';
 665     } else {
 666         $key = $isBase ? 'base' : 'current';
 667     }
 668
 669     return 1 if $features->{$feature}->{$key};
 670
 671     return undef;
 672 }
 673
 674 1;