PVE/Storage/ZFSPlugin.pm

   1 package PVE::Storage::ZFSPlugin;
   2
   3 use strict;
   4 use warnings;
   5 use IO::File;
   6 use POSIX;
   7 use PVE::Tools qw(run_command);
   8 use PVE::Storage::Plugin;
   9
  10 use base qw(PVE::Storage::Plugin);
  11 use PVE::Storage::LunCmd::Comstar;
  12 use PVE::Storage::LunCmd::Istgt;
  13 use PVE::Storage::LunCmd::Iet;
  14
  15 my @ssh_opts = ('-o', 'BatchMode=yes');
  16 my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
  17 my $id_rsa_path = '/etc/pve/priv/zfs';
  18
  19 my $lun_cmds = {
  20     create_lu   => 1,
  21     delete_lu   => 1,
  22     import_lu   => 1,
  23     modify_lu   => 1,
  24     add_view    => 1,
  25     list_view   => 1,
  26     list_lu     => 1,
  27 };
  28
  29 my $zfs_unknown_scsi_provider = sub {
  30     my ($provider) = @_;
  31
  32     die "$provider: unknown iscsi provider. Available [comstar, istgt, iet]";
  33 };
  34
  35 my $zfs_get_base = sub {
  36     my ($scfg) = @_;
  37
  38     if ($scfg->{iscsiprovider} eq 'comstar') {
  39         return PVE::Storage::LunCmd::Comstar::get_base;
  40     } elsif ($scfg->{iscsiprovider} eq 'istgt') {
  41         return PVE::Storage::LunCmd::Istgt::get_base;
  42     } elsif ($scfg->{iscsiprovider} eq 'iet') {
  43         return PVE::Storage::LunCmd::Iet::get_base;
  44     } else {
  45         $zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
  46     }
  47 };
  48
  49 sub zfs_request {
  50     my ($scfg, $timeout, $method, @params) = @_;
  51
  52     my $cmdmap;
  53     my $zfscmd;
  54     my $target;
  55     my $msg;
  56
  57     $timeout = 5 if !$timeout;
  58
  59     if ($lun_cmds->{$method}) {
  60         if ($scfg->{iscsiprovider} eq 'comstar') {
  61             $msg = PVE::Storage::LunCmd::Comstar::run_lun_command($scfg, $timeout, $method, @params);
  62         } elsif ($scfg->{iscsiprovider} eq 'istgt') {
  63             $msg = PVE::Storage::LunCmd::Istgt::run_lun_command($scfg, $timeout, $method, @params);
  64         } elsif ($scfg->{iscsiprovider} eq 'iet') {
  65             $msg = PVE::Storage::LunCmd::Iet::run_lun_command($scfg, $timeout, $method, @params);
  66         } else {
  67             $zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
  68         }
  69     } else {
  70         if ($method eq 'zpool_list') {
  71             $zfscmd = 'zpool';
  72             $method = 'list',
  73         } else {
  74             $zfscmd = 'zfs';
  75         }
  76
  77         $target = 'root@' . $scfg->{portal};
  78
  79         my $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, $zfscmd, $method, @params];
  80
  81         $msg = '';
  82
  83         my $output = sub {
  84         my $line = shift;
  85         $msg .= "$line\n";
  86         };
  87
  88         run_command($cmd, outfunc => $output, timeout => $timeout);
  89     }
  90
  91     return $msg;
  92 }
  93
  94 sub zfs_parse_size {
  95     my ($text) = @_;
  96
  97     return 0 if !$text;
  98
  99     if ($text =~ m/^(\d+(\.\d+)?)([TGMK])?$/) {
 100     my ($size, $reminder, $unit) = ($1, $2, $3);
 101     return $size if !$unit;
 102     if ($unit eq 'K') {
 103         $size *= 1024;
 104     } elsif ($unit eq 'M') {
 105         $size *= 1024*1024;
 106     } elsif ($unit eq 'G') {
 107         $size *= 1024*1024*1024;
 108     } elsif ($unit eq 'T') {
 109         $size *= 1024*1024*1024*1024;
 110     }
 111
 112     if ($reminder) {
 113         $size = ceil($size);
 114     }
 115     return $size;
 116     } else {
 117     return 0;
 118     }
 119 }
 120
 121 sub zfs_get_pool_stats {
 122     my ($scfg) = @_;
 123
 124     my $available = 0;
 125     my $used = 0;
 126
 127     my $text = zfs_request($scfg, undef, 'get', '-o', 'value', '-Hp',
 128                'available,used', $scfg->{pool});
 129
 130     my @lines = split /\n/, $text;
 131
 132     if($lines[0] =~ /^(\d+)$/) {
 133     $available = $1;
 134     }
 135
 136     if($lines[1] =~ /^(\d+)$/) {
 137     $used = $1;
 138     }
 139
 140     return ($available, $used);
 141 }
 142
 143 sub zfs_parse_zvol_list {
 144     my ($text) = @_;
 145
 146     my $list = ();
 147
 148     return $list if !$text;
 149
 150     my @lines = split /\n/, $text;
 151     foreach my $line (@lines) {
 152     if ($line =~ /^(.+)\s+([a-zA-Z0-9\.]+|\-)\s+(.+)$/) {
 153         my $zvol = {};
 154         my @parts = split /\//, $1;
 155         my $name = pop @parts;
 156         my $pool = join('/', @parts);
 157
 158         if ($pool !~ /^rpool$/) {
 159             next unless $name =~ m!^(\w+)-(\d+)-(\w+)-(\d+)$!;
 160             $name = $pool . '/' . $name;
 161         } else {
 162             next;
 163         }
 164
 165         $zvol->{pool} = $pool;
 166         $zvol->{name} = $name;
 167         $zvol->{size} = zfs_parse_size($2);
 168         if ($3 !~ /^-$/) {
 169             $zvol->{origin} = $3;
 170         }
 171         push @$list, $zvol;
 172     }
 173     }
 174
 175     return $list;
 176 }
 177
 178 sub zfs_get_lu_name {
 179     my ($scfg, $zvol) = @_;
 180     my $object;
 181
 182     my $base = $zfs_get_base->($scfg);
 183     if ($zvol =~ /^.+\/.+/) {
 184         $object = "$base/$zvol";
 185     } else {
 186         $object = "$base/$scfg->{pool}/$zvol";
 187     }
 188
 189     my $lu_name = zfs_request($scfg, undef, 'list_lu', $object);
 190
 191     return $lu_name if $lu_name;
 192
 193     die "Could not find lu_name for zvol $zvol";
 194 }
 195
 196 sub zfs_get_zvol_size {
 197     my ($scfg, $zvol) = @_;
 198
 199     my $text = zfs_request($scfg, undef, 'get', '-Hp', 'volsize', "$scfg->{pool}/$zvol");
 200
 201     if($text =~ /volsize\s(\d+)/){
 202     return $1;
 203     }
 204
 205     die "Could not get zvol size";
 206 }
 207
 208 sub zfs_add_lun_mapping_entry {
 209     my ($scfg, $zvol, $guid) = @_;
 210
 211     if (! defined($guid)) {
 212     $guid = zfs_get_lu_name($scfg, $zvol);
 213     }
 214
 215     zfs_request($scfg, undef, 'add_view', $guid);
 216 }
 217
 218 sub zfs_delete_lu {
 219     my ($scfg, $zvol) = @_;
 220
 221     my $guid = zfs_get_lu_name($scfg, $zvol);
 222
 223     zfs_request($scfg, undef, 'delete_lu', $guid);
 224 }
 225
 226 sub zfs_create_lu {
 227     my ($scfg, $zvol) = @_;
 228
 229     my $base = $zfs_get_base->($scfg);
 230     my $guid = zfs_request($scfg, undef, 'create_lu', "$base/$scfg->{pool}/$zvol");
 231
 232     return $guid;
 233 }
 234
 235 sub zfs_import_lu {
 236     my ($scfg, $zvol) = @_;
 237
 238     my $base = $zfs_get_base->($scfg);
 239     zfs_request($scfg, undef, 'import_lu', "$base/$scfg->{pool}/$zvol");
 240 }
 241
 242 sub zfs_resize_lu {
 243     my ($scfg, $zvol, $size) = @_;
 244
 245     my $guid = zfs_get_lu_name($scfg, $zvol);
 246
 247     zfs_request($scfg, undef, 'modify_lu', "${size}K", $guid);
 248 }
 249
 250 sub zfs_create_zvol {
 251     my ($scfg, $zvol, $size) = @_;
 252
 253     my $sparse = '';
 254     if ($scfg->{sparse}) {
 255         $sparse = '-s';
 256     }
 257
 258     zfs_request($scfg, undef, 'create', $sparse, '-b', $scfg->{blocksize}, '-V', "${size}k", "$scfg->{pool}/$zvol");
 259 }
 260
 261 sub zfs_delete_zvol {
 262     my ($scfg, $zvol) = @_;
 263
 264     zfs_request($scfg, undef, 'destroy', '-r', "$scfg->{pool}/$zvol");
 265 }
 266
 267 sub zfs_get_lun_number {
 268     my ($scfg, $guid) = @_;
 269
 270     die "could not find lun_number for guid $guid" if !$guid;
 271
 272     return zfs_request($scfg, undef, 'list_view', $guid);
 273 }
 274
 275 sub zfs_list_zvol {
 276     my ($scfg) = @_;
 277
 278     my $text = zfs_request($scfg, 10, 'list', '-o', 'name,volsize,origin', '-t', 'volume', '-Hr');
 279     my $zvols = zfs_parse_zvol_list($text);
 280     return undef if !$zvols;
 281
 282     my $list = ();
 283     foreach my $zvol (@$zvols) {
 284     my @values = split('/', $zvol->{name});
 285
 286     my $image = pop @values;
 287     my $pool = join('/', @values);
 288
 289     next if $image !~ m/^((vm|base)-(\d+)-\S+)$/;
 290     my $owner = $3;
 291
 292     my $parent = $zvol->{origin};
 293     if($zvol->{origin} && $zvol->{origin} =~ m/^$scfg->{pool}\/(\S+)$/){
 294         $parent = $1;
 295     }
 296
 297     $list->{$pool}->{$image} = {
 298         name => $image,
 299         size => $zvol->{size},
 300         parent => $parent,
 301         format => 'raw',
 302         vmid => $owner
 303     };
 304     }
 305
 306     return $list;
 307 }
 308
 309 # Configuration
 310
 311 sub type {
 312     return 'zfs';
 313 }
 314
 315 sub plugindata {
 316     return {
 317     content => [ {images => 1}, { images => 1 }],
 318     };
 319 }
 320
 321 sub properties {
 322     return {
 323     iscsiprovider => {
 324         description => "iscsi provider",
 325         type => 'string',
 326     },
 327     blocksize => {
 328         description => "block size",
 329         type => 'string',
 330     },
 331     # this will disable write caching on comstar and istgt.
 332     # it is not implemented for iet. iet blockio always operates with
 333     # writethrough caching when not in readonly mode
 334     nowritecache => {
 335         description => "disable write caching on the target",
 336         type => 'boolean',
 337     },
 338     sparse => {
 339         description => "use sparse volumes",
 340         type => 'boolean',
 341     },
 342     comstar_tg => {
 343         description => "target group for comstar views",
 344         type => 'string',
 345     },
 346     comstar_hg => {
 347         description => "host group for comstar views",
 348         type => 'string',
 349     },
 350     };
 351 }
 352
 353 sub options {
 354     return {
 355     nodes => { optional => 1 },
 356     disable => { optional => 1 },
 357     portal => { fixed => 1 },
 358     target => { fixed => 1 },
 359     pool => { fixed => 1 },
 360     blocksize => { fixed => 1 },
 361     iscsiprovider => { fixed => 1 },
 362     nowritecache => { optional => 1 },
 363     sparse => { optional => 1 },
 364     comstar_hg => { optional => 1 },
 365     comstar_tg => { optional => 1 },
 366     content => { optional => 1 },
 367     };
 368 }
 369
 370 # Storage implementation
 371
 372 sub parse_volname {
 373     my ($class, $volname) = @_;
 374
 375     if ($volname =~ m/^(((base|vm)-(\d+)-\S+)\/)?((base)?(vm)?-(\d+)-\S+)$/) {
 376     return ('images', $5, $8, $2, $4, $6);
 377     }
 378
 379     die "unable to parse zfs volume name '$volname'\n";
 380 }
 381
 382 sub path {
 383     my ($class, $scfg, $volname) = @_;
 384
 385     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
 386
 387     my $target = $scfg->{target};
 388     my $portal = $scfg->{portal};
 389
 390     my $guid = zfs_get_lu_name($scfg, $name);
 391     my $lun = zfs_get_lun_number($scfg, $guid);
 392
 393     my $path = "iscsi://$portal/$target/$lun";
 394
 395     return ($path, $vmid, $vtype);
 396 }
 397
 398 my $find_free_diskname = sub {
 399     my ($storeid, $scfg, $vmid) = @_;
 400
 401     my $name = undef;
 402     my $volumes = zfs_list_zvol($scfg);
 403
 404     my $disk_ids = {};
 405     my $dat = $volumes->{$scfg->{pool}};
 406
 407     foreach my $image (keys %$dat) {
 408         my $volname = $dat->{$image}->{name};
 409         if ($volname =~ m/(vm|base)-$vmid-disk-(\d+)/){
 410             $disk_ids->{$2} = 1;
 411         }
 412     }
 413
 414     for (my $i = 1; $i < 100; $i++) {
 415         if (!$disk_ids->{$i}) {
 416             return "vm-$vmid-disk-$i";
 417         }
 418     }
 419
 420     die "unable to allocate an image name for VM $vmid in storage '$storeid'\n";
 421 };
 422
 423 sub create_base {
 424     my ($class, $storeid, $scfg, $volname) = @_;
 425
 426     my $snap = '__base__';
 427
 428     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
 429         $class->parse_volname($volname);
 430
 431     die "create_base not possible with base image\n" if $isBase;
 432
 433     my $newname = $name;
 434     $newname =~ s/^vm-/base-/;
 435
 436     my $newvolname = $basename ? "$basename/$newname" : "$newname";
 437
 438     zfs_delete_lu($scfg, $name);
 439     zfs_request($scfg, undef, 'rename', "$scfg->{pool}/$name", "$scfg->{pool}/$newname");
 440
 441     my $guid = zfs_create_lu($scfg, $newname);
 442     zfs_add_lun_mapping_entry($scfg, $newname, $guid);
 443
 444     my $running  = undef; #fixme : is create_base always offline ?
 445
 446     $class->volume_snapshot($scfg, $storeid, $newname, $snap, $running);
 447
 448     return $newvolname;
 449 }
 450
 451 sub clone_image {
 452     my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
 453
 454     $snap ||= '__base__';
 455
 456     my ($vtype, $basename, $basevmid, undef, undef, $isBase) =
 457         $class->parse_volname($volname);
 458
 459     die "clone_image only works on base images\n" if !$isBase;
 460
 461     my $name = &$find_free_diskname($storeid, $scfg, $vmid);
 462
 463     warn "clone $volname: $basename to $name\n";
 464
 465     zfs_request($scfg, undef, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$name");
 466
 467     my $guid = zfs_create_lu($scfg, $name);
 468     zfs_add_lun_mapping_entry($scfg, $name, $guid);
 469
 470     return $name;
 471 }
 472
 473 sub alloc_image {
 474     my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
 475
 476     die "unsupported format '$fmt'" if $fmt ne 'raw';
 477
 478     die "illegal name '$name' - sould be 'vm-$vmid-*'\n"
 479     if $name && $name !~ m/^vm-$vmid-/;
 480
 481     $name = &$find_free_diskname($storeid, $scfg, $vmid);
 482
 483     zfs_create_zvol($scfg, $name, $size);
 484     my $guid = zfs_create_lu($scfg, $name);
 485     zfs_add_lun_mapping_entry($scfg, $name, $guid);
 486
 487     return $name;
 488 }
 489
 490 sub free_image {
 491     my ($class, $storeid, $scfg, $volname, $isBase) = @_;
 492
 493     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
 494
 495     zfs_delete_lu($scfg, $name);
 496     eval {
 497         zfs_delete_zvol($scfg, $name);
 498     };
 499     do {
 500         my $err = $@;
 501         my $guid = zfs_create_lu($scfg, $name);
 502         zfs_add_lun_mapping_entry($scfg, $name, $guid);
 503         die $err;
 504     } if $@;
 505
 506     return undef;
 507 }
 508
 509 sub list_images {
 510     my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
 511
 512     $cache->{zfs} = zfs_list_zvol($scfg) if !$cache->{zfs};
 513     my $zfspool = $scfg->{pool};
 514     my $res = [];
 515
 516     if (my $dat = $cache->{zfs}->{$zfspool}) {
 517
 518     foreach my $image (keys %$dat) {
 519
 520         my $volname = $dat->{$image}->{name};
 521         my $parent = $dat->{$image}->{parent};
 522
 523         my $volid = undef;
 524             if ($parent && $parent =~ m/^(\S+)@(\S+)$/) {
 525         my ($basename) = ($1);
 526         $volid = "$storeid:$basename/$volname";
 527         } else {
 528         $volid = "$storeid:$volname";
 529         }
 530
 531         my $owner = $dat->{$volname}->{vmid};
 532         if ($vollist) {
 533         my $found = grep { $_ eq $volid } @$vollist;
 534         next if !$found;
 535         } else {
 536         next if defined ($vmid) && ($owner ne $vmid);
 537         }
 538
 539         my $info = $dat->{$volname};
 540         $info->{volid} = $volid;
 541         push @$res, $info;
 542     }
 543     }
 544
 545     return $res;
 546 }
 547
 548 sub status {
 549     my ($class, $storeid, $scfg, $cache) = @_;
 550
 551     my $total = 0;
 552     my $free = 0;
 553     my $used = 0;
 554     my $active = 0;
 555
 556     eval {
 557     ($free, $used) = zfs_get_pool_stats($scfg);
 558     $active = 1;
 559     $total = $free + $used;
 560     };
 561     warn $@ if $@;
 562
 563     return ($total, $free, $used, $active);
 564 }
 565
 566 sub activate_storage {
 567     my ($class, $storeid, $scfg, $cache) = @_;
 568     return 1;
 569 }
 570
 571 sub deactivate_storage {
 572     my ($class, $storeid, $scfg, $cache) = @_;
 573     return 1;
 574 }
 575
 576 sub activate_volume {
 577     my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
 578     return 1;
 579 }
 580
 581 sub deactivate_volume {
 582     my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
 583     return 1;
 584 }
 585
 586 sub volume_size_info {
 587     my ($class, $scfg, $storeid, $volname, $timeout) = @_;
 588
 589     return zfs_get_zvol_size($scfg, $volname);
 590 }
 591
 592 sub volume_resize {
 593     my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
 594
 595     my $new_size = ($size/1024);
 596
 597     zfs_request($scfg, undef, 'set', 'volsize=' . $new_size . 'k', "$scfg->{pool}/$volname");
 598     zfs_resize_lu($scfg, $volname, $new_size);
 599 }
 600
 601 sub volume_snapshot {
 602     my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 603
 604     zfs_request($scfg, undef, 'snapshot', "$scfg->{pool}/$volname\@$snap");
 605 }
 606
 607 sub volume_snapshot_rollback {
 608     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 609
 610     # abort rollback if snapshot is not the latest
 611     my @params = ('-t', 'snapshot', '-o', 'name', '-s', 'creation');
 612     my $text = zfs_request($scfg, undef, 'list', @params);
 613     my @snapshots = split(/\n/, $text);
 614     my $recentsnap = undef;
 615     foreach (@snapshots) {
 616         if (/$scfg->{pool}\/$volname/) {
 617             s/^.*@//;
 618             $recentsnap = $_;
 619         }
 620     }
 621     if ($snap ne $recentsnap) {
 622         die "cannot rollback, more recent snapshots exist\n";
 623     }
 624
 625     zfs_delete_lu($scfg, $volname);
 626
 627     zfs_request($scfg, undef, 'rollback', "$scfg->{pool}/$volname\@$snap");
 628
 629     zfs_import_lu($scfg, $volname);
 630
 631     zfs_add_lun_mapping_entry($scfg, $volname);
 632 }
 633
 634 sub volume_snapshot_delete {
 635     my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 636
 637     zfs_request($scfg, undef, 'destroy', "$scfg->{pool}/$volname\@$snap");
 638 }
 639
 640 sub volume_has_feature {
 641     my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
 642
 643     my $features = {
 644     snapshot => { current => 1, snap => 1},
 645     clone => { base => 1},
 646     template => { current => 1},
 647     copy => { base => 1, current => 1},
 648     };
 649
 650     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
 651     $class->parse_volname($volname);
 652
 653     my $key = undef;
 654
 655     if ($snapname) {
 656     $key = 'snap';
 657     } else {
 658     $key = $isBase ? 'base' : 'current';
 659     }
 660
 661     return 1 if $features->{$feature}->{$key};
 662
 663     return undef;
 664 }
 665
 666 1;