PVE/Storage/ZFSPlugin.pm

   1 package PVE::Storage::ZFSPlugin;
   2
   3 use strict;
   4 use warnings;
   5 use IO::File;
   6 use POSIX;
   7 use PVE::Tools qw(run_command);
   8 use PVE::Storage::Plugin;
   9
  10 use base qw(PVE::Storage::Plugin);
  11 use PVE::Storage::LunCmd::Comstar;
  12 use PVE::Storage::LunCmd::Istgt;
  13 use PVE::Storage::LunCmd::Iet;
  14
  15 my @ssh_opts = ('-o', 'BatchMode=yes');
  16 my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
  17 my $id_rsa_path = '/etc/pve/priv/zfs';
  18
  19 my $lun_cmds = {
  20     create_lu   => 1,
  21     delete_lu   => 1,
  22     import_lu   => 1,
  23     modify_lu   => 1,
  24     add_view    => 1,
  25     list_view   => 1,
  26     list_lu     => 1,
  27 };
  28
  29 my $zfs_unknown_scsi_provider = sub {
  30     my ($provider) = @_;
  31
  32     die "$provider: unknown iscsi provider. Available [comstar, istgt, iet]";
  33 };
  34
  35 my $zfs_get_base = sub {
  36     my ($scfg) = @_;
  37
  38     if ($scfg->{iscsiprovider} eq 'comstar') {
  39         return PVE::Storage::LunCmd::Comstar::get_base;
  40     } elsif ($scfg->{iscsiprovider} eq 'istgt') {
  41         return PVE::Storage::LunCmd::Istgt::get_base;
  42     } elsif ($scfg->{iscsiprovider} eq 'iet') {
  43         return PVE::Storage::LunCmd::Iet::get_base;
  44     } else {
  45         $zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
  46     }
  47 };
  48
  49 sub zfs_request {
  50     my ($scfg, $timeout, $method, @params) = @_;
  51
  52     my $cmdmap;
  53     my $zfscmd;
  54     my $target;
  55     my $msg;
  56
  57     $timeout = 5 if !$timeout;
  58
  59     if ($lun_cmds->{$method}) {
  60         if ($scfg->{iscsiprovider} eq 'comstar') {
  61             $msg = PVE::Storage::LunCmd::Comstar::run_lun_command($scfg, $timeout, $method, @params);
  62         } elsif ($scfg->{iscsiprovider} eq 'istgt') {
  63             $msg = PVE::Storage::LunCmd::Istgt::run_lun_command($scfg, $timeout, $method, @params);
  64         } elsif ($scfg->{iscsiprovider} eq 'iet') {
  65             $msg = PVE::Storage::LunCmd::Iet::run_lun_command($scfg, $timeout, $method, @params);
  66         } else {
  67             $zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
  68         }
  69     } else {
  70         if ($method eq 'zpool_list') {
  71             $zfscmd = 'zpool';
  72             $method = 'list',
  73         } else {
  74             $zfscmd = 'zfs';
  75         }
  76
  77         $target = 'root@' . $scfg->{portal};
  78
  79         my $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, $zfscmd, $method, @params];
  80
  81         $msg = '';
  82
  83         my $output = sub {
  84         my $line = shift;
  85         $msg .= "$line\n";
  86         };
  87
  88         run_command($cmd, outfunc => $output, timeout => $timeout);
  89     }
  90
  91     return $msg;
  92 }
  93
  94 sub zfs_parse_size {
  95     my ($text) = @_;
  96
  97     return 0 if !$text;
  98
  99     if ($text =~ m/^(\d+(\.\d+)?)([TGMK])?$/) {
 100     my ($size, $reminder, $unit) = ($1, $2, $3);
 101     return $size if !$unit;
 102     if ($unit eq 'K') {
 103         $size *= 1024;
 104     } elsif ($unit eq 'M') {
 105         $size *= 1024*1024;
 106     } elsif ($unit eq 'G') {
 107         $size *= 1024*1024*1024;
 108     } elsif ($unit eq 'T') {
 109         $size *= 1024*1024*1024*1024;
 110     }
 111
 112     if ($reminder) {
 113         $size = ceil($size);
 114     }
 115     return $size;
 116     } else {
 117     return 0;
 118     }
 119 }
 120
 121 sub zfs_get_pool_stats {
 122     my ($scfg) = @_;
 123
 124     my $available = 0;
 125     my $used = 0;
 126
 127     my $text = zfs_request($scfg, undef, 'get', '-o', 'value', '-Hp',
 128                'available,used', $scfg->{pool});
 129
 130     my @lines = split /\n/, $text;
 131
 132     if($lines[0] =~ /^(\d+)$/) {
 133     $available = $1;
 134     }
 135
 136     if($lines[1] =~ /^(\d+)$/) {
 137     $used = $1;
 138     }
 139
 140     return ($available, $used);
 141 }
 142
 143 sub zfs_parse_zvol_list {
 144     my ($text) = @_;
 145
 146     my $list = ();
 147
 148     return $list if !$text;
 149
 150     my @lines = split /\n/, $text;
 151     foreach my $line (@lines) {
 152     if ($line =~ /^(.+)\s+([a-zA-Z0-9\.]+|\-)\s+(.+)$/) {
 153         my $zvol = {};
 154         my @parts = split /\//, $1;
 155         my $name = pop @parts;
 156         my $pool = join('/', @parts);
 157
 158         if ($pool !~ /^rpool$/) {
 159             next unless $name =~ m!^(\w+)-(\d+)-(\w+)-(\d+)$!;
 160             $name = $pool . '/' . $name;
 161         } else {
 162             next;
 163         }
 164
 165         $zvol->{pool} = $pool;
 166         $zvol->{name} = $name;
 167         $zvol->{size} = zfs_parse_size($2);
 168         if ($3 !~ /^-$/) {
 169             $zvol->{origin} = $3;
 170         }
 171         push @$list, $zvol;
 172     }
 173     }
 174
 175     return $list;
 176 }
 177
 178 sub zfs_get_lu_name {
 179     my ($scfg, $zvol) = @_;
 180     my $object;
 181
 182     my $base = $zfs_get_base->($scfg);
 183     if ($zvol =~ /^.+\/.+/) {
 184         $object = "$base/$zvol";
 185     } else {
 186         $object = "$base/$scfg->{pool}/$zvol";
 187     }
 188
 189     my $lu_name = zfs_request($scfg, undef, 'list_lu', $object);
 190
 191     return $lu_name if $lu_name;
 192
 193     die "Could not find lu_name for zvol $zvol";
 194 }
 195
 196 sub zfs_get_zvol_size {
 197     my ($scfg, $zvol) = @_;
 198
 199     my $text = zfs_request($scfg, undef, 'get', '-Hp', 'volsize', "$scfg->{pool}/$zvol");
 200
 201     if($text =~ /volsize\s(\d+)/){
 202     return $1;
 203     }
 204
 205     die "Could not get zvol size";
 206 }
 207
 208 sub zfs_add_lun_mapping_entry {
 209     my ($scfg, $zvol, $guid) = @_;
 210
 211     if (! defined($guid)) {
 212     $guid = zfs_get_lu_name($scfg, $zvol);
 213     }
 214
 215     zfs_request($scfg, undef, 'add_view', $guid);
 216 }
 217
 218 sub zfs_delete_lu {
 219     my ($scfg, $zvol) = @_;
 220
 221     my $guid = zfs_get_lu_name($scfg, $zvol);
 222
 223     zfs_request($scfg, undef, 'delete_lu', $guid);
 224 }
 225
 226 sub zfs_create_lu {
 227     my ($scfg, $zvol) = @_;
 228
 229     my $base = $zfs_get_base->($scfg);
 230     my $guid = zfs_request($scfg, undef, 'create_lu', "$base/$scfg->{pool}/$zvol");
 231
 232     return $guid;
 233 }
 234
 235 sub zfs_import_lu {
 236     my ($scfg, $zvol) = @_;
 237
 238     my $base = $zfs_get_base->($scfg);
 239     zfs_request($scfg, undef, 'import_lu', "$base/$scfg->{pool}/$zvol");
 240 }
 241
 242 sub zfs_resize_lu {
 243     my ($scfg, $zvol, $size) = @_;
 244
 245     my $guid = zfs_get_lu_name($scfg, $zvol);
 246
 247     zfs_request($scfg, undef, 'modify_lu', "${size}K", $guid);
 248 }
 249
 250 sub zfs_create_zvol {
 251     my ($scfg, $zvol, $size) = @_;
 252
 253     my $sparse = '';
 254     if ($scfg->{sparse}) {
 255         $sparse = '-s';
 256     }
 257
 258     zfs_request($scfg, undef, 'create', $sparse, '-b', $scfg->{blocksize}, '-V', "${size}k", "$scfg->{pool}/$zvol");
 259 }
 260
 261 sub zfs_delete_zvol {
 262     my ($scfg, $zvol) = @_;
 263
 264     zfs_request($scfg, undef, 'destroy', '-r', "$scfg->{pool}/$zvol");
 265 }
 266
 267 sub zfs_get_lun_number {
 268     my ($scfg, $guid) = @_;
 269
 270     die "could not find lun_number for guid $guid" if !$guid;
 271
 272     return zfs_request($scfg, undef, 'list_view', $guid);
 273 }
 274
 275 sub zfs_list_zvol {
 276     my ($scfg) = @_;
 277
 278     my $text = zfs_request($scfg, 10, 'list', '-o', 'name,volsize,origin', '-t', 'volume', '-Hr');
 279     my $zvols = zfs_parse_zvol_list($text);
 280     return undef if !$zvols;
 281
 282     my $list = ();
 283     foreach my $zvol (@$zvols) {
 284     my @values = split('/', $zvol->{name});
 285
 286     my $image = pop @values;
 287     my $pool = join('/', @values);
 288
 289     next if $image !~ m/^((vm|base)-(\d+)-\S+)$/;
 290     my $owner = $3;
 291
 292     my $parent = $zvol->{origin};
 293     if($zvol->{origin} && $zvol->{origin} =~ m/^$scfg->{pool}\/(\S+)$/){
 294         $parent = $1;
 295     }
 296
 297     $list->{$pool}->{$image} = {
 298         name => $image,
 299         size => $zvol->{size},
 300         parent => $parent,
 301         format => 'raw',
 302         vmid => $owner
 303     };
 304     }
 305
 306     return $list;
 307 }
 308
 309 # Configuration
 310
 311 sub type {
 312     return 'zfs';
 313 }
 314
 315 sub plugindata {
 316     return {
 317     content => [ {images => 1}, { images => 1 }],
 318     };
 319 }
 320
 321 sub properties {
 322     return {
 323     iscsiprovider => {
 324         description => "iscsi provider",
 325         type => 'string',
 326     },
 327     blocksize => {
 328         description => "block size",
 329         type => 'string',
 330     },
 331     # this will disable write caching on comstar and istgt.
 332     # it is not implemented for iet. iet blockio always operates with
 333     # writethrough caching when not in readonly mode
 334     nowritecache => {
 335         description => "disable write caching on the target",
 336         type => 'boolean',
 337     },
 338     sparse => {
 339         description => "use sparse volumes",
 340         type => 'boolean',
 341     }
 342     };
 343 }
 344
 345 sub options {
 346     return {
 347     nodes => { optional => 1 },
 348     disable => { optional => 1 },
 349     portal => { fixed => 1 },
 350     target => { fixed => 1 },
 351     pool => { fixed => 1 },
 352     blocksize => { fixed => 1 },
 353     iscsiprovider => { fixed => 1 },
 354     nowritecache => { optional => 1 },
 355     sparse => { optional => 1 },
 356     content => { optional => 1 },
 357     };
 358 }
 359
 360 # Storage implementation
 361
 362 sub parse_volname {
 363     my ($class, $volname) = @_;
 364
 365     if ($volname =~ m/^(((base|vm)-(\d+)-\S+)\/)?((base)?(vm)?-(\d+)-\S+)$/) {
 366     return ('images', $5, $8, $2, $4, $6);
 367     }
 368
 369     die "unable to parse zfs volume name '$volname'\n";
 370 }
 371
 372 sub path {
 373     my ($class, $scfg, $volname) = @_;
 374
 375     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
 376
 377     my $target = $scfg->{target};
 378     my $portal = $scfg->{portal};
 379
 380     my $guid = zfs_get_lu_name($scfg, $name);
 381     my $lun = zfs_get_lun_number($scfg, $guid);
 382
 383     my $path = "iscsi://$portal/$target/$lun";
 384
 385     return ($path, $vmid, $vtype);
 386 }
 387
 388 my $find_free_diskname = sub {
 389     my ($storeid, $scfg, $vmid) = @_;
 390
 391     my $name = undef;
 392     my $volumes = zfs_list_zvol($scfg);
 393
 394     my $disk_ids = {};
 395     my $dat = $volumes->{$scfg->{pool}};
 396
 397     foreach my $image (keys %$dat) {
 398         my $volname = $dat->{$image}->{name};
 399         if ($volname =~ m/(vm|base)-$vmid-disk-(\d+)/){
 400             $disk_ids->{$2} = 1;
 401         }
 402     }
 403
 404     for (my $i = 1; $i < 100; $i++) {
 405         if (!$disk_ids->{$i}) {
 406             return "vm-$vmid-disk-$i";
 407         }
 408     }
 409
 410     die "unable to allocate an image name for VM $vmid in storage '$storeid'\n";
 411 };
 412
 413 sub create_base {
 414     my ($class, $storeid, $scfg, $volname) = @_;
 415
 416     my $snap = '__base__';
 417
 418     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
 419         $class->parse_volname($volname);
 420
 421     die "create_base not possible with base image\n" if $isBase;
 422
 423     my $newname = $name;
 424     $newname =~ s/^vm-/base-/;
 425
 426     my $newvolname = $basename ? "$basename/$newname" : "$newname";
 427
 428     zfs_delete_lu($scfg, $name);
 429     zfs_request($scfg, undef, 'rename', "$scfg->{pool}/$name", "$scfg->{pool}/$newname");
 430
 431     my $guid = zfs_create_lu($scfg, $newname);
 432     zfs_add_lun_mapping_entry($scfg, $newname, $guid);
 433
 434     my $running  = undef; #fixme : is create_base always offline ?
 435
 436     $class->volume_snapshot($scfg, $storeid, $newname, $snap, $running);
 437
 438     return $newvolname;
 439 }
 440
 441 sub clone_image {
 442     my ($class, $scfg, $storeid, $volname, $vmid) = @_;
 443
 444     my $snap = '__base__';
 445
 446     my ($vtype, $basename, $basevmid, undef, undef, $isBase) =
 447         $class->parse_volname($volname);
 448
 449     die "clone_image only works on base images\n" if !$isBase;
 450
 451     my $name = &$find_free_diskname($storeid, $scfg, $vmid);
 452
 453     warn "clone $volname: $basename to $name\n";
 454
 455     zfs_request($scfg, undef, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$name");
 456
 457     my $guid = zfs_create_lu($scfg, $name);
 458     zfs_add_lun_mapping_entry($scfg, $name, $guid);
 459
 460     return $name;
 461 }
 462
 463 sub alloc_image {
 464     my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
 465
 466     die "unsupported format '$fmt'" if $fmt ne 'raw';
 467
 468     die "illegal name '$name' - sould be 'vm-$vmid-*'\n"
 469     if $name && $name !~ m/^vm-$vmid-/;
 470
 471     $name = &$find_free_diskname($storeid, $scfg, $vmid);
 472
 473     zfs_create_zvol($scfg, $name, $size);
 474     my $guid = zfs_create_lu($scfg, $name);
 475     zfs_add_lun_mapping_entry($scfg, $name, $guid);
 476
 477     return $name;
 478 }
 479
 480 sub free_image {
 481     my ($class, $storeid, $scfg, $volname, $isBase) = @_;
 482
 483     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
 484
 485     zfs_delete_lu($scfg, $name);
 486     eval {
 487         zfs_delete_zvol($scfg, $name);
 488     };
 489     do {
 490         my $err = $@;
 491         my $guid = zfs_create_lu($scfg, $name);
 492         zfs_add_lun_mapping_entry($scfg, $name, $guid);
 493         die $err;
 494     } if $@;
 495
 496     return undef;
 497 }
 498
 499 sub list_images {
 500     my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
 501
 502     $cache->{zfs} = zfs_list_zvol($scfg) if !$cache->{zfs};
 503     my $zfspool = $scfg->{pool};
 504     my $res = [];
 505
 506     if (my $dat = $cache->{zfs}->{$zfspool}) {
 507
 508     foreach my $image (keys %$dat) {
 509
 510         my $volname = $dat->{$image}->{name};
 511         my $parent = $dat->{$image}->{parent};
 512
 513         my $volid = undef;
 514             if ($parent && $parent =~ m/^(\S+)@(\S+)$/) {
 515         my ($basename) = ($1);
 516         $volid = "$storeid:$basename/$volname";
 517         } else {
 518         $volid = "$storeid:$volname";
 519         }
 520
 521         my $owner = $dat->{$volname}->{vmid};
 522         if ($vollist) {
 523         my $found = grep { $_ eq $volid } @$vollist;
 524         next if !$found;
 525         } else {
 526         next if defined ($vmid) && ($owner ne $vmid);
 527         }
 528
 529         my $info = $dat->{$volname};
 530         $info->{volid} = $volid;
 531         push @$res, $info;
 532     }
 533     }
 534
 535     return $res;
 536 }
 537
 538 sub status {
 539     my ($class, $storeid, $scfg, $cache) = @_;
 540
 541     my $total = 0;
 542     my $free = 0;
 543     my $used = 0;
 544     my $active = 0;
 545
 546     eval {
 547     ($free, $used) = zfs_get_pool_stats($scfg);
 548     $active = 1;
 549     $total = $free + $used;
 550     };
 551     warn $@ if $@;
 552
 553     return ($total, $free, $used, $active);
 554 }
 555
 556 sub activate_storage {
 557     my ($class, $storeid, $scfg, $cache) = @_;
 558     return 1;
 559 }
 560
 561 sub deactivate_storage {
 562     my ($class, $storeid, $scfg, $cache) = @_;
 563     return 1;
 564 }
 565
 566 sub activate_volume {
 567     my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
 568     return 1;
 569 }
 570
 571 sub deactivate_volume {
 572     my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
 573     return 1;
 574 }
 575
 576 sub volume_size_info {
 577     my ($class, $scfg, $storeid, $volname, $timeout) = @_;
 578
 579     return zfs_get_zvol_size($scfg, $volname);
 580 }
 581
 582 sub volume_resize {
 583     my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
 584
 585     my $new_size = ($size/1024);
 586
 587     zfs_request($scfg, undef, 'set', 'volsize=' . $new_size . 'k', "$scfg->{pool}/$volname");
 588     zfs_resize_lu($scfg, $volname, $new_size);
 589 }
 590
 591 sub volume_snapshot {
 592     my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 593
 594     zfs_request($scfg, undef, 'snapshot', "$scfg->{pool}/$volname\@$snap");
 595 }
 596
 597 sub volume_snapshot_rollback {
 598     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 599
 600     zfs_delete_lu($scfg, $volname);
 601
 602     zfs_request($scfg, undef, 'rollback', "$scfg->{pool}/$volname\@$snap");
 603
 604     zfs_import_lu($scfg, $volname);
 605
 606     zfs_add_lun_mapping_entry($scfg, $volname);
 607 }
 608
 609 sub volume_snapshot_delete {
 610     my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 611
 612     zfs_request($scfg, undef, 'destroy', "$scfg->{pool}/$volname\@$snap");
 613 }
 614
 615 sub volume_has_feature {
 616     my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
 617
 618     my $features = {
 619     snapshot => { current => 1, snap => 1},
 620     clone => { base => 1},
 621     template => { current => 1},
 622     copy => { base => 1, current => 1},
 623     };
 624
 625     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
 626     $class->parse_volname($volname);
 627
 628     my $key = undef;
 629
 630     if ($snapname) {
 631     $key = 'snap';
 632     } else {
 633     $key = $isBase ? 'base' : 'current';
 634     }
 635
 636     return 1 if $features->{$feature}->{$key};
 637
 638     return undef;
 639 }
 640
 641 1;