PVE/Storage/LunCmd/LIO.pm

   1 package PVE::Storage::LunCmd::LIO;
   2
   3 # lightly based on code from Iet.pm
   4 #
   5 # additional changes:
   6 # -----------------------------------------------------------------
   7 # Copyright (c) 2018 BestSolution.at EDV Systemhaus GmbH
   8 # All Rights Reserved.
   9 #
  10 # This software is released under the terms of the
  11 #
  12 #            "GNU Affero General Public License"
  13 #
  14 # and may only be distributed and used under the terms of the
  15 # mentioned license. You should have received a copy of the license
  16 # along with this software product, if not you can download it from
  17 # https://www.gnu.org/licenses/agpl-3.0.en.html
  18 #
  19 # Author: udo.rader@bestsolution.at
  20 # -----------------------------------------------------------------
  21
  22 use strict;
  23 use warnings;
  24 use PVE::Tools qw(run_command);
  25 use JSON;
  26
  27 sub get_base;
  28
  29 # targetcli constants
  30 # config file location differs from distro to distro
  31 my @CONFIG_FILES = (
  32         '/etc/rtslib-fb-target/saveconfig.json',        # Debian 9.x et al
  33         '/etc/target/saveconfig.json' ,                 # ArchLinux, CentOS
  34 );
  35 my $BACKSTORE = '/backstores/block';
  36
  37 my $SETTINGS = undef;
  38 my $SETTINGS_TIMESTAMP = 0;
  39 my $SETTINGS_MAXAGE = 15; # in seconds
  40
  41 my @ssh_opts = ('-o', 'BatchMode=yes');
  42 my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
  43 my $id_rsa_path = '/etc/pve/priv/zfs';
  44 my $targetcli = '/usr/bin/targetcli';
  45
  46 my $execute_remote_command = sub {
  47     my ($scfg, $timeout, $remote_command, @params) = @_;
  48
  49     my $msg = '';
  50     my $err = undef;
  51     my $target;
  52     my $cmd;
  53     my $res = ();
  54
  55     $timeout = 10 if !$timeout;
  56
  57     my $output = sub { $msg .= "$_[0]\n" };
  58     my $errfunc = sub { $err .= "$_[0]\n" };
  59
  60     $target = 'root@' . $scfg->{portal};
  61     $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, '--', $remote_command, @params];
  62
  63     eval {
  64         run_command($cmd, outfunc => $output, errfunc => $errfunc, timeout => $timeout);
  65     };
  66     if ($@) {
  67         $res = {
  68             result => 0,
  69             msg => $err,
  70         }
  71     } else {
  72         $res = {
  73             result => 1,
  74             msg => $msg,
  75         }
  76     }
  77
  78     return $res;
  79 };
  80
  81 # fetch targetcli configuration from the portal
  82 my $read_config = sub {
  83     my ($scfg, $timeout) = @_;
  84
  85     my $msg = '';
  86     my $err = undef;
  87     my $luncmd = 'cat';
  88     my $target;
  89     my $retry = 1;
  90
  91     $timeout = 10 if !$timeout;
  92
  93     my $output = sub { $msg .= "$_[0]\n" };
  94     my $errfunc = sub { $err .= "$_[0]\n" };
  95
  96     $target = 'root@' . $scfg->{portal};
  97
  98     foreach my $oneFile (@CONFIG_FILES) {
  99         my $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, $luncmd, $oneFile];
 100         eval {
 101             run_command($cmd, outfunc => $output, errfunc => $errfunc, timeout => $timeout);
 102         };
 103         if ($@) {
 104             die $err if ($err !~ /No such file or directory/);
 105         }
 106         return $msg if $msg ne '';
 107     }
 108
 109     die "No configuration found. Install targetcli on $scfg->{portal}\n" if $msg eq '';
 110
 111     return $msg;
 112 };
 113
 114 my $get_config = sub {
 115     my ($scfg) = @_;
 116     my @conf = undef;
 117
 118     my $config = $read_config->($scfg, undef);
 119     die "Missing config file" unless $config;
 120
 121     return $config;
 122 };
 123
 124 # Return settings of a specific target
 125 my $get_target_settings = sub {
 126    my ($scfg) = @_;
 127
 128    my $id = "$scfg->{portal}.$scfg->{target}";
 129    return undef if !$SETTINGS;
 130    return $SETTINGS->{$id};
 131 };
 132
 133 # fetches and parses targetcli config from the portal
 134 my $parser = sub {
 135     my ($scfg) = @_;
 136     my $tpg = $scfg->{lio_tpg} || die "Target Portal Group not set, aborting!\n";
 137     my $tpg_tag;
 138
 139     if ($tpg =~ /^tpg(\d+)$/) {
 140         $tpg_tag = $1;
 141     } else {
 142         die "Target Portal Group has invalid value, must contain string 'tpg' and a suffix number, eg 'tpg17'\n";
 143     }
 144
 145     my $base = get_base;
 146
 147     my $config = $get_config->($scfg);
 148     my $jsonconfig = JSON->new->utf8->decode($config);
 149
 150     my $haveTarget = 0;
 151     foreach my $target (@{$jsonconfig->{targets}}) {
 152         # only interested in iSCSI targets
 153         next if !($target->{fabric} eq 'iscsi' && $target->{wwn} eq $scfg->{target});
 154         # find correct TPG
 155         foreach my $tpg (@{$target->{tpgs}}) {
 156             if ($tpg->{tag} == $tpg_tag) {
 157                 my $res = [];
 158                 foreach my $lun (@{$tpg->{luns}}) {
 159                     my ($idx, $storage_object);
 160                     if ($lun->{index} =~ /^(\d+)$/) {
 161                         $idx = $1;
 162                     }
 163                     if ($lun->{storage_object} =~ m|^($BACKSTORE/.*)$|) {
 164                         $storage_object = $1;
 165                     }
 166                     die "Invalid lun definition in config!\n"
 167                         if !(defined($idx) && defined($storage_object));
 168                     push @$res, { index => $idx, storage_object => $storage_object };
 169                 }
 170
 171                 my $id = "$scfg->{portal}.$scfg->{target}";
 172                 $SETTINGS->{$id}->{luns} = $res;
 173                 $haveTarget = 1;
 174                 last;
 175             }
 176         }
 177     }
 178
 179     # seriously unhappy if the target server lacks iSCSI target configuration ...
 180     if (!$haveTarget) {
 181         die "target portal group tpg$tpg_tag not found!\n";
 182     }
 183 };
 184
 185 # Get prefix for backstores
 186 my $get_backstore_prefix = sub {
 187     my ($scfg) = @_;
 188     my $pool = $scfg->{pool};
 189     $pool =~ s/\//-/g;
 190     return $pool . '-';
 191 };
 192
 193 # removes the given lu_name from the local list of luns
 194 my $free_lu_name = sub {
 195     my ($scfg, $lu_name) = @_;
 196
 197     my $new = [];
 198     my $target = $get_target_settings->($scfg);
 199     foreach my $lun (@{$target->{luns}}) {
 200         if ($lun->{storage_object} ne "$BACKSTORE/$lu_name") {
 201             push @$new, $lun;
 202         }
 203     }
 204
 205     $target->{luns} = $new;
 206 };
 207
 208 # locally registers a new lun
 209 my $register_lun = sub {
 210     my ($scfg, $idx, $volname) = @_;
 211
 212     my $conf = {
 213         index => $idx,
 214         storage_object => "$BACKSTORE/$volname",
 215         is_new => 1,
 216     };
 217     my $target = $get_target_settings->($scfg);
 218     push @{$target->{luns}}, $conf;
 219
 220     return $conf;
 221 };
 222
 223 # extracts the ZFS volume name from a device path
 224 my $extract_volname = sub {
 225     my ($scfg, $lunpath) = @_;
 226     my $volname = undef;
 227
 228     my $base = get_base;
 229     if ($lunpath =~ /^$base\/$scfg->{pool}\/([\w\-]+)$/) {
 230         $volname = $1;
 231         my $prefix = $get_backstore_prefix->($scfg);
 232         my $target = $get_target_settings->($scfg);
 233         foreach my $lun (@{$target->{luns}}) {
 234             # If we have a lun with the pool prefix matching this vol, then return this one
 235             # like pool-pve-vm-100-disk-0
 236             # Else, just fallback to the old name scheme which is vm-100-disk-0
 237             if ($lun->{storage_object} =~ /^$BACKSTORE\/($prefix$volname)$/) {
 238                 return $1;
 239             }
 240         }
 241     }
 242
 243     return $volname;
 244 };
 245
 246 # retrieves the LUN index for a particular object
 247 my $list_view = sub {
 248     my ($scfg, $timeout, $method, @params) = @_;
 249     my $lun = undef;
 250
 251     my $object = $params[0];
 252     my $volname = $extract_volname->($scfg, $object);
 253     my $target = $get_target_settings->($scfg);
 254
 255     return undef if !defined($volname); # nothing to search for..
 256
 257     foreach my $lun (@{$target->{luns}}) {
 258         if ($lun->{storage_object} eq "$BACKSTORE/$volname") {
 259             return $lun->{index};
 260         }
 261     }
 262
 263     return $lun;
 264 };
 265
 266 # determines, if the given object exists on the portal
 267 my $list_lun = sub {
 268     my ($scfg, $timeout, $method, @params) = @_;
 269     my $name = undef;
 270
 271     my $object = $params[0];
 272     my $volname = $extract_volname->($scfg, $object);
 273     my $target = $get_target_settings->($scfg);
 274
 275     foreach my $lun (@{$target->{luns}}) {
 276         if ($lun->{storage_object} eq "$BACKSTORE/$volname") {
 277             return $object;
 278         }
 279     }
 280
 281     return $name;
 282 };
 283
 284 # adds a new LUN to the target
 285 my $create_lun = sub {
 286     my ($scfg, $timeout, $method, @params) = @_;
 287
 288     if ($list_lun->($scfg, $timeout, $method, @params)) {
 289         die "$params[0]: LUN already exists!";
 290     }
 291
 292     my $device = $params[0];
 293     my $volname = $extract_volname->($scfg, $device);
 294     # Here we create a new device, so we didn't get the volname prefixed with the pool name
 295     # as extract_volname couldn't find a matching vol yet
 296     $volname = $get_backstore_prefix->($scfg) . $volname;
 297     my $tpg = $scfg->{lio_tpg} || die "Target Portal Group not set, aborting!\n";
 298
 299     # step 1: create backstore for device
 300     my @cliparams = ($BACKSTORE, 'create', "name=$volname", "dev=$device" );
 301     my $res = $execute_remote_command->($scfg, $timeout, $targetcli, @cliparams);
 302     die $res->{msg} if !$res->{result};
 303
 304     # step 2: enable unmap support on the backstore
 305     @cliparams = ($BACKSTORE . '/' . $volname, 'set', 'attribute', 'emulate_tpu=1' );
 306     $res = $execute_remote_command->($scfg, $timeout, $targetcli, @cliparams);
 307     die $res->{msg} if !$res->{result};
 308
 309     # step 3: register lun with target
 310     # targetcli /iscsi/iqn.2018-04.at.bestsolution.somehost:target/tpg1/luns/ create /backstores/block/foobar
 311     @cliparams = ("/iscsi/$scfg->{target}/$tpg/luns/", 'create', "$BACKSTORE/$volname" );
 312     $res = $execute_remote_command->($scfg, $timeout, $targetcli, @cliparams);
 313     die $res->{msg} if !$res->{result};
 314
 315     # targetcli responds with "Created LUN 99"
 316     # not calculating the index ourselves, because the index at the portal might have
 317     # changed without our knowledge, so relying on the number that targetcli returns
 318     my $lun_idx;
 319     if ($res->{msg} =~ /LUN (\d+)/) {
 320         $lun_idx = $1;
 321     } else {
 322         die "unable to determine new LUN index: $res->{msg}";
 323     }
 324
 325     $register_lun->($scfg, $lun_idx, $volname);
 326
 327     # step 3: unfortunately, targetcli doesn't always save changes, no matter
 328     #         if auto_save_on_exit is true or not. So saving to be safe ...
 329     $execute_remote_command->($scfg, $timeout, $targetcli, 'saveconfig');
 330
 331     return $res->{msg};
 332 };
 333
 334 my $delete_lun = sub {
 335     my ($scfg, $timeout, $method, @params) = @_;
 336     my $res = {msg => undef};
 337
 338     my $tpg = $scfg->{lio_tpg} || die "Target Portal Group not set, aborting!\n";
 339
 340     my $path = $params[0];
 341     my $volname = $extract_volname->($scfg, $path);
 342     my $target = $get_target_settings->($scfg);
 343
 344     foreach my $lun (@{$target->{luns}}) {
 345         next if $lun->{storage_object} ne "$BACKSTORE/$volname";
 346
 347         # step 1: delete the lun
 348         my @cliparams = ("/iscsi/$scfg->{target}/$tpg/luns/", 'delete', "lun$lun->{index}" );
 349         my $res = $execute_remote_command->($scfg, $timeout, $targetcli, @cliparams);
 350         do {
 351             die $res->{msg};
 352         } unless $res->{result};
 353
 354         # step 2: delete the backstore
 355         @cliparams = ($BACKSTORE, 'delete', $volname);
 356         $res = $execute_remote_command->($scfg, $timeout, $targetcli, @cliparams);
 357         do {
 358             die $res->{msg};
 359         } unless $res->{result};
 360
 361         # step 3: save to be safe ...
 362         $execute_remote_command->($scfg, $timeout, $targetcli, 'saveconfig');
 363
 364         # update interal cache
 365         $free_lu_name->($scfg, $volname);
 366
 367         last;
 368     }
 369
 370     return $res->{msg};
 371 };
 372
 373 my $import_lun = sub {
 374     my ($scfg, $timeout, $method, @params) = @_;
 375
 376     return $create_lun->($scfg, $timeout, $method, @params);
 377 };
 378
 379 # needed for example when the underlying ZFS volume has been resized
 380 my $modify_lun = sub {
 381     my ($scfg, $timeout, $method, @params) = @_;
 382     # Nothing to do on volume modification for LIO
 383     return undef;
 384 };
 385
 386 my $add_view = sub {
 387     my ($scfg, $timeout, $method, @params) = @_;
 388
 389     return '';
 390 };
 391
 392 my %lun_cmd_map = (
 393     create_lu   =>  $create_lun,
 394     delete_lu   =>  $delete_lun,
 395     import_lu   =>  $import_lun,
 396     modify_lu   =>  $modify_lun,
 397     add_view    =>  $add_view,
 398     list_view   =>  $list_view,
 399     list_lu     =>  $list_lun,
 400 );
 401
 402 sub run_lun_command {
 403     my ($scfg, $timeout, $method, @params) = @_;
 404
 405     # fetch configuration from target if we haven't yet or if it is stale
 406     my $timediff = time - $SETTINGS_TIMESTAMP;
 407     my $target = $get_target_settings->($scfg);
 408     if (!$target || $timediff > $SETTINGS_MAXAGE) {
 409         $SETTINGS_TIMESTAMP = time;
 410         $parser->($scfg);
 411     }
 412
 413     die "unknown command '$method'" unless exists $lun_cmd_map{$method};
 414     my $msg = $lun_cmd_map{$method}->($scfg, $timeout, $method, @params);
 415
 416     return $msg;
 417 }
 418
 419 sub get_base {
 420     return '/dev';
 421 }
 422
 423 1;