From 46c6107eb1a9e2a096a4ba89600788d9bea3c2f4 Mon Sep 17 00:00:00 2001 From: Udo Rader Date: Thu, 26 Jul 2018 23:26:27 +0200 Subject: [PATCH] Linux LIO/targetcli support Introducing LIO/targetcli support allowing to use recent linux distributions as iSCSI targets for ZFS volumes. In order for this to work, two preconditions have to be met: 1. the portal has to be set up correctly using targetcli 2. the initiator has to be authorized to connect to the target based on the initiator's InitiatorName When adding a LIO iSCSI target, a new "LIO target portal group" field needs to be correctly populated in the "Add: ZFS over iSCSI" popup, containing the fitting "LIO target portal group" name (typically something like 'tpg1'). Signed-Off-By: Udo Rader Tested-by: Stoiko Ivanov --- PVE/Storage/LunCmd/LIO.pm | 387 ++++++++++++++++++++++++++++++++++++ PVE/Storage/LunCmd/Makefile | 2 +- PVE/Storage/ZFSPlugin.pm | 12 +- 3 files changed, 399 insertions(+), 2 deletions(-) create mode 100644 PVE/Storage/LunCmd/LIO.pm diff --git a/PVE/Storage/LunCmd/LIO.pm b/PVE/Storage/LunCmd/LIO.pm new file mode 100644 index 0000000..a2050d4 --- /dev/null +++ b/PVE/Storage/LunCmd/LIO.pm @@ -0,0 +1,387 @@ +package PVE::Storage::LunCmd::LIO; + +# lightly based on code from Iet.pm +# +# additional changes: +# ----------------------------------------------------------------- +# Copyright (c) 2018 BestSolution.at EDV Systemhaus GmbH +# All Rights Reserved. +# +# This software is released under the terms of the +# +# "GNU Affero General Public License" +# +# and may only be distributed and used under the terms of the +# mentioned license. You should have received a copy of the license +# along with this software product, if not you can download it from +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Author: udo.rader@bestsolution.at +# ----------------------------------------------------------------- + +use strict; +use warnings; +use PVE::Tools qw(run_command); +use JSON; + +sub get_base; + +# targetcli constants +# config file location differs from distro to distro +my @CONFIG_FILES = ( + '/etc/rtslib-fb-target/saveconfig.json', # Debian 9.x et al + '/etc/target/saveconfig.json' , # ArchLinux, CentOS +); +my $BACKSTORE = '/backstores/block'; + +my $SETTINGS = undef; +my $SETTINGS_TIMESTAMP = 0; +my $SETTINGS_MAXAGE = 15; # in seconds + +my @ssh_opts = ('-o', 'BatchMode=yes'); +my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts); +my $id_rsa_path = '/etc/pve/priv/zfs'; +my $targetcli = '/usr/bin/targetcli'; + +my $execute_remote_command = sub { + my ($scfg, $timeout, $remote_command, @params) = @_; + + my $msg = ''; + my $err = undef; + my $target; + my $cmd; + my $res = (); + + $timeout = 10 if !$timeout; + + my $output = sub { + my $line = shift; + $msg .= "$line\n"; + }; + + my $errfunc = sub { + my $line = shift; + $err .= "$line"; + }; + + $target = 'root@' . $scfg->{portal}; + $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, '--', $remote_command, @params]; + + eval { + run_command($cmd, outfunc => $output, errfunc => $errfunc, timeout => $timeout); + }; + if ($@) { + $res = { + result => 0, + msg => $err, + } + } else { + $res = { + result => 1, + msg => $msg, + } + } + + return $res; +}; + +# fetch targetcli configuration from the portal +my $read_config = sub { + my ($scfg, $timeout) = @_; + + my $msg = ''; + my $err = undef; + my $luncmd = 'cat'; + my $target; + my $retry = 1; + + $timeout = 10 if !$timeout; + + my $output = sub { + my $line = shift; + $msg .= "$line\n"; + }; + + my $errfunc = sub { + my $line = shift; + $err .= "$line"; + }; + + $target = 'root@' . $scfg->{portal}; + + foreach my $oneFile (@CONFIG_FILES) { + my $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, $luncmd, $oneFile]; + eval { + run_command($cmd, outfunc => $output, errfunc => $errfunc, timeout => $timeout); + }; + if ($@) { + die $err if ($err !~ /No such file or directory/); + } + return $msg if $msg ne ''; + } + + die "No configuration found. Install targetcli on $scfg->{portal}\n" if $msg eq ''; + + return $msg; +}; + +my $get_config = sub { + my ($scfg) = @_; + my @conf = undef; + + my $config = $read_config->($scfg, undef); + die "Missing config file" unless $config; + + return $config; +}; + +# fetches and parses targetcli config from the portal +my $parser = sub { + my ($scfg) = @_; + my $tpg = $scfg->{lio_tpg} || die "Target Portal Group not set, aborting!\n"; + my $tpg_tag; + + if ($tpg =~ /^tpg(\d+)$/) { + $tpg_tag = $1; + } else { + die "Target Portal Group has invalid value, must contain string 'tpg' and a suffix number, eg 'tpg17'\n"; + } + + my $base = get_base; + + my $config = $get_config->($scfg); + my $jsonconfig = JSON->new->utf8->decode($config); + + my $haveTarget = 0; + foreach my $oneTarget (@{$jsonconfig->{targets}}) { + # only interested in iSCSI targets + if ($oneTarget->{fabric} eq 'iscsi' && $oneTarget->{wwn} eq $scfg->{target}) { + # find correct TPG + foreach my $oneTpg (@{$oneTarget->{tpgs}}) { + if ($oneTpg->{tag} == $tpg_tag) { + $SETTINGS->{target} = $oneTpg; + $haveTarget = 1; + last; + } + } + } + } + + # seriously unhappy if the target server lacks iSCSI target configuration ... + if (!$haveTarget) { + die "target portal group tpg$tpg_tag not found!\n"; + } +}; + +# removes the given lu_name from the local list of luns +my $free_lu_name = sub { + my ($lu_name) = @_; + my $new; + + foreach my $lun (@{$SETTINGS->{target}->{luns}}) { + if ($lun->{storage_object} ne "$BACKSTORE/$lu_name") { + push @$new, $lun; + } + } + + $SETTINGS->{target}->{luns} = $new; +}; + +# locally registers a new lun +my $register_lun = sub { + my ($scfg, $idx, $volname) = @_; + + my $conf = { + index => $idx, + storage_object => "$BACKSTORE/$volname", + is_new => 1, + }; + push @{$SETTINGS->{target}->{luns}}, $conf; + + return $conf; +}; + +# extracts the ZFS volume name from a device path +my $extract_volname = sub { + my ($scfg, $lunpath) = @_; + my $volname = undef; + + my $base = get_base; + if ($lunpath =~ /^$base\/$scfg->{pool}\/([\w\-]+)$/) { + $volname = $1; + } + + return $volname; +}; + +# retrieves the LUN index for a particular object +my $list_view = sub { + my ($scfg, $timeout, $method, @params) = @_; + my $lun = undef; + + my $object = $params[0]; + my $volname = $extract_volname->($scfg, $params[0]); + + foreach my $lun (@{$SETTINGS->{target}->{luns}}) { + if ($lun->{storage_object} eq "$BACKSTORE/$volname") { + return $lun->{index}; + } + } + + return $lun; +}; + +# determines, if the given object exists on the portal +my $list_lun = sub { + my ($scfg, $timeout, $method, @params) = @_; + my $name = undef; + + my $object = $params[0]; + my $volname = $extract_volname->($scfg, $params[0]); + + foreach my $lun (@{$SETTINGS->{target}->{luns}}) { + if ($lun->{storage_object} eq "$BACKSTORE/$volname") { + return $object; + } + } + + return $name; +}; + +# adds a new LUN to the target +my $create_lun = sub { + my ($scfg, $timeout, $method, @params) = @_; + + if ($list_lun->($scfg, $timeout, $method, @params)) { + die "$params[0]: LUN already exists!"; + } + + my $device = $params[0]; + my $volname = $extract_volname->($scfg, $device); + my $tpg = $scfg->{lio_tpg} || die "Target Portal Group not set, aborting!\n"; + + # step 1: create backstore for device + my @cliparams = ($BACKSTORE, 'create', "name=$volname", "dev=$device" ); + my $res = $execute_remote_command->($scfg, $timeout, $targetcli, @cliparams); + die $res->{msg} if !$res->{result}; + + # step 2: register lun with target + # targetcli /iscsi/iqn.2018-04.at.bestsolution.somehost:target/tpg1/luns/ create /backstores/block/foobar + @cliparams = ("/iscsi/$scfg->{target}/$tpg/luns/", 'create', "$BACKSTORE/$volname" ); + $res = $execute_remote_command->($scfg, $timeout, $targetcli, @cliparams); + die $res->{msg} if !$res->{result}; + + # targetcli responds with "Created LUN 99" + # not calculating the index ourselves, because the index at the portal might have + # changed without our knowledge, so relying on the number that targetcli returns + my $lun_idx; + if ($res->{msg} =~ /LUN (\d+)/) { + $lun_idx = $1; + } else { + die "unable to determine new LUN index: $res->{msg}"; + } + + $register_lun->($scfg, $lun_idx, $volname); + + # step 3: unfortunately, targetcli doesn't always save changes, no matter + # if auto_save_on_exit is true or not. So saving to be safe ... + $execute_remote_command->($scfg, $timeout, $targetcli, 'saveconfig'); + + return $res->{msg}; +}; + +my $delete_lun = sub { + my ($scfg, $timeout, $method, @params) = @_; + my $res = {msg => undef}; + + my $tpg = $scfg->{lio_tpg} || die "Target Portal Group not set, aborting!\n"; + + my $path = $params[0]; + my $volname = $extract_volname->($scfg, $params[0]); + + foreach my $lun (@{$SETTINGS->{target}->{luns}}) { + if ($lun->{storage_object} eq "$BACKSTORE/$volname") { + # step 1: delete the lun + my @cliparams = ("/iscsi/$scfg->{target}/$tpg/luns/", 'delete', "lun$lun->{index}" ); + my $res = $execute_remote_command->($scfg, $timeout, $targetcli, @cliparams); + do { + die $res->{msg}; + } unless $res->{result}; + + # step 2: delete the backstore + @cliparams = ($BACKSTORE, 'delete', $volname); + $res = $execute_remote_command->($scfg, $timeout, $targetcli, @cliparams); + do { + die $res->{msg}; + } unless $res->{result}; + + # step 3: save to be safe ... + $execute_remote_command->($scfg, $timeout, $targetcli, 'saveconfig'); + + # update interal cache + $free_lu_name->($volname); + + last; + } + } + + return $res->{msg}; +}; + +my $import_lun = sub { + my ($scfg, $timeout, $method, @params) = @_; + + return $create_lun->($scfg, $timeout, $method, @params); +}; + +# needed for example when the underlying ZFS volume has been resized +my $modify_lun = sub { + my ($scfg, $timeout, $method, @params) = @_; + my $msg; + + $msg = $delete_lun->($scfg, $timeout, $method, @params); + if ($msg) { + $msg = $create_lun->($scfg, $timeout, $method, @params); + } + + return $msg; +}; + +my $add_view = sub { + my ($scfg, $timeout, $method, @params) = @_; + + return ''; +}; + +my %lun_cmd_map = ( + create_lu => $create_lun, + delete_lu => $delete_lun, + import_lu => $import_lun, + modify_lu => $modify_lun, + add_view => $add_view, + list_view => $list_view, + list_lu => $list_lun, +); + +sub run_lun_command { + my ($scfg, $timeout, $method, @params) = @_; + + # fetch configuration from target if we haven't yet + # or if our configuration is stale + my $timediff = time - $SETTINGS_TIMESTAMP; + if ( ! $SETTINGS || $timediff > $SETTINGS_MAXAGE ) { + $SETTINGS_TIMESTAMP = time; + $parser->($scfg); + } + + die "unknown command '$method'" unless exists $lun_cmd_map{$method}; + my $msg = $lun_cmd_map{$method}->($scfg, $timeout, $method, @params); + + return $msg; +} + +sub get_base { + return '/dev'; +} + +1; diff --git a/PVE/Storage/LunCmd/Makefile b/PVE/Storage/LunCmd/Makefile index b959255..a7209d1 100644 --- a/PVE/Storage/LunCmd/Makefile +++ b/PVE/Storage/LunCmd/Makefile @@ -1,4 +1,4 @@ -SOURCES=Comstar.pm Istgt.pm Iet.pm +SOURCES=Comstar.pm Istgt.pm Iet.pm LIO.pm .PHONY: install install: diff --git a/PVE/Storage/ZFSPlugin.pm b/PVE/Storage/ZFSPlugin.pm index f88fe94..1fac811 100644 --- a/PVE/Storage/ZFSPlugin.pm +++ b/PVE/Storage/ZFSPlugin.pm @@ -12,6 +12,7 @@ use base qw(PVE::Storage::ZFSPoolPlugin); use PVE::Storage::LunCmd::Comstar; use PVE::Storage::LunCmd::Istgt; use PVE::Storage::LunCmd::Iet; +use PVE::Storage::LunCmd::LIO; my @ssh_opts = ('-o', 'BatchMode=yes'); @@ -31,7 +32,7 @@ my $lun_cmds = { my $zfs_unknown_scsi_provider = sub { my ($provider) = @_; - die "$provider: unknown iscsi provider. Available [comstar, istgt, iet]"; + die "$provider: unknown iscsi provider. Available [comstar, istgt, iet, LIO]"; }; my $zfs_get_base = sub { @@ -43,6 +44,8 @@ my $zfs_get_base = sub { return PVE::Storage::LunCmd::Istgt::get_base; } elsif ($scfg->{iscsiprovider} eq 'iet') { return PVE::Storage::LunCmd::Iet::get_base; + } elsif ($scfg->{iscsiprovider} eq 'LIO') { + return PVE::Storage::LunCmd::LIO::get_base; } else { $zfs_unknown_scsi_provider->($scfg->{iscsiprovider}); } @@ -63,6 +66,8 @@ sub zfs_request { $msg = PVE::Storage::LunCmd::Istgt::run_lun_command($scfg, $timeout, $method, @params); } elsif ($scfg->{iscsiprovider} eq 'iet') { $msg = PVE::Storage::LunCmd::Iet::run_lun_command($scfg, $timeout, $method, @params); + } elsif ($scfg->{iscsiprovider} eq 'LIO') { + $msg = PVE::Storage::LunCmd::LIO::run_lun_command($scfg, $timeout, $method, @params); } else { $zfs_unknown_scsi_provider->($scfg->{iscsiprovider}); } @@ -188,6 +193,10 @@ sub properties { description => "host group for comstar views", type => 'string', }, + lio_tpg => { + description => "target portal group for Linux LIO targets", + type => 'string', + }, }; } @@ -204,6 +213,7 @@ sub options { sparse => { optional => 1 }, comstar_hg => { optional => 1 }, comstar_tg => { optional => 1 }, + lio_tpg => { optional => 1 }, content => { optional => 1 }, bwlimit => { optional => 1 }, }; -- 2.39.2