Initial commit

Makefile

@@ -0,0 +1,42 @@
+PACKAGE=pve-iscsi-zfs-scst
+PKGVER=0.0.1
+
+DEB=${PACKAGE}_${PKGVER}_all.deb
+
+DESTDIR=
+
+PERL5DIR=${DESTDIR}/usr/share/perl5
+
+all:
+
+.PHONY: deb
+deb: ${DEB}
+${DEB}:
+	rm -rf build
+	rsync -a * build
+	cd build; dpkg-buildpackage -b -us -uc
+	pwd
+	echo ${DEB}
+	lintian ${DEB}
+
+install: PVE
+	install -d ${PERL5DIR}/PVE/Storage/Custom
+	install -m 0644 PVE/Storage/Custom/scstZFSPlugin.pm ${PERL5DIR}/PVE/Storage/Custom/
+	install -d ${PERL5DIR}/PVE/Storage/Custom/Shared
+	install -m 0644 PVE/Storage/Custom/Shared/ZFSPluginPlus.pm ${PERL5DIR}/PVE/Storage/Custom/Shared/
+	install -m 0644 QemuServer.patch ${PERL5DIR}/PVE/Storage/Custom/Shared/
+	install -d ${PERL5DIR}/PVE/Storage/Custom/LunCmd
+	install -m 0644 PVE/Storage/Custom/LunCmd/SCST.pm ${PERL5DIR}/PVE/Storage/Custom/LunCmd/
+
+.PHONY: upload
+upload: ${DEB}
+	tar cf - ${DEB} | ssh repoman@repo.proxmox.com -- upload --product pve --dist stretch
+
+distclean: clean
+
+clean:
+	rm -rf ./build *.deb *.changes *.buildinfo
+
+.PHONY: dinstall
+dinstall: ${DEB}
+	dpkg -i ${DEB}

PVE/Storage/Custom/LunCmd/SCST.pm

@@ -0,0 +1,565 @@
+package PVE::Storage::Custom::LunCmd::SCST;
+
+# iscsi storage running SCST on Linux
+#
+
+#
+#
+# https://pve.proxmox.com/wiki/Storage:_ZFS_over_iSCSI
+#
+#
+# mkdir /etc/pve/priv/zfs
+# ssh-keygen -f /etc/pve/priv/zfs/<IP>_id_rsa
+#
+# ssh-copy-id -i /etc/pve/priv/zfs/<IP>_id_rsa.pub root@<IP>
+#
+# ssh -i /etc/pve/priv/zfs/<IP>_id_rsa root@<IP>
+#
+# On one of the proxmox nodes:
+# 1) Login as root
+# 2) ssh-copy-id <ip_of_iscsi_storage>
+#
+#
+# On SCST LUN0 is ignored ...
+# make sure it is always configured as a dummy-device
+#
+use strict;
+use warnings;
+use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach);
+use Data::Dumper;
+
+use PVE::SafeSyslog;
+use POSIX qw(strftime);
+use File::Basename;
+
+sub get_base;
+
+# A logical unit can max have 16864 LUNs
+# http://manpages.ubuntu.com/manpages/precise/man5/ietd.conf.5.html
+#my $MAX_LUNS = 16864;
+my $MAX_LUNS = 20; # more handy for development / dumping used luns
+
+my $SETTINGS = undef;
+
+my @ssh_opts = ('-o', 'BatchMode=yes');
+my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
+my @scp_cmd = ('/usr/bin/scp', @ssh_opts);
+my $id_rsa_path = '/etc/pve/priv/zfs';
+
+my $scstadm = '/usr/local/sbin/scstadmin';
+
+
+my $debugmsg = sub {
+    my ($mtype, $msg, $logfd) = @_;
+
+    chomp $msg;
+
+    return if !$msg;
+
+#    my $pre = $debugstattxt->{$mtype} || $debugstattxt->{'err'};
+    my $pre = "SCST-ZFS";
+
+    my $timestr = strftime ("%b %d %H:%M:%S", CORE::localtime);
+
+    syslog ($mtype eq 'info' ? 'info' : 'err', "$pre $msg");
+
+    foreach my $line (split (/\n/, $msg)) {
+        print STDERR "$pre $line\n";
+        print $logfd "$timestr $pre $line\n" if $logfd;
+    }
+};
+
+my $execute_command = sub {
+    my ($scfg, $exec, $timeout, $method, @params) = @_;
+
+    my $msg = '';
+    my $err = undef;
+    my $target;
+    my $cmd;
+    my $res = ();
+
+    $timeout = 10 if !$timeout;
+
+    my $output = sub {
+        my $line = shift;
+        #$debugmsg->('info','#READ '.$line);
+        $msg .= "$line\n";
+    };
+
+    my $errfunc = sub {
+        my $line = shift;
+        #$debugmsg->('info','#ERR '.$line);
+        $err .= "$line";
+    };
+
+    if ($exec eq 'scp') {
+        $target = 'root@[' . $scfg->{portal} . ']';
+        $cmd = [@scp_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", '--', $method, "$target:$params[0]"];
+    } else {
+        $target = 'root@' . $scfg->{portal};
+        $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, '--', $method, @params];
+    }
+
+    eval {
+        run_command($cmd, outfunc => $output, errfunc => $errfunc, timeout => $timeout);
+    };
+    if ($@) {
+        $res = {
+            result => 0,
+            msg => $err,
+        }
+    } else {
+        $res = {
+            result => 1,
+            msg => $msg,
+        }
+    }
+
+    return $res;
+};
+
+
+my $get_target_path = sub {
+    my ($scfg) = @_;
+    if(defined $scfg->{ini_group} and length $scfg->{ini_group}) {
+        return "/sys/kernel/scst_tgt/targets/iscsi/$scfg->{target}/ini_groups/$scfg->{ini_group}/luns/";
+    } else {
+        return "/sys/kernel/scst_tgt/targets/iscsi/$scfg->{target}/luns/";
+    }
+};
+
+
+#
+# Read config via SCST's sysfs, for the target in question
+# no need for a parser with scst as we don't parse an actual config-file
+#
+# $SETTINS should list all LUN's
+# what more ?
+#
+my $read_configured_luns = sub {
+    my ($scfg, $timeout) = @_;
+    my $msg = '';
+    my $err = undef;
+    my $target;
+    my $targetPath;
+
+    my $output = sub {
+        my $line = shift;
+        $msg .= "$line\n";
+    };
+
+    my $errfunc = sub {
+        my $line = shift;
+        $err .= "$line";
+    };
+
+    $timeout = 10 if !$timeout;
+
+
+    $target = 'root@' . $scfg->{portal};
+
+    $targetPath = $get_target_path->($scfg);
+
+
+# esos find has no posix-extended and can't prinf !
+#    my @listLunParams = (
+#        $targetPath,
+#        ,'-maxdepth','1'
+#        ,'-regextype','posix-extended'
+#        ,'-regex',"\'^.*/luns/\([0-9]+\)\$\'"
+#        ,'-printf',"'%f\n'"
+#    );
+
+
+    my $findRegexp = "\'^.*luns/[0-9][0-9]\?[0-9]\?[0-9]\?\$'";
+    if( defined $scfg->{esos} && $scfg->{esos} eq "1") {
+        $findRegexp = "\'^.*luns/[0-9][0-9]\\?[0-9]\\?[0-9]\\?\$'";
+    }
+    my @listLunParams = (
+        $targetPath,
+        ,'-maxdepth','1'
+        ,'-regex', $findRegexp
+        ,'-print'
+    );
+    $debugmsg->('info',"find $targetPath -maxdepth 1 -regex $findRegexp -print");
+    my $res = $execute_command->($scfg,'ssh',$timeout,'find',@listLunParams);
+    die $res->{msg} unless $res->{result};
+
+    my @lunsList = split "\n", $res->{msg};
+    my $currentLun = undef;
+
+    foreach (@lunsList) {
+        $currentLun  = basename($_); # was genau kommt da ?
+        if($currentLun != 0) {
+
+            my $conf = undef;
+            $conf->{include} = 1;
+            $conf->{lun} = $currentLun;
+
+            my $pathRes = $execute_command->($scfg, 'ssh', undef,
+                'cat', "$targetPath$currentLun/device/filename");
+
+            die $pathRes->{msg} unless $pathRes->{result};
+
+            my @lunPath = split "\n", $pathRes->{msg};
+            $conf->{Path} = $lunPath[0];
+
+            push @{$SETTINGS->{luns}}, $conf;
+        } else {
+            # this is LUN 0 with vdisk_nullio
+            my $conf = undef;
+            $conf->{include} = 1;
+            $conf->{lun} = $currentLun;
+            $conf->{Path} = '/dev/null';
+
+            push @{$SETTINGS->{luns}}, $conf;
+        }
+        $SETTINGS->{used}->{$currentLun} = 1;
+    }
+    return $res->{msg};
+};
+
+
+#
+# read initial config, build settings and read luns
+#
+my $get_config = sub {
+    my ($scfg) = @_;
+
+    if(! defined $SETTINGS ) {
+        $SETTINGS->{target} = $scfg->{target};
+        $read_configured_luns->($scfg, undef);
+    }
+};
+
+my $clear_config = sub {
+    my ($scfg) = @_;
+    $SETTINGS = undef;
+};
+
+#
+# Write-out current config of running scst to /etc/scst.conf on Storage
+# Makes use of use scstadmin --write_config
+#
+my $update_config = sub {
+    my ($scfg) = @_;
+    my $file = "/etc/scst.conf";
+
+    my @params = ('--write_config',$file);
+    my $res = $execute_command->($scfg, 'ssh',undef,'scstadmin', @params);
+    die $res->{msg} unless $res->{result};
+
+    if( defined $scfg->{esos} && $scfg->{esos} eq "1") {
+        my $syncEsosRes = $execute_command->($scfg,'ssh',undef,'/usr/local/sbin/conf_sync.sh');
+        die $syncEsosRes->{msg} unless $syncEsosRes->{result};
+    }
+
+};
+
+my $get_target_tid = sub {
+    my ($scfg) = @_;
+
+    return 0;
+
+    # what exactly is output of this ?
+    # first * is iscsi
+    # find /sys/kernel/scst_tgt/targets/*/iqn*/enabled 2> /dev/null
+    # example-result for a enabled target:
+    # /sys/kernel/scst_tgt/targets/iscsi/iqn.2016-06-02.modula-shop-systems.de:tgt/enabled
+
+    my $proc = '/sys/kernel/scst_tgt/targets/iscsi/iqn*/enabled 2> /dev/null';
+    my $tid = undef;
+
+    my @params = ($proc);
+    my $res = $execute_command->($scfg, 'ssh', undef, 'find', @params);
+    die $res->{msg} unless $res->{result};
+    my @cfg = split "\n", $res->{msg};
+
+    foreach (@cfg) {
+
+        #        $debugmsg->('info','target is : '.$scfg->{target}.' $_'. $_);
+# gives:
+# SCST-ZFS target is :
+# iqn.2016-06-02.modula-shop-systems.de:tgt
+# $_/sys/kernel/scst_tgt/targets/iscsi/iqn.2016-06-02.modula-shop-systems.de:tgt/enabled
+        if ($_ =~ /^\s*tid:(\d+)\s+name:([\w\-\:\.]+)\s*$/) {
+            if ($2 && $2 eq $scfg->{target}) {
+                $tid = $1;
+                last;
+            }
+        }
+    }
+
+    return $tid;
+};
+
+#
+# Gets next free LUN-Number to be created
+#
+my $get_free_lun = sub {
+    my $usedLuns = ();
+    my $i=0;
+
+    # LUN0 on SCST is reserved:
+    $usedLuns->{$i} = 1;
+
+    for ($i = 1; $i < $MAX_LUNS; $i++) {
+        $usedLuns->{$i} = 0;
+    }
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+        $usedLuns->{$lun->{lun}} = 1;
+    }
+    $SETTINGS->{used} = $usedLuns;
+
+    for ($i = 0; $i < $MAX_LUNS; $i++) {
+        if(!$SETTINGS->{used}->{$i}) {
+            #if(!$usedLuns->{$i}) {
+
+#            $SETTINGS->{used}->{$i} = 1;
+#            $debugmsg->('info',"get_lu_name result $i");
+            return $i;
+        }
+    }
+};
+
+#
+# Removes LUN from Settings and free`s the slot / LUN Number
+#
+my $free_lu_name = sub {
+    my ($lu_name) = @_;
+    my $updated_luns;
+
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+        if ($lun->{lun} != $lu_name) {
+            push @$updated_luns, $lun;
+        }
+    }
+    $SETTINGS->{luns} = $updated_luns;
+    $SETTINGS->{used}->{$lu_name} = undef;
+};
+
+
+#
+# Creates a stub for a new LUN
+#
+my $make_lun = sub {
+    my ($scfg, $path) = @_;
+
+    die 'Maximum number of LUNs per target is $MAX_LUNS' if scalar @{$SETTINGS->{luns}} >= $MAX_LUNS;
+
+    $get_config->($scfg);
+
+    my $lun = $get_free_lun->();
+    my $conf = {
+        lun => $lun,
+        Path => $path,
+        Type => 'blockio',
+        include => 1,
+    };
+    push @{$SETTINGS->{luns}}, $conf;
+
+    return $conf;
+};
+
+my $list_view = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+    my $lun = undef;
+    my $object = $params[0];
+
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+        next unless $lun->{include} == 1;
+        if ($lun->{Path} =~ /^$object$/) {
+            return $lun->{lun} if (defined($lun->{lun}));
+            die "$lun->{Path}: Missing LUN";
+        }
+    }
+
+    return $lun;
+};
+
+#
+# Returns Path of LUN when present, undef when Lun is not found
+# Params: [0] - Path of LUN to search
+#
+my $list_lun = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+    my $name = undef;
+    my $searchLun = $params[0];
+
+    $clear_config->($scfg);
+    $get_config->($scfg);
+
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+        next unless $lun->{include} == 1;
+        if ($lun->{Path} =~ /^$searchLun$/) {
+            return $lun->{Path};
+        }
+    }
+    return $name;
+};
+
+#
+# Creates a new LUN on scst target
+#
+# First: crate device on scst
+# Second: add the device to target with LUN
+#
+my $create_lun = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+
+    $clear_config->($scfg);
+    $get_config->($scfg);
+
+    if ($list_lun->($scfg, $timeout, $method, @params)) {
+        die "$params[0]: LUN exists";
+    }
+    my $lun = $params[0];
+    $lun = $make_lun->($scfg, $lun);
+    my $tid = $get_target_tid->($scfg);
+    my $path = "Path=$lun->{Path},Type=$lun->{Type}";
+
+#    $debugmsg->('info', "CMD: --op new --tid=$tid --lun=$lun->{lun} --params $path");
+
+    # add device in scst
+    # echo "add_device disk1 filename=/disk1; blocksize=1" >/sys/kernel/scst_tgt/handlers/vdisk_fileio/mgmt
+    #
+    # add the device to target at lun:
+    # echo "add disk1 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.vlnb:tgt/luns/mgmt
+
+    my $scstDiskName = basename($lun->{Path});
+
+    my @paramsAddDevice = ('"','add_device',$scstDiskName,"filename=$lun->{Path}".'"',">/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt");
+    my $resAddDevice = $execute_command->($scfg, 'ssh', $timeout, 'echo', @paramsAddDevice);
+
+    my $targetPath;
+    $targetPath = $get_target_path->($scfg);
+    my @paramsAddLun = (
+        '"','add ',$scstDiskName,$lun->{lun},'"',
+        ">$targetPath/mgmt"
+    );
+    my $resAddLun = $execute_command->($scfg, 'ssh', $timeout, 'echo', @paramsAddLun);
+    do {
+        my @paramsRemoveDevice = (
+            "\"del_device $scstDiskName\"",
+            ">/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt"
+        );
+
+        my $resRemoveDevice = $execute_command->($scfg, 'ssh', $timeout, 'echo', @paramsRemoveDevice);
+
+        $free_lu_name->($lun->{lun});
+        $update_config->($scfg);
+        die "Could not create LUN: $lun->{lun} PATH: $lun->{Path} NAME: $scstDiskName: ".$resAddLun->{msg};
+    } unless $resAddLun->{result};
+
+    # make config persist here ?
+    $update_config->($scfg);
+    $clear_config->($scfg);
+
+    return $resAddLun->{msg};
+};
+
+my $delete_lun = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+    my $res = {msg => undef};
+    $get_config->($scfg);
+
+    my $path = $params[0];
+    my $targetPath;
+    $targetPath = $get_target_path->($scfg);
+#    my $tid = $get_target_tid->($scfg);
+
+#    $debugmsg->('info',"Delete LUN $path");
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+        if ($lun->{Path} eq $path) {
+#            $debugmsg->('info',"DO DELETE: $lun->{Path} $lun->{lun}");
+
+            # Delete LUN
+            @params = (
+                "\"del $lun->{lun}\"",
+                ">$targetPath/mgmt"
+            );
+            $res = $execute_command->($scfg, 'ssh', $timeout, 'echo', @params);
+            if ($res->{result}) {
+                $debugmsg->('info',"[OK - delete LUN]: $lun->{Path} : $res->{msg} -> call: free_lu_name: $lun->{lun}");
+
+                # Delete Device
+                my $scstDiskName = basename($lun->{Path});
+                @params = (
+                    "\"del_device $scstDiskName\"",
+                    ">/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt"
+                );
+                $res = $execute_command->($scfg, 'ssh', $timeout, 'echo', @params);
+
+                if ($res->{result}) {
+                    $free_lu_name->($lun->{lun});
+                    $update_config->($scfg);
+                    $clear_config->($scfg);
+                    $get_config->($scfg);
+                    last;
+                } else {
+                    die $res->{msg};
+                }
+            } else {
+                die $res->{msg};
+            }
+        }
+    }
+    return $res->{msg};
+};
+
+my $import_lun = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+
+    return $create_lun->($scfg, $timeout, $method, @params);
+};
+
+my $modify_lun = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+    my $res;
+    my $scstDevice = basename($params[1]);
+
+    @params = ('echo 1',">/sys/kernel/scst_tgt/devices/$scstDevice/resync_size");
+    $res = $execute_command->($scfg, 'ssh', $timeout, @params);
+    die $res->{msg} unless $res->{result};
+
+    return $res->{msg};
+};
+
+my $add_view = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+
+    return '';
+};
+
+my $get_lun_cmd_map = sub {
+    my ($method) = @_;
+
+    my $cmdmap = {
+        create_lu   =>  { cmd => $create_lun },
+        delete_lu   =>  { cmd => $delete_lun },
+        import_lu   =>  { cmd => $import_lun },
+        modify_lu   =>  { cmd => $modify_lun },
+        add_view    =>  { cmd => $add_view },
+        list_view   =>  { cmd => $list_view },
+        list_lu     =>  { cmd => $list_lun },
+    };
+
+    die "unknown command '$method'" unless exists $cmdmap->{$method};
+
+    return $cmdmap->{$method};
+};
+
+sub run_lun_command {
+    my ($scfg, $timeout, $method, @params) = @_;
+    $get_config->($scfg);
+    my $cmdmap = $get_lun_cmd_map->($method);
+    my $msg = $cmdmap->{cmd}->($scfg, $timeout, $method, @params);
+    return $msg;
+}
+
+sub get_base {
+    return '/dev';
+}
+
+1;
+

PVE/Storage/Custom/Shared/ZFSPluginPlus.pm

@@ -0,0 +1,322 @@
+package PVE::Storage::Custom::Shared::ZFSPluginPlus;
+
+use PVE::Storage::Custom::LunCmd::SCST;
+use Data::Dumper;
+use PVE::Tools qw(run_command);
+
+# inherit on the ZFSPlugin
+#use base qw(PVE::Storage::ZFSPlugin);
+@ISA = qw(PVE::Storage::ZFSPlugin);
+
+my @ssh_opts = ('-o', 'BatchMode=yes');
+my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
+my $id_rsa_path = '/etc/pve/priv/zfs';
+
+# plugin configuration
+
+# we want to allow copy of snap, currently dies at line 377 in ZFSPlugin.pm
+sub volume_has_feature {
+    my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
+
+    my $features = {
+		snapshot => { current => 1, snap => 1},
+		clone => { base => 1},
+		template => { current => 1},
+		copy => { base => 1, current => 1, snap=>1}, #added snap
+    };
+
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+	$class->parse_volname($volname);
+
+    my $key = undef;
+
+    if ($snapname) {
+	$key = 'snap';
+    } else {
+	$key = $isBase ? 'base' : 'current';
+    }
+
+    return 1 if $features->{$feature}->{$key};
+
+    return undef;
+}
+
+#
+# + added port, chap-auth and transport (iser)
+# + allow access to materialized snapshots
+#
+sub path {
+    my ($class, $scfg, $volname, $storeid, $snapname) = @_;
+
+#    die "direct access to snapshots not implemented"
+#	if defined($snapname);
+
+    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+	if(defined($snapname)) {
+		$name = materialize_snapshot($class,$scfg,$volname,$snapname);
+	}
+
+    my $target = $scfg->{target};
+    my $portal = $scfg->{portal};
+
+    my $guid = $class->zfs_get_lu_name($scfg, $name);
+    my $lun = $class->zfs_get_lun_number($scfg, $guid);
+
+	my $transport = $scfg->{transport};
+	if($transport eq 'rdma') {
+		$transport = 'iser';
+	} else {
+		$transport = 'iscsi';
+	}
+	my $authString = $scfg->{chap_user} && $scfg->{chap_pass}
+		? "$scfg->{chap_user}%$scfg->{chap_pass}@" : '';
+
+	my $port = $scfg->{port} ? ":$scfg->{port}" : '';
+
+    my $path = "$transport://$authString$portal$port/$target/$lun";
+
+    return ($path, $vmid, $vtype);
+}
+
+my $zfs_get_base = sub {
+	my ($scfg) = @_;
+
+	if ($scfg->{iscsiprovider} eq 'comstar') {
+		return PVE::Storage::LunCmd::Comstar::get_base;
+	} elsif ($scfg->{iscsiprovider} eq 'istgt') {
+		return PVE::Storage::LunCmd::Istgt::get_base;
+	} elsif ($scfg->{iscsiprovider} eq 'iet') {
+		return PVE::Storage::LunCmd::Iet::get_base;
+	} elsif ($scfg->{iscsiprovider} eq 'scstzfs') {
+		return PVE::Storage::Custom::LunCmd::SCST::get_base;
+	} elsif ($scfg->{iscsiprovider} eq 'freenas') {
+		return PVE::Storage::Custom::LunCmd::FreeNas::get_base;
+	} else {
+#		$zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
+	}
+};
+
+sub zfs_get_lu_name {
+	my ($class, $scfg, $zvol) = @_;
+
+	my $base = $zfs_get_base->($scfg);
+
+	my $object = ($zvol =~ /^.+\/.+/) ? "$base/$zvol" : "$base/$scfg->{pool}/$zvol";
+
+	my $lu_name = $class->zfs_request($scfg, undef, 'list_lu', $object);
+
+	return $lu_name if $lu_name;
+	die "Could not find lu_name for zvol $zvol base $base object $object";
+}
+
+
+#
+# Allow Activation of Snap via ZFS send/receive
+#
+sub activate_volume {
+    my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
+
+	if($snapname) {
+		materialize_snapshot($class,$scfg,$volname,$snapname);
+	}
+    return 1;
+}
+
+#
+# Allow Deactivation of Snap-Volume
+#
+sub deactivate_volume {
+    my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
+	if($snapname) {
+		$class->dematerialize_snapshot($scfg,$volname,$snapname);
+	}
+    return 1;
+}
+
+
+#
+# materializes a snapshot of volname by zfs send/revc and creates an iscsi LUN
+#
+sub materialize_snapshot {
+	my ($class, $scfg, $volname, $snapshot) = @_;
+
+	my $srcVol = get_snapshot_path($class,$scfg,$volname,$snapshot);
+	my $snapTmpVol = get_snapshot_tmp_name($class,$scfg,$volname,$snapshot);
+
+	my $base = $class->zfs_get_base($scfg);
+	my $object = ($zvol =~ /^.+\/.+/) ? "$base/$snapTmpVol" : "$base/$scfg->{pool}/$snapTmpVol";
+	my $lu_name = $class->zfs_request($scfg, undef, 'list_lu', $object);
+
+	if ($lu_name) {
+		return $snapTmpVol;
+	}
+
+	# create the clone if not present
+	my $cloneExists = $class->zfs_volume_exists($scfg,"$scfg->{pool}/$snapTmpVol");
+	if(!$cloneExists) {
+		$class->zfs_request($scfg, undef,
+			'clone', $srcVol, "$scfg->{pool}/$snapTmpVol"
+		);
+	}
+
+	# and add it to the iscsi-target
+	my $guid = $class->zfs_create_lu($scfg, $snapTmpVol);
+	$class->zfs_add_lun_mapping_entry($scfg, $snapTmpVol, $guid);
+
+	return $snapTmpVol;
+}
+
+#
+# removes lun and zvol for snapshot of volname
+#
+sub dematerialize_snapshot {
+	my ($class, $scfg, $volname, $snapname) = @_;
+	my $snapTmpVol = $class->get_snapshot_tmp_name($scfg,$volname,$snapname);
+
+	# remove iscsi-target if present
+	my $base = $class->zfs_get_base($scfg);
+	my $object = ($zvol =~ /^.+\/.+/) ? "$base/$snapTmpVol" : "$base/$scfg->{pool}/$snapTmpVol";
+	my $lu_name = $class->zfs_request($scfg, undef, 'list_lu', $object);
+	if ($lu_name) {
+		$class->free_image(undef, $scfg, $snapTmpVol, undef);
+	}
+
+	# destroy clone
+	$class->zfs_request($scfg,undef,
+		'destroy', "$scfg->{pool}/$snapTmpVol"
+	);
+
+	return 1;
+}
+
+
+#
+# Create a copy of snapshot from srcVol and provide as dstVol
+#
+sub volume_snapshot_copy {
+	my($class, $scfg, $srcVol, $snapshot, $dstVol) = @_;
+	my $materializedPath = $class->materialize_snapshot($scfg, $srcVol, $snapshot);
+
+	# remove iscsi-target of tmp-snap when present
+	my $src_guid = $class->zfs_get_lu_name($scfg, $materializedPath);
+	if( $src_guid ) {
+		$class->zfs_delete_lu($scfg, $materializedPath);
+	}
+
+	# free the image in case it is exported
+	my $dst_guid = $class->zfs_get_lu_name($scfg, $dstVol);
+	if( $dst_guid ) {
+		$class->zfs_delete_lu($scfg, $dstVol);
+	}
+
+	# zfs-destroy a present volume
+	$class->zfs_request($scfg, undef,
+		'destroy','-r',"$scfg->{pool}/$dstVol"
+	);
+
+	# create current snap on clone
+	$class->zfs_request($scfg,undef,
+		'snapshot',"$scfg->{pool}/$materializedPath\@now"
+	);
+
+	# zfs send/rcv from snap of clone
+	$class->zfs_request($scfg, 86400,
+		"send", '-v', "$scfg->{pool}/$materializedPath\@now", '|',
+		'zfs', 'recv', '-F', "$scfg->{pool}/$dstVol"
+	);
+
+	# rollback to @now on dstVol
+	$class->zfs_request($scfg, undef,
+		'rollback', "$scfg->{pool}/$dstVol\@now"
+	);
+
+	# destroy snap @now on dstVol
+	$class->zfs_request($scfg,undef,
+		'destroy', "$scfg->{pool}/$dstVol\@now"
+	);
+
+	# destroy tmp snap @now on clone
+	$class->zfs_request($scfg,undef,
+		'destroy', "$scfg->{pool}/$materializedPath\@now"
+	);
+
+	# attach iscsi-target for dstVol
+	my ($vtype, $dst_name, $vmid) = $class->parse_volname($dstVol);
+	my $dst_guid = $class->zfs_create_lu($scfg, $dst_name);
+	$class->zfs_add_lun_mapping_entry($scfg, $dst_name, $dst_guid);
+}
+
+sub volume_copy {
+	my($class, $scfg, $srcVol, $dstVol) = @_;
+
+	my $snap = 'volume-copy-base-tmp';
+
+	# test if tmp-snapshot exists and should be deleted
+	if ( $class->volume_has_snapshot($scfg, "$scfg->{pool}/$srcVol", $snap) ) {
+		$class->zfs_request($scfg, undef, 'destroy','-R', "$scfg->{pool}/$srcVol\@$snap");
+	}
+
+	# create a tmp snap from base
+	$class->zfs_request($scfg, undef, 'snapshot', "$scfg->{pool}/$srcVol\@$snap");
+
+	# volume_snapshot_copy
+	$class->volume_snapshot_copy($scfg,$srcVol,$snap,$dstVol);
+
+	# remove the tmp snapshot, including the clone (-R)
+	$class->zfs_request($scfg, undef, 'destroy','-R', "$scfg->{pool}/$srcVol\@$snap");
+}
+
+
+#
+# Test if snapshot exists in volume
+#
+sub volume_has_snapshot {
+	my ($class, $scfg, $volname, $snapname) = @_;
+
+	my $fullSnapName = "$volname\@$snapname";
+
+	my @params = ('-t', 'snapshot', '-o', 'name', '-s', 'creation');
+	my $text = $class->zfs_request($scfg, undef, 'list', @params);
+	my @snapshots = split(/\n/, $text);
+    foreach (@snapshots) {
+		if( $_ eq $fullSnapName) {
+			return 1;
+		}
+	}
+}
+
+#
+# test if a zvol exists
+#
+sub zfs_volume_exists {
+	my ($class, $scfg, $volname) = @_;
+
+	my $volumes_text = $class->zfs_request($scfg, 10,
+		'list', '-o', 'name', '-t', 'volume', '-Hr'
+	);
+	my @lines = split /\n/, $volumes_text;
+	foreach my $zvol (@lines) {
+		return 1 if $zvol eq $volname;
+	}
+}
+
+#
+# creates tmp-name for zvol a snapshot will be / has been created for volname
+#
+sub get_snapshot_tmp_name {
+	my ($class, $scfg, $volname, $snapshot) = @_;
+	return $volname.'-clone-at-'.$snapshot;
+}
+
+#
+# returns zvol name of snapshot for volname
+#
+sub get_snapshot_path {
+	my ($class, $scfg, $volname, $snapshot) = @_;
+	return "$scfg->{pool}/$volname\@$snapshot";
+}
+
+# Other parts of Storage implementation is identical to ZFSPlugin and therefore not changed
+
+1;

PVE/Storage/Custom/scstZFSPlugin.pm

@@ -0,0 +1,211 @@
+package PVE::Storage::Custom::scstZFSPlugin;
+
+eval {
+	require 'PVE/Storage/Custom/Shared/ZFSPluginPlus.pm';
+};
+
+use PVE::Storage::Custom::LunCmd::SCST;
+use Data::Dumper;
+use PVE::Tools qw(run_command);
+
+# inherit on the ZFSPlugin
+#use base qw(PVE::Storage::ZFSPlugin);
+@ISA = qw(PVE::Storage::Custom::Shared::ZFSPluginPlus);
+
+my @ssh_opts = ('-o', 'BatchMode=yes');
+my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
+my $id_rsa_path = '/etc/pve/priv/zfs';
+
+# plugin configuration
+sub api {
+	return 1;
+}
+
+sub type {
+	return 'scstzfs';
+}
+
+sub plugindata {
+	return {
+		content => [ { images => 1 }, {images => 1} ]
+	};
+}
+
+sub properties {
+	return {
+		nowritecache_scst => {
+			description => "disable write cache on the target",
+			type => 'boolean'
+		},
+		chap_user => {
+			type => 'string'
+		},
+		chap_pass => {
+			type => 'string'
+		},
+		volume_prefix => {
+			type => 'string'
+		},
+		port => {
+			type => 'int'
+		},
+		ini_group => {
+		    type => 'string'
+		},
+		esos => {
+			type => 'int'
+		}
+	}
+}
+
+sub options {
+    return {
+		nodes => { optional => 1 },
+		disable => { optional => 1 },
+		portal => { fixed => 1 },
+		target => { fixed => 1 },
+		iscsiprovider => {fixed => 1},
+		pool => { fixed => 1 },
+		blocksize => { fixed => 1 },
+		nowritecache => { optional => 1 },
+		sparse => { optional => 1 },
+		comstar_hg => { optional => 1 },
+		comstar_tg => { optional => 1 },
+		content => { optional => 1 },
+		shared => { fixed => 1 },
+		transport => { optional => 1, default => 'iscsi' }, #iscsi or rdma
+		chap_user => { optional => 1 },
+		chap_pass => { optional => 1 },
+		port => { optional => 1 },
+		volume_prefix => { optional => 1, default => '' }, #prefixes created volume-names - allows to share pool for multiple clusters
+		ini_group => { optional => 1, default => ''},
+		esos => { optional=>1, default => 0 }
+    };
+}
+
+my $lun_cmds = {
+    create_lu   => 1,
+    delete_lu   => 1,
+    import_lu   => 1,
+    modify_lu   => 1,
+    add_view    => 1,
+    list_view   => 1,
+    list_lu     => 1,
+};
+
+# SCST-specifics
+
+
+my $zfs_get_base = sub {
+    my ($scfg) = @_;
+	return PVE::Storage::Custom::LunCmd::SCST::get_base;
+};
+
+sub zfs_get_base {
+    my ($scfg) = @_;
+	return $zfs_get_base->($scfg);
+}
+
+sub zfs_request {
+    my ($class, $scfg, $timeout, $method, @params) = @_;
+
+    $timeout = PVE::RPCEnvironment::is_worker() ? 60*60 : 10
+	if !$timeout;
+
+    my $msg = '';
+
+    if ($lun_cmds->{$method}) {
+		$msg = PVE::Storage::Custom::LunCmd::SCST::run_lun_command($scfg, $timeout, $method, @params);
+    } else {
+
+		my $target = 'root@' . $scfg->{portal};
+
+		my $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target];
+
+		if ($method eq 'zpool_list') {
+			push @$cmd, 'zpool', 'list';
+		} else {
+			push @$cmd, 'zfs', $method;
+		}
+
+		push @$cmd, @params;
+
+		my $output = sub {
+			my $line = shift;
+			$msg .= "$line\n";
+		};
+
+#		print Dumper($cmd);
+
+        run_command($cmd, outfunc => $output, timeout => $timeout);
+    }
+
+    return $msg;
+}
+
+# we want to allow copy of snap, currently dies at line 377 in ZFSPlugin.pm
+sub volume_has_feature {
+    my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
+
+    my $features = {
+		snapshot => { current => 1, snap => 1},
+		clone => { base => 1},
+		template => { current => 1},
+		copy => { base => 1, current => 1, snap=>1}, #added snap
+    };
+
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+	$class->parse_volname($volname);
+
+    my $key = undef;
+
+    if ($snapname) {
+	$key = 'snap';
+    } else {
+	$key = $isBase ? 'base' : 'current';
+    }
+
+    return 1 if $features->{$feature}->{$key};
+
+    return undef;
+}
+
+#
+# + added port, chap-auth and transport (iser)
+# + allow access to materialized snapshots
+sub path {
+    my ($class, $scfg, $volname, $storeid, $snapname) = @_;
+
+#    die "direct access to snapshots not implemented"
+#	if defined($snapname);
+
+    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+	if(defined($snapname)) {
+		$name = $class->materialize_snapshot($scfg,$volname,$snapname);
+	}
+
+    my $target = $scfg->{target};
+    my $portal = $scfg->{portal};
+
+    my $guid = $class->zfs_get_lu_name($scfg, $name);
+    my $lun = $class->zfs_get_lun_number($scfg, $guid);
+
+	my $transport = $scfg->{transport};
+	if($transport eq 'rdma') {
+		$transport = 'iser';
+	} else {
+		$transport = 'iscsi';
+	}
+	my $authString = $scfg->{chap_user} && $scfg->{chap_pass} ? "$scfg->{chap_user}%$scfg->{chap_pass}@" : '';
+
+	my $port = $scfg->{port} ? ":$scfg->{port}" : ':3260'; #qemu-image insists on port !
+
+    my $path = "$transport://$authString$portal$port/$target/$lun";
+
+    return ($path, $vmid, $vtype);
+}
+
+# Other parts of Storage implementation is identical to ZFSPlugin and therefore not changed
+
+1;

QemuServer.patch

@@ -0,0 +1,50 @@
+5925a5926,5966
+> 	#BEGIN addition
+> 	my $src_storage_plugin = PVE::Storage::Plugin->lookup($src_scfg->{type});
+> 	my $dst_storage_plugin = PVE::Storage::Plugin->lookup($dst_scfg->{type});
+> 
+> 	# in many cases the storage may have a better idea how to convert:
+> 	# if src and dst format is the same
+> 	# and src_storeid is dst_storeid
+> 	# and the storage has the feature
+> 	# and the feature is implemented as volume_copy / volume_snapshot_copy
+> 	# let the storage do the work as it quite sure is faster
+> 	if ( $src_format eq $dst_format ) {
+> 		if( $src_storeid eq $dst_storeid ) {
+> 			# this is the same storage
+> 			# and storage is capable of materializing snaps
+> 			if( $snapname ) {
+> 				my $canCopySnap = PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, $snapname, undef) && $src_storage_plugin->can('volume_snapshot_copy');
+> 				if( $canCopySnap ) {
+> 					print "delegate to storage plugin volume_snapshot_copy (snap $snapname)\n";
+> 					$src_storage_plugin->volume_snapshot_copy($src_scfg, $src_volname, $snapname, $dst_volname);
+> 
+> 					$src_storage_plugin->deactivate_volume($src_storeid, $src_scfg, $src_volname, $snapname);
+> 
+> 					return 1;
+> 				}
+> 			}
+> 
+> 			my $canCopyVolume = PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, undef) && $src_storage_plugin->can('volume_copy');
+> 			if ( $canCopyVolume ) {
+> 				print "delegate to storage plugin: volume_copy (base)\n";
+> 				$src_storage_plugin->volume_copy($src_scfg, $src_volname, $dst_volname);
+> 
+> 				$src_storage_plugin->deactivate_volume($src_storeid, $src_scfg, $src_volname, $snapname);
+> 
+> 				return 1;
+> 			}
+> 		}
+> 		# @todo: two zfs-storages might be able to receive stream from each other
+> 	}
+> 	#END addition
+> 
+> 
+5949a5991,5995
+> 
+> 	# added: deactivate the src-volume, even on failure
+> 	# this can be safely called as qemu_img_convert is never called on a running volume (that is not a snap)
+> 	$src_storage_plugin->deactivate_volume($src_storeid, $src_scfg, $src_volname, $snapname);
+> 
+5950a5997
+> 

README.md

@@ -0,0 +1,124 @@
+# SCST Target Support for zfs over iSCSI on Proxmox
+
+Allows the use of an SCST Target with the zfs over iSCSI Storage of Proxmox.
+
+This plugin applies a Patch to QemuServer.pm that uses zfs send/rcv in favour of qemu when creating clones.
+
+The Patch also allows to create a clone from older snapshots than 'current' by materializing a snapshot before cloninng.
+
+The Patch will be reverted when the Plugin is uninstalled.
+
+Due to this Patch the dependencies are set very tight on a specific Version of libpve-storage-perl and qemu-server.
+
+**Use this Plugin at your own risk!**  
+**We don't take any responsibility for you loosing or corrupting data.  Make sure you have tested and validated in a non critical environment before putting this into production.**
+
+## Requirements
+
+Proxmox > 5.0  
+SCST based Storage with ZFS Filesystem, and a separate iSCSI Target for the Cluster.
+
+ 
+ESOS works fine but requires a bit of extra setup, see the notes.  
+Also make sure you have built ESOS with ZFS support from Sources.
+
+  
+The Storage can also be provided by a Proxmox Host running SCST.
+
+## Setup
+
+**Setup SSH-Keys and publish to the SCST Target**
+
+See: https://pve.proxmox.com/wiki/Storage:_ZFS_over_iSCSI#Platform_notes  
+
+Persist the SSH-Key on ESOS:
+
+See: https://github.com/parodyne/esos/issues/201
+ 
+```
+mount /mnt/conf/
+cp -r /root/.ssh /mnt/conf/etc/
+vi /mnt/conf/etc/rc.local
+=> add line: 
+cp -r /etc/.ssh /root/
+umount /mnt/conf/
+```
+
+
+**Create LUN0 (dummy) on the Target**
+
+/etc/scst.conf should look like this: 
+```
+HANDLER vdisk_nullio {
+    DEVICE dummy {
+        dummy 1
+    }
+}
+[...]
+TARGET_DRIVER iscsi {
+    enabled 1
+    TARGET <target name, i.e. iqn.2017-05.esos:81bfb> {
+        enabled 1
+        LUN 0 dummy
+    }
+}
+``` 
+
+**When running ESOS:**  
+Create an initiator group and add each Proxmox Host to it via TUI.  
+Get the intiator name from /etc/iscsi/initiatorname.iscsi 
+
+/etc/scst.conf should look like this:
+
+```
+HANDLER vdisk_nullio {
+    DEVICE dummy {
+        dummy 1
+    }
+}
+[...]
+TARGET_DRIVER iscsi {
+    enabled 1
+    TARGET <target name, i.e. iqn.2017-05.esos:81bfb> {
+        enabled 1
+        GROUP <intiator group name, i.e. pvedevcluster> {
+            LUN 0 dummy {
+                    read_only 1
+            }
+            
+            INITIATOR <iscsi initiator name of Proxmox host, i.e. iqn.1993-08.org.debian:01:cd8e73367472>
+            [... one line for each Proxmox Host]
+        }
+    }
+}
+
+
+``` 
+
+
+**Configure the Storage in /etc/pve/storage.cfg** 
+
+```
+scstzfs: esoszfs
+        blocksize 128k
+        iscsiprovider iet
+        pool <zfs pool name, i.e. tank/devfs>
+        portal <storage ip>
+        shared 1
+        target <traget name, i.e. iqn.2017-05.esos:81bfb>
+        content images
+```
+
+ESOS: add this 
+```
+        esos 1
+        ini_group <initiator group>
+```
+
+## Build and install from Source
+```
+git clone https://git.modula-shop-systems/modula/pve-iscsi-zfs-scst.git  
+cd pve-iscsi-zfs-scst  
+make deb  
+dpkg -i pve-iscsi-zfs-scst_0.0.1_all.deb
+```

debian/changelog

@@ -0,0 +1,5 @@
+pve-iscsi-zfs-scst (0.0.1) unstable; urgency=medium
+
+  * Initial build
+
+ -- Alexander Schmid <alex@modula-shop-systems.de>  Fri, 06 Oct 2017 17:46:05 +0200

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,13 @@
+Source: pve-iscsi-zfs-scst
+Section: perl
+Priority: optional
+Maintainer: Alexander Schmid <alex@modula-shop-systemsx.de>
+Build-Depends: debhelper (>=10), perl (>= 5.10.0-19), lintian
+Standards-Version: 0.0.1
+Homepage: https://git.modula-shop-systems.de/modula/pve-iscsi-zfs-scst/
+
+Architecture: all
+Depends: libpve-storage-perl (=5.0-15), qemu-server (=5.0-15), ${perl:Depends},
+Package: pve-iscsi-zfs-scst
+Description: zfs over iscsi with scst target
+ adds support for scst and ESOS target to the zfs over iscsi storage plugin

debian/copyright

@@ -0,0 +1,16 @@
+Copyright (C) 2017 Modula Shop Systems, Alexander Schmid
+
+This software is written by Modula Shop Systems <alex@modula-shop-systems.de>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.

debian/postinst

@@ -0,0 +1,2 @@
+cp /usr/share/perl5/PVE/QemuServer.pm /usr/share/perl5/PVE/Storage/Custom/Shared/QemuServer.orig.pm
+patch /usr/share/perl5/PVE/QemuServer.pm /usr/share/perl5/PVE/Storage/Custom/Shared/QemuServer.patch

debian/postrm

@@ -0,0 +1 @@
+mv /usr/share/perl5/PVE/Storage/Custom/Shared/QemuServer.orig.pm /usr/share/perl5/PVE/QemuServer.pm

debian/rules

@@ -0,0 +1,7 @@
+#!/usr/bin/make -f
+# See debhelper(7) (uncomment to enable)
+# output every command that modifies files on the build system.
+#DH_VERBOSE = 1
+
+%:
+	dh $@