Re: [PATCH v2 00/12] IB: Replace safe uses for ib_get_dma_mr with pd->local_dma_lkey

[Bart Van Assche <bart.vanassche-XdAiOPVOjttBDgjK7y7TUQ@public.gmane.org>]

[-- Attachment #1: Type: text/plain, Size: 1133 bytes --]

On 07/31/2015 04:14 PM, Jason Gunthorpe wrote:
> Do you by chance have a straightforward recipe to setup SRP and SRPT
> on two Linux's for this simple purpose?

Some time ago I wrote a script that loads the upstream SRP target driver 
and creates two LUNs for local RAM disks (attached to this e-mail). The 
only disadvantage of that script is that it is based on the obsolete 
lio-utils package.

In case you would prefer the SCST SRP target driver, the following 
instructions should be sufficient to build, install, load and configure 
it on an RPM based system (please replace the GUID shown below by a GUID 
of a port of the HCA in your the target system):

svn co svn://svn.code.sf.net/p/scst/svn/trunk scst-trunk ||
   git clone https://github.com/bvanassche/scst.git scst-trunk
cd scst-trunk
make rpm
su
rpm -U {,scstadmin/}rpmbuilddir/RPMS/x86_64/*.rpm
cat <<EOF >/etc/scst.conf
HANDLER vdisk_blockio {
     DEVICE brd {
	filename /dev/ram0
     }
}
TARGET_DRIVER ib_srpt {
     TARGET fe80:0000:0000:0000:24be:05ff:ffa9:cbb1 {
         enabled 1
         LUN 0 brd
     }
}
EOF
modprobe brd
/etc/init.d/scst restart

Bart.

[-- Attachment #2: restart-lio-srpt --]
[-- Type: text/plain, Size: 3426 bytes --]

#!/bin/bash

filesize=$((16*1024*1024))
hcas="$(cat /sys/devices/*/*/*/infiniband/*/ports/*/gids/0 | sed 's/^\(....\):\(....\):\(....\):\(....\):\(....\):\(....\):\(....\):\(....\)$/0x0000\2\3\4\5\6\7\8 0x\5\6\7\8 0x\1\2\3\4\5\6\7\8/')"
# Output of cat /sys/devices/*/*/*/infiniband/*/ports/*/gids/0 on the remote
# systems
initiators="\
0x00000000000000000002c9030003cca7 \
0x00000000000000000002c9030003cca8 \
0x00000000000000000002c9030005f34f \
0x00000000000000000002c9030005f350 \
0x00000000000000000002c90300a34271 \
0x00000000000000000002c90300a34272 \
0x00000000000000000002c90300fab7f1 \
0x00000000000000000002c90300fab7f2"

if [ ! -e /sys/module/configfs ]; then
  modprobe configfs
fi
if ! mount | grep -qw configfs; then
  mount -t configfs none /sys/kernel/config
fi

if cd /sys/kernel/config/target >&/dev/null; then
  for hca in ${hcas}; do
    if [ -e srpt/$hca/$hca/enable ]; then
      echo 0 >srpt/$hca/$hca/enable
    fi
  done
fi
cd /

rm -f /sys/kernel/config/target/srpt/*/*/acls/*/*/* >&/dev/null
rmdir /sys/kernel/config/target/srpt/*/*/acls/*/* >&/dev/null
rmdir /sys/kernel/config/target/srpt/*/*/acls/* >&/dev/null
rm -f /sys/kernel/config/target/srpt/*/*/lun/*/* >&/dev/null
rmdir /sys/kernel/config/target/srpt/*/*/lun/* >&/dev/null
rmdir /sys/kernel/config/target/srpt/*/* >&/dev/null
rmdir /sys/kernel/config/target/srpt/* >&/dev/null
rmdir /sys/kernel/config/target/srpt >&/dev/null

if [ -e /sys/module/ib_srpt ]; then
  rmmod ib_srpt
fi

rmdir /sys/kernel/config/target/core/*/* >&/dev/null
rmdir /sys/kernel/config/target/core/* >&/dev/null

#find /sys/kernel/config/target

for m in ib_srpt target_core_pscsi target_core_iblock target_core_file target_core_stgt target_core_user target_core_mod
do
  if [ -e /sys/module/$m ]; then
    rmmod $m
  fi
done

if [ "$1" = "stop" ]; then
  exit 0
fi

modprobe target_core_mod

if [ -e /sys/kernel/debug/dynamic_debug/control ]; then
  #echo 'module target_core_mod +p' > /sys/kernel/debug/dynamic_debug/control
  :
fi

insmod /lib/modules/$(uname -r)/kernel/drivers/infiniband/ulp/srpt/ib_srpt.ko srp_max_req_size=4200 || exit $?

if [ -e /sys/kernel/debug/dynamic_debug/control ]; then
  echo 'module ib_srpt +p' > /sys/kernel/debug/dynamic_debug/control
fi

modprobe target_core_file || exit $?
if [ ! -e /dev/ramdisk ]; then 
  dd if=/dev/zero of=/dev/ramdisk bs=${filesize} count=1
fi

if false && cd /sys/kernel/debug/tracing; then
  echo function >current_tracer
  { echo 'srpt_*'; echo 'transport_*'; } >set_ftrace_filter
  echo 1 >tracing_on
fi

vdev0="fileio_0/vdev0"
vdev1="fileio_1/vdev2"
vdev2="rd_dr_0/vdev1"
vdevs="$vdev0 $vdev1"
vdev_count=2
tcm_node --fileio $vdev0 /dev/ramdisk ${filesize}
tcm_node --fileio $vdev1 /dev/ramdisk 65536
if [ -e /sys/kernel/config/target/core/rd_dr_0 ]; then
  tcm_node --ramdisk $vdev2 2
  vdevs="$vdevs vdev2"
  vdev_count=3
fi
cd /sys/kernel/config/target || exit $?
mkdir srpt || exit $?
cd srpt || exit $?
for hca in ${hcas}; do
  mkdir $hca
  [ -e $hca ] || continue
  echo $hca
  mkdir $hca/$hca
  i=0
  for v in ${vdevs}; do
    mkdir $hca/$hca/lun/lun_$i
    ( cd $hca/$hca/lun/lun_$i && ln -s ../../../../../core/$v )
    i=$((i+1))
  done
  for ini in ${initiators}; do
  (
    cd $hca/$hca/acls
    mkdir ${ini}
    cd ${ini}
    for ((i = 0; i < $vdev_count; i++)) do
      ( mkdir lun_$i && cd lun_$i && ln -s ../../../lun/lun_$i )
    done
  )
  echo 1 >$hca/$hca/enable
  done
done